[Twisted-Python] twisted.news cleanups and tests

Neil Blakey-Milner nbm at mithrandr.moria.org
Fri Apr 11 10:09:16 EDT 2003


On Thu 2003-04-10 (23:35), Neil Blakey-Milner wrote:
> I'd like to know what people think about the following patch and tests.
> 
> These are my first tests for Twisted, so I'm not sure whether I'm
> stepping over any style rules.
> 
> The patch fixes the unusability of NewsShelf due to dirdbm wanting to
> try use directories as files.  It also stops us from putting '\n\r\n'
> into the start of the body, and it puts '\r\n' termination on the last
> line of the headers.  It moves placing the extra '\r\n' between the
> headers and body in the one place it's necessary - articleRequest.
> 
> I haven't tested NewsStorage or NewsStorageAugmentation yet, but these
> changes shouldn't be a problem.
> 
> The test cases detect the problems listed above, and perform some
> general sanity tests.

Here're updated versions of the patch and the test case.  We also pass
test_nntp now...

Neil
-- 
Neil Blakey-Milner
nbm at mithrandr.moria.org
-------------- next part --------------
Index: database.py
===================================================================
RCS file: /cvs/Twisted/twisted/news/database.py,v
retrieving revision 1.17
diff -u -r1.17 database.py
--- database.py	5 Nov 2002 03:42:12 -0000	1.17
+++ database.py	11 Apr 2003 08:33:39 -0000
@@ -50,15 +50,16 @@
 class Article:
     def __init__(self, head, body):
         self.body = body
-        head = map(lambda x: string.split(x, ': ', 1), string.split(head, '\r\n'))
         self.headers = {}
-        for i in head:
-            if len(i) == 0:
-                continue
-            elif len(i) == 1:
-                self.headers[string.lower(i[0])] = (i, '')
-            else:
-                self.headers[string.lower(i[0])] = tuple(i)
+        header = None
+        for line in head.split('\r\n'):
+            if line[0] in ' \t':
+                i = list(self.headers[header])
+                i[1] += '\r\n' + line
+            else:
+                i = line.split(': ', 1)
+                header = i[0].lower()
+            self.headers[header] = tuple(i)
 
         if not self.getHeader('Message-ID'):
             s = str(time.time()) + self.body
@@ -90,7 +91,7 @@
         headers = []
         for i in self.headers.values():
             headers.append('%s: %s' % i)
-        return string.join(headers, '\r\n')
+        return string.join(headers, '\r\n') + '\r\n'
     
     def overview(self):
         xover = []
@@ -275,7 +276,7 @@
 
     def postRequest(self, message):
         cleave = string.find(message, '\r\n\r\n')
-        headers, article = message[:cleave], message[cleave + 1:]
+        headers, article = message[:cleave], message[cleave + 4:]
 
         a = Article(headers, article)
         groups = string.split(a.getHeader('Newsgroups'))
@@ -359,7 +360,7 @@
         if self.db.has_key(group):
             if self.db[group].has_key(index):
                 a = self.db[group][index]
-                return defer.succeed((index, a.getHeader('Message-ID'), a.textHeaders() + a.body))
+                return defer.succeed((index, a.getHeader('Message-ID'), a.textHeaders() + '\r\n' + a.body))
             else:
                 return defer.fail(ERR_NOARTICLE)
         else:
@@ -431,7 +432,10 @@
         self.path = path
         self.mailhost = mailhost
 
-        self.dbm = dirdbm.Shelf(path)
+        if not os.path.exists(path):
+            os.mkdir(path)
+
+        self.dbm = dirdbm.Shelf(os.path.join(path, "newsshelf"))
         if not len(self.dbm.keys()):
             self.initialize()
 
@@ -500,7 +504,7 @@
 
     def postRequest(self, message):
         cleave = message.find('\r\n\r\n')
-        headers, article = message[:cleave], message[cleave + 1:]
+        headers, article = message[:cleave], message[cleave + 4:]
         
         article = Article(headers, article)
         groups = article.getHeader('Newsgroups').split()
@@ -600,7 +604,7 @@
         except KeyError:
             return defer.fail(NewsServerError("No such group: " + group))
         else:
-            return defer.succeed((index, a.getHeader('Message-ID'), a.textHeaders() + a.body))
+            return defer.succeed((index, a.getHeader('Message-ID'), a.textHeaders() + '\r\n' + a.body))
     
     
     def headRequest(self, group, index, id = None):
@@ -722,7 +726,7 @@
 
     def postRequest(self, message):
         cleave = string.find(message, '\r\n\r\n')
-        headers, article = message[:cleave], message[cleave + 1:]
+        headers, article = message[:cleave], message[cleave + 4:]
         article = Article(headers, article)
         return self.runInteraction(self._doPost, article)
 
-------------- next part --------------
# Twisted, the Framework of Your Internet
# Copyright (C) 2003 Matthew W. Lefkowitz
# 
# This library is free software; you can redistribute it and/or
# modify it under the terms of version 2.1 of the GNU Lesser General Public
# License as published by the Free Software Foundation.
# 
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
# 
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
import os, types
from pprint import pformat

from twisted.trial import unittest
from twisted.news import news, database
from twisted.internet import reactor

MESSAGE_ID = "f83ba57450ed0fd8ac9a472b847e830e"

POST_STRING = """Path: not-for-mail
From: <exarkun at somehost.domain.com>
Subject: a test
Newsgroups: alt.test.nntp
Organization: 
Summary: 
Keywords: 
Message-Id: %s
User-Agent: tin/1.4.5-20010409 ("One More Nightmare") (UNIX) (Linux/2.4.17 (i686))

this is a test
...
lala
moo
-- 
"One World, one Web, one Program." - Microsoft(R) promotional ad
"Ein Volk, ein Reich, ein Fuhrer." - Adolf Hitler
--
 10:56pm up 4 days, 4:42, 1 user, load average: 0.08, 0.08, 0.12
""" % (MESSAGE_ID)

class NewsTestCase(unittest.TestCase):
    def callback(self, result):
        self.result = result

    def errback(self, failure):
        try:
            self.fail('Errback called: ' + str(failure))
        except Exception, e:
            self.error = sys.exc_info()
            raise

    def timeout(self):
        reactor.crash()
        self.fail('Timed out')

    def setUp(self):
        self.backend = database.NewsShelf(None, 'news2.db')
        self.backend.addGroup('alt.test.nntp', 'y')
        self.backend.postRequest(POST_STRING.replace('\n', '\r\n'))

    def tearDown(self):
        try:
            del self.result
        except:
            pass
        try:
            del self.error
        except:
            pass

    def testArticleExists(self):
        d = self.backend.articleExistsRequest(MESSAGE_ID)
        self.assert_(d.result)

    def testArticleRequest(self):
        d = self.backend.articleRequest(None, None, MESSAGE_ID)
        d.addCallbacks(self.callback, self.errback)

        id = reactor.callLater(5, self.timeout)
        while not hasattr(self, 'result') and not hasattr(self, 'error'):
            reactor.iterate()
        try:
            id.cancel()
        except ValueError: pass

        error = getattr(self, 'error', None)
        if error:
            raise error[0], error[1], error[2]

        self.failUnless(type(self.result) == types.TupleType,
                        'callback result is wrong type: ' + str(self.result))
        self.failUnless(len(self.result) == 3,
                        'callback result list should have three entries: ' +
                        str(self.result))
        self.failUnless(self.result[1] == MESSAGE_ID,
                        "callback result Message-Id doesn't match: %s vs %s" %
                        (MESSAGE_ID, self.result[1]))
        self.failUnless(self.result[2].find('\r\n\r\n'),
                        "Can't find \\r\\n\\r\\n between header and body")

    def testHeadRequest(self):
        self.testArticleRequest()
        index = self.result[0]

        try: del self.result
        except: pass

        try: del self.error
        except: pass

        d = self.backend.headRequest("alt.test.nntp", index)
        d.addCallbacks(self.callback, self.errback)

        id = reactor.callLater(5, self.timeout)
        while not hasattr(self, 'result') and not hasattr(self, 'error'):
            reactor.iterate()
        try:
            id.cancel()
        except ValueError: pass

        error = getattr(self, 'error', None)
        if error:
            raise error[0], error[1], error[2]

        self.failUnless(self.result[1] == MESSAGE_ID,
                        "callback result Message-Id doesn't match: %s vs %s" %
                        (MESSAGE_ID, self.result[1]))

        self.failUnless(self.result[2][-2:] == '\r\n',
                        "headers must be \\r\\n terminated.")

    def testBodyRequest(self):
        self.testArticleRequest()
        index = self.result[0]

        try: del self.result
        except: pass

        try: del self.error
        except: pass

        d = self.backend.bodyRequest("alt.test.nntp", index)
        d.addCallbacks(self.callback, self.errback)

        id = reactor.callLater(5, self.timeout)
        while not hasattr(self, 'result') and not hasattr(self, 'error'):
            reactor.iterate()
        try:
            id.cancel()
        except ValueError: pass

        error = getattr(self, 'error', None)
        if error:
            raise error[0], error[1], error[2]

        self.failUnless(self.result[2][0:4] == 'this', "message body has been altered: " +
                        pformat(self.result[2][0:4]))


More information about the Twisted-Python mailing list