root/trunk/twisted/mail/pop3client.py

Revision 30752, 23.9 KB (checked in by exarkun, 15 months ago)

Rewrite the copyright headers to exclude date information.

Author: exarkun
Reviewer: glyph
Fixes: #4857

To avoid the need to perpetually update copyright dates in each file in Twisted,
remove the dates from most files and just leave them in the LICENSE file.

As a side effect, some files also have had a trailing newline added where it was
missing before.

Line 
1# -*- test-case-name: twisted.mail.test.test_pop3client -*-
2# Copyright (c) 2001-2004 Divmod Inc.
3# Copyright (c) Twisted Matrix Laboratories.
4# See LICENSE for details.
5
6"""
7POP3 client protocol implementation
8
9Don't use this module directly.  Use twisted.mail.pop3 instead.
10
11@author: Jp Calderone
12"""
13
14import re
15
16from twisted.python import log
17from twisted.python.hashlib import md5
18from twisted.internet import defer
19from twisted.protocols import basic
20from twisted.protocols import policies
21from twisted.internet import error
22from twisted.internet import interfaces
23
24OK = '+OK'
25ERR = '-ERR'
26
27class POP3ClientError(Exception):
28    """Base class for all exceptions raised by POP3Client.
29    """
30
31class InsecureAuthenticationDisallowed(POP3ClientError):
32    """Secure authentication was required but no mechanism could be found.
33    """
34
35class TLSError(POP3ClientError):
36    """
37    Secure authentication was required but either the transport does
38    not support TLS or no TLS context factory was supplied.
39    """
40
41class TLSNotSupportedError(POP3ClientError):
42    """
43    Secure authentication was required but the server does not support
44    TLS.
45    """
46
47class ServerErrorResponse(POP3ClientError):
48    """The server returned an error response to a request.
49    """
50    def __init__(self, reason, consumer=None):
51        POP3ClientError.__init__(self, reason)
52        self.consumer = consumer
53
54class LineTooLong(POP3ClientError):
55    """The server sent an extremely long line.
56    """
57
58class _ListSetter:
59    # Internal helper.  POP3 responses sometimes occur in the
60    # form of a list of lines containing two pieces of data,
61    # a message index and a value of some sort.  When a message
62    # is deleted, it is omitted from these responses.  The
63    # setitem method of this class is meant to be called with
64    # these two values.  In the cases where indexes are skipped,
65    # it takes care of padding out the missing values with None.
66    def __init__(self, L):
67        self.L = L
68    def setitem(self, (item, value)):
69        diff = item - len(self.L) + 1
70        if diff > 0:
71            self.L.extend([None] * diff)
72        self.L[item] = value
73
74
75def _statXform(line):
76    # Parse a STAT response
77    numMsgs, totalSize = line.split(None, 1)
78    return int(numMsgs), int(totalSize)
79
80
81def _listXform(line):
82    # Parse a LIST response
83    index, size = line.split(None, 1)
84    return int(index) - 1, int(size)
85
86
87def _uidXform(line):
88    # Parse a UIDL response
89    index, uid = line.split(None, 1)
90    return int(index) - 1, uid
91
92def _codeStatusSplit(line):
93    # Parse an +OK or -ERR response
94    parts = line.split(' ', 1)
95    if len(parts) == 1:
96        return parts[0], ''
97    return parts
98
99def _dotUnquoter(line):
100    """
101    C{'.'} characters which begin a line of a message are doubled to avoid
102    confusing with the terminating C{'.\\r\\n'} sequence.  This function
103    unquotes them.
104    """
105    if line.startswith('..'):
106        return line[1:]
107    return line
108
109class POP3Client(basic.LineOnlyReceiver, policies.TimeoutMixin):
110    """POP3 client protocol implementation class
111
112    Instances of this class provide a convenient, efficient API for
113    retrieving and deleting messages from a POP3 server.
114
115    @type startedTLS: C{bool}
116    @ivar startedTLS: Whether TLS has been negotiated successfully.
117
118
119    @type allowInsecureLogin: C{bool}
120    @ivar allowInsecureLogin: Indicate whether login() should be
121    allowed if the server offers no authentication challenge and if
122    our transport does not offer any protection via encryption.
123
124    @type serverChallenge: C{str} or C{None}
125    @ivar serverChallenge: Challenge received from the server
126
127    @type timeout: C{int}
128    @ivar timeout: Number of seconds to wait before timing out a
129    connection.  If the number is <= 0, no timeout checking will be
130    performed.
131    """
132
133    startedTLS = False
134    allowInsecureLogin = False
135    timeout = 0
136    serverChallenge = None
137
138    # Capabilities are not allowed to change during the session
139    # (except when TLS is negotiated), so cache the first response and
140    # use that for all later lookups
141    _capCache = None
142
143    # Regular expression to search for in the challenge string in the server
144    # greeting line.
145    _challengeMagicRe = re.compile('(<[^>]+>)')
146
147    # List of pending calls.
148    # We are a pipelining API but don't actually
149    # support pipelining on the network yet.
150    _blockedQueue = None
151
152    # The Deferred to which the very next result will go.
153    _waiting = None
154
155    # Whether we dropped the connection because of a timeout
156    _timedOut = False
157
158    # If the server sends an initial -ERR, this is the message it sent
159    # with it.
160    _greetingError = None
161
162    def _blocked(self, f, *a):
163        # Internal helper.  If commands are being blocked, append
164        # the given command and arguments to a list and return a Deferred
165        # that will be chained with the return value of the function
166        # when it eventually runs.  Otherwise, set up for commands to be
167
168        # blocked and return None.
169        if self._blockedQueue is not None:
170            d = defer.Deferred()
171            self._blockedQueue.append((d, f, a))
172            return d
173        self._blockedQueue = []
174        return None
175
176    def _unblock(self):
177        # Internal helper.  Indicate that a function has completed.
178        # If there are blocked commands, run the next one.  If there
179        # are not, set up for the next command to not be blocked.
180        if self._blockedQueue == []:
181            self._blockedQueue = None
182        elif self._blockedQueue is not None:
183            _blockedQueue = self._blockedQueue
184            self._blockedQueue = None
185
186            d, f, a = _blockedQueue.pop(0)
187            d2 = f(*a)
188            d2.chainDeferred(d)
189            # f is a function which uses _blocked (otherwise it wouldn't
190            # have gotten into the blocked queue), which means it will have
191            # re-set _blockedQueue to an empty list, so we can put the rest
192            # of the blocked queue back into it now.
193            self._blockedQueue.extend(_blockedQueue)
194
195
196    def sendShort(self, cmd, args):
197        # Internal helper.  Send a command to which a short response
198        # is expected.  Return a Deferred that fires when the response
199        # is received.  Block all further commands from being sent until
200        # the response is received.  Transition the state to SHORT.
201        d = self._blocked(self.sendShort, cmd, args)
202        if d is not None:
203            return d
204
205        if args:
206            self.sendLine(cmd + ' ' + args)
207        else:
208            self.sendLine(cmd)
209        self.state = 'SHORT'
210        self._waiting = defer.Deferred()
211        return self._waiting
212
213    def sendLong(self, cmd, args, consumer, xform):
214        # Internal helper.  Send a command to which a multiline
215        # response is expected.  Return a Deferred that fires when
216        # the entire response is received.  Block all further commands
217        # from being sent until the entire response is received.
218        # Transition the state to LONG_INITIAL.
219        d = self._blocked(self.sendLong, cmd, args, consumer, xform)
220        if d is not None:
221            return d
222
223        if args:
224            self.sendLine(cmd + ' ' + args)
225        else:
226            self.sendLine(cmd)
227        self.state = 'LONG_INITIAL'
228        self._xform = xform
229        self._consumer = consumer
230        self._waiting = defer.Deferred()
231        return self._waiting
232
233    # Twisted protocol callback
234    def connectionMade(self):
235        if self.timeout > 0:
236            self.setTimeout(self.timeout)
237
238        self.state = 'WELCOME'
239        self._blockedQueue = []
240
241    def timeoutConnection(self):
242        self._timedOut = True
243        self.transport.loseConnection()
244
245    def connectionLost(self, reason):
246        if self.timeout > 0:
247            self.setTimeout(None)
248
249        if self._timedOut:
250            reason = error.TimeoutError()
251        elif self._greetingError:
252            reason = ServerErrorResponse(self._greetingError)
253
254        d = []
255        if self._waiting is not None:
256            d.append(self._waiting)
257            self._waiting = None
258        if self._blockedQueue is not None:
259            d.extend([deferred for (deferred, f, a) in self._blockedQueue])
260            self._blockedQueue = None
261        for w in d:
262            w.errback(reason)
263
264    def lineReceived(self, line):
265        if self.timeout > 0:
266            self.resetTimeout()
267
268        state = self.state
269        self.state = None
270        state = getattr(self, 'state_' + state)(line) or state
271        if self.state is None:
272            self.state = state
273
274    def lineLengthExceeded(self, buffer):
275        # XXX - We need to be smarter about this
276        if self._waiting is not None:
277            waiting, self._waiting = self._waiting, None
278            waiting.errback(LineTooLong())
279        self.transport.loseConnection()
280
281    # POP3 Client state logic - don't touch this.
282    def state_WELCOME(self, line):
283        # WELCOME is the first state.  The server sends one line of text
284        # greeting us, possibly with an APOP challenge.  Transition the
285        # state to WAITING.
286        code, status = _codeStatusSplit(line)
287        if code != OK:
288            self._greetingError = status
289            self.transport.loseConnection()
290        else:
291            m = self._challengeMagicRe.search(status)
292
293            if m is not None:
294                self.serverChallenge = m.group(1)
295
296            self.serverGreeting(status)
297
298        self._unblock()
299        return 'WAITING'
300
301    def state_WAITING(self, line):
302        # The server isn't supposed to send us anything in this state.
303        log.msg("Illegal line from server: " + repr(line))
304
305    def state_SHORT(self, line):
306        # This is the state we are in when waiting for a single
307        # line response.  Parse it and fire the appropriate callback
308        # or errback.  Transition the state back to WAITING.
309        deferred, self._waiting = self._waiting, None
310        self._unblock()
311        code, status = _codeStatusSplit(line)
312        if code == OK:
313            deferred.callback(status)
314        else:
315            deferred.errback(ServerErrorResponse(status))
316        return 'WAITING'
317
318    def state_LONG_INITIAL(self, line):
319        # This is the state we are in when waiting for the first
320        # line of a long response.  Parse it and transition the
321        # state to LONG if it is an okay response; if it is an
322        # error response, fire an errback, clean up the things
323        # waiting for a long response, and transition the state
324        # to WAITING.
325        code, status = _codeStatusSplit(line)
326        if code == OK:
327            return 'LONG'
328        consumer = self._consumer
329        deferred = self._waiting
330        self._consumer = self._waiting = self._xform = None
331        self._unblock()
332        deferred.errback(ServerErrorResponse(status, consumer))
333        return 'WAITING'
334
335    def state_LONG(self, line):
336        # This is the state for each line of a long response.
337        # If it is the last line, finish things, fire the
338        # Deferred, and transition the state to WAITING.
339        # Otherwise, pass the line to the consumer.
340        if line == '.':
341            consumer = self._consumer
342            deferred = self._waiting
343            self._consumer = self._waiting = self._xform = None
344            self._unblock()
345            deferred.callback(consumer)
346            return 'WAITING'
347        else:
348            if self._xform is not None:
349                self._consumer(self._xform(line))
350            else:
351                self._consumer(line)
352            return 'LONG'
353
354
355    # Callbacks - override these
356    def serverGreeting(self, greeting):
357        """Called when the server has sent us a greeting.
358
359        @type greeting: C{str} or C{None}
360        @param greeting: The status message sent with the server
361        greeting.  For servers implementing APOP authentication, this
362        will be a challenge string.  .
363        """
364
365
366    # External API - call these (most of 'em anyway)
367    def startTLS(self, contextFactory=None):
368        """
369        Initiates a 'STLS' request and negotiates the TLS / SSL
370        Handshake.
371
372        @type contextFactory: C{ssl.ClientContextFactory} @param
373        contextFactory: The context factory with which to negotiate
374        TLS.  If C{None}, try to create a new one.
375
376        @return: A Deferred which fires when the transport has been
377        secured according to the given contextFactory, or which fails
378        if the transport cannot be secured.
379        """
380        tls = interfaces.ITLSTransport(self.transport, None)
381        if tls is None:
382            return defer.fail(TLSError(
383                "POP3Client transport does not implement "
384                "interfaces.ITLSTransport"))
385
386        if contextFactory is None:
387            contextFactory = self._getContextFactory()
388
389        if contextFactory is None:
390            return defer.fail(TLSError(
391                "POP3Client requires a TLS context to "
392                "initiate the STLS handshake"))
393
394        d = self.capabilities()
395        d.addCallback(self._startTLS, contextFactory, tls)
396        return d
397
398
399    def _startTLS(self, caps, contextFactory, tls):
400        assert not self.startedTLS, "Client and Server are currently communicating via TLS"
401
402        if 'STLS' not in caps:
403            return defer.fail(TLSNotSupportedError(
404                "Server does not support secure communication "
405                "via TLS / SSL"))
406
407        d = self.sendShort('STLS', None)
408        d.addCallback(self._startedTLS, contextFactory, tls)
409        d.addCallback(lambda _: self.capabilities())
410        return d
411
412
413    def _startedTLS(self, result, context, tls):
414        self.transport = tls
415        self.transport.startTLS(context)
416        self._capCache = None
417        self.startedTLS = True
418        return result
419
420
421    def _getContextFactory(self):
422        try:
423            from twisted.internet import ssl
424        except ImportError:
425            return None
426        else:
427            context = ssl.ClientContextFactory()
428            context.method = ssl.SSL.TLSv1_METHOD
429            return context
430
431
432    def login(self, username, password):
433        """Log into the server.
434
435        If APOP is available it will be used.  Otherwise, if TLS is
436        available an 'STLS' session will be started and plaintext
437        login will proceed.  Otherwise, if the instance attribute
438        allowInsecureLogin is set to True, insecure plaintext login
439        will proceed.  Otherwise, InsecureAuthenticationDisallowed
440        will be raised (asynchronously).
441
442        @param username: The username with which to log in.
443        @param password: The password with which to log in.
444
445        @rtype: C{Deferred}
446        @return: A deferred which fires when login has
447        completed.
448        """
449        d = self.capabilities()
450        d.addCallback(self._login, username, password)
451        return d
452
453
454    def _login(self, caps, username, password):
455        if self.serverChallenge is not None:
456            return self._apop(username, password, self.serverChallenge)
457
458        tryTLS = 'STLS' in caps
459
460        #If our transport supports switching to TLS, we might want to try to switch to TLS.
461        tlsableTransport = interfaces.ITLSTransport(self.transport, None) is not None
462
463        # If our transport is not already using TLS, we might want to try to switch to TLS.
464        nontlsTransport = interfaces.ISSLTransport(self.transport, None) is None
465
466        if not self.startedTLS and tryTLS and tlsableTransport and nontlsTransport:
467            d = self.startTLS()
468
469            d.addCallback(self._loginTLS, username, password)
470            return d
471
472        elif self.startedTLS or not nontlsTransport or self.allowInsecureLogin:
473            return self._plaintext(username, password)
474        else:
475            return defer.fail(InsecureAuthenticationDisallowed())
476
477
478    def _loginTLS(self, res, username, password):
479        return self._plaintext(username, password)
480
481    def _plaintext(self, username, password):
482        # Internal helper.  Send a username/password pair, returning a Deferred
483        # that fires when both have succeeded or fails when the server rejects
484        # either.
485        return self.user(username).addCallback(lambda r: self.password(password))
486
487    def _apop(self, username, password, challenge):
488        # Internal helper.  Computes and sends an APOP response.  Returns
489        # a Deferred that fires when the server responds to the response.
490        digest = md5(challenge + password).hexdigest()
491        return self.apop(username, digest)
492
493    def apop(self, username, digest):
494        """Perform APOP login.
495
496        This should be used in special circumstances only, when it is
497        known that the server supports APOP authentication, and APOP
498        authentication is absolutely required.  For the common case,
499        use L{login} instead.
500
501        @param username: The username with which to log in.
502        @param digest: The challenge response to authenticate with.
503        """
504        return self.sendShort('APOP', username + ' ' + digest)
505
506    def user(self, username):
507        """Send the user command.
508
509        This performs the first half of plaintext login.  Unless this
510        is absolutely required, use the L{login} method instead.
511
512        @param username: The username with which to log in.
513        """
514        return self.sendShort('USER', username)
515
516    def password(self, password):
517        """Send the password command.
518
519        This performs the second half of plaintext login.  Unless this
520        is absolutely required, use the L{login} method instead.
521
522        @param password: The plaintext password with which to authenticate.
523        """
524        return self.sendShort('PASS', password)
525
526    def delete(self, index):
527        """Delete a message from the server.
528
529        @type index: C{int}
530        @param index: The index of the message to delete.
531        This is 0-based.
532
533        @rtype: C{Deferred}
534        @return: A deferred which fires when the delete command
535        is successful, or fails if the server returns an error.
536        """
537        return self.sendShort('DELE', str(index + 1))
538
539    def _consumeOrSetItem(self, cmd, args, consumer, xform):
540        # Internal helper.  Send a long command.  If no consumer is
541        # provided, create a consumer that puts results into a list
542        # and return a Deferred that fires with that list when it
543        # is complete.
544        if consumer is None:
545            L = []
546            consumer = _ListSetter(L).setitem
547            return self.sendLong(cmd, args, consumer, xform).addCallback(lambda r: L)
548        return self.sendLong(cmd, args, consumer, xform)
549
550    def _consumeOrAppend(self, cmd, args, consumer, xform):
551        # Internal helper.  Send a long command.  If no consumer is
552        # provided, create a consumer that appends results to a list
553        # and return a Deferred that fires with that list when it is
554        # complete.
555        if consumer is None:
556            L = []
557            consumer = L.append
558            return self.sendLong(cmd, args, consumer, xform).addCallback(lambda r: L)
559        return self.sendLong(cmd, args, consumer, xform)
560
561    def capabilities(self, useCache=True):
562        """Retrieve the capabilities supported by this server.
563
564        Not all servers support this command.  If the server does not
565        support this, it is treated as though it returned a successful
566        response listing no capabilities.  At some future time, this may be
567        changed to instead seek out information about a server's
568        capabilities in some other fashion (only if it proves useful to do
569        so, and only if there are servers still in use which do not support
570        CAPA but which do support POP3 extensions that are useful).
571
572        @type useCache: C{bool}
573        @param useCache: If set, and if capabilities have been
574        retrieved previously, just return the previously retrieved
575        results.
576
577        @return: A Deferred which fires with a C{dict} mapping C{str}
578        to C{None} or C{list}s of C{str}.  For example::
579
580            C: CAPA
581            S: +OK Capability list follows
582            S: TOP
583            S: USER
584            S: SASL CRAM-MD5 KERBEROS_V4
585            S: RESP-CODES
586            S: LOGIN-DELAY 900
587            S: PIPELINING
588            S: EXPIRE 60
589            S: UIDL
590            S: IMPLEMENTATION Shlemazle-Plotz-v302
591            S: .
592
593        will be lead to a result of::
594
595            | {'TOP': None,
596            |  'USER': None,
597            |  'SASL': ['CRAM-MD5', 'KERBEROS_V4'],
598            |  'RESP-CODES': None,
599            |  'LOGIN-DELAY': ['900'],
600            |  'PIPELINING': None,
601            |  'EXPIRE': ['60'],
602            |  'UIDL': None,
603            |  'IMPLEMENTATION': ['Shlemazle-Plotz-v302']}
604        """
605        if useCache and self._capCache is not None:
606            return defer.succeed(self._capCache)
607
608        cache = {}
609        def consume(line):
610            tmp = line.split()
611            if len(tmp) == 1:
612                cache[tmp[0]] = None
613            elif len(tmp) > 1:
614                cache[tmp[0]] = tmp[1:]
615
616        def capaNotSupported(err):
617            err.trap(ServerErrorResponse)
618            return None
619
620        def gotCapabilities(result):
621            self._capCache = cache
622            return cache
623
624        d = self._consumeOrAppend('CAPA', None, consume, None)
625        d.addErrback(capaNotSupported).addCallback(gotCapabilities)
626        return d
627
628
629    def noop(self):
630        """Do nothing, with the help of the server.
631
632        No operation is performed.  The returned Deferred fires when
633        the server responds.
634        """
635        return self.sendShort("NOOP", None)
636
637
638    def reset(self):
639        """Remove the deleted flag from any messages which have it.
640
641        The returned Deferred fires when the server responds.
642        """
643        return self.sendShort("RSET", None)
644
645
646    def retrieve(self, index, consumer=None, lines=None):
647        """Retrieve a message from the server.
648
649        If L{consumer} is not None, it will be called with
650        each line of the message as it is received.  Otherwise,
651        the returned Deferred will be fired with a list of all
652        the lines when the message has been completely received.
653        """
654        idx = str(index + 1)
655        if lines is None:
656            return self._consumeOrAppend('RETR', idx, consumer, _dotUnquoter)
657
658        return self._consumeOrAppend('TOP', '%s %d' % (idx, lines), consumer, _dotUnquoter)
659
660
661    def stat(self):
662        """Get information about the size of this mailbox.
663
664        The returned Deferred will be fired with a tuple containing
665        the number or messages in the mailbox and the size (in bytes)
666        of the mailbox.
667        """
668        return self.sendShort('STAT', None).addCallback(_statXform)
669
670
671    def listSize(self, consumer=None):
672        """Retrieve a list of the size of all messages on the server.
673
674        If L{consumer} is not None, it will be called with two-tuples
675        of message index number and message size as they are received.
676        Otherwise, a Deferred which will fire with a list of B{only}
677        message sizes will be returned.  For messages which have been
678        deleted, None will be used in place of the message size.
679        """
680        return self._consumeOrSetItem('LIST', None, consumer, _listXform)
681
682
683    def listUID(self, consumer=None):
684        """Retrieve a list of the UIDs of all messages on the server.
685
686        If L{consumer} is not None, it will be called with two-tuples
687        of message index number and message UID as they are received.
688        Otherwise, a Deferred which will fire with of list of B{only}
689        message UIDs will be returned.  For messages which have been
690        deleted, None will be used in place of the message UID.
691        """
692        return self._consumeOrSetItem('UIDL', None, consumer, _uidXform)
693
694
695    def quit(self):
696        """Disconnect from the server.
697        """
698        return self.sendShort('QUIT', None)
699
700__all__ = [
701    # Exceptions
702    'InsecureAuthenticationDisallowed', 'LineTooLong', 'POP3ClientError',
703    'ServerErrorResponse', 'TLSError', 'TLSNotSupportedError',
704
705    # Protocol classes
706    'POP3Client']
Note: See TracBrowser for help on using the browser.