root / trunk / twisted / mail / imap4.py

# -*- test-case-name: twisted.mail.test.test_imap -*-
# Copyright (c) 2001-2009 Twisted Matrix Laboratories.
# See LICENSE for details.


"""
An IMAP4 protocol implementation

@author: Jp Calderone

To do::
  Suspend idle timeout while server is processing
  Use an async message parser instead of buffering in memory
  Figure out a way to not queue multi-message client requests (Flow? A simple callback?)
  Clarify some API docs (Query, etc)
  Make APPEND recognize (again) non-existent mailboxes before accepting the literal
"""

import rfc822
import base64
import binascii
import hmac
import re
import tempfile
import string
import time
import random
import types

import email.Utils

try:
    import cStringIO as StringIO
except:
    import StringIO

from zope.interface import implements, Interface

from twisted.protocols import basic
from twisted.protocols import policies
from twisted.internet import defer
from twisted.internet import error
from twisted.internet.defer import maybeDeferred
from twisted.python import log, text
from twisted.internet import interfaces

from twisted import cred
import twisted.cred.error
import twisted.cred.credentials

class MessageSet(object):
    """
    Essentially an infinite bitfield, with some extra features.

    @type getnext: Function taking C{int} returning C{int}
    @ivar getnext: A function that returns the next message number,
    used when iterating through the MessageSet. By default, a function
    returning the next integer is supplied, but as this can be rather
    inefficient for sparse UID iterations, it is recommended to supply
    one when messages are requested by UID.  The argument is provided
    as a hint to the implementation and may be ignored if it makes sense
    to do so (eg, if an iterator is being used that maintains its own
    state, it is guaranteed that it will not be called out-of-order).
    """
    _empty = []

    def __init__(self, start=_empty, end=_empty):
        """
        Create a new MessageSet()

        @type start: Optional C{int}
        @param start: Start of range, or only message number

        @type end: Optional C{int}
        @param end: End of range.
        """
        self._last = self._empty # Last message/UID in use
        self.ranges = [] # List of ranges included
        self.getnext = lambda x: x+1 # A function which will return the next
                                     # message id. Handy for UID requests.

        if start is self._empty:
            return

        if isinstance(start, types.ListType):
            self.ranges = start[:]
            self.clean()
        else:
            self.add(start,end)

    # Ooo.  A property.
    def last():
        def _setLast(self,value):
            if self._last is not self._empty:
                raise ValueError("last already set")

            self._last = value
            for i,(l,h) in enumerate(self.ranges):
                if l is not None:
                    break # There are no more Nones after this
                l = value
                if h is None:
                    h = value
                if l > h:
                    l, h = h, l
                self.ranges[i] = (l,h)

            self.clean()

        def _getLast(self):
            return self._last

        doc = '''
            "Highest" message number, refered to by "*".
            Must be set before attempting to use the MessageSet.
        '''
        return _getLast, _setLast, None, doc
    last = property(*last())

    def add(self, start, end=_empty):
        """
        Add another range

        @type start: C{int}
        @param start: Start of range, or only message number

        @type end: Optional C{int}
        @param end: End of range.
        """
        if end is self._empty:
            end = start

        if self._last is not self._empty:
            if start is None:
                start = self.last
            if end is None:
                end = self.last

        if start > end:
            # Try to keep in low, high order if possible
            # (But we don't know what None means, this will keep
            # None at the start of the ranges list)
            start, end = end, start

        self.ranges.append((start,end))
        self.clean()

    def __add__(self, other):
        if isinstance(other, MessageSet):
            ranges = self.ranges + other.ranges
            return MessageSet(ranges)
        else:
            res = MessageSet(self.ranges)
            try:
                res.add(*other)
            except TypeError:
                res.add(other)
            return res

    def extend(self, other):
        if isinstance(other, MessageSet):
            self.ranges.extend(other.ranges)
            self.clean()
        else:
            try:
                self.add(*other)
            except TypeError:
                self.add(other)

        return self

    def clean(self):
        """
        Clean ranges list, combining adjacent ranges
        """

        self.ranges.sort()

        oldl, oldh = None, None
        for i,(l,h) in enumerate(self.ranges):
            if l is None:
                continue
            # l is >= oldl and h is >= oldh due to sort()
            if oldl is not None and l <= oldh+1:
                l = oldl
                h = max(oldh,h)
                self.ranges[i-1] = None
                self.ranges[i] = (l,h)

            oldl,oldh = l,h

        self.ranges = filter(None, self.ranges)

    def __contains__(self, value):
        """
        May raise TypeError if we encounter unknown "high" values
        """
        for l,h in self.ranges:
            if l is None:
                raise TypeError(
                    "Can't determine membership; last value not set")
            if l <= value <= h:
                return True

        return False

    def _iterator(self):
        for l,h in self.ranges:
            l = self.getnext(l-1)
            while l <= h:
                yield l
                l = self.getnext(l)
                if l is None:
                    break

    def __iter__(self):
        if self.ranges and self.ranges[0][0] is None:
            raise TypeError("Can't iterate; last value not set")

        return self._iterator()

    def __len__(self):
        res = 0
        for l, h in self.ranges:
            if l is None:
                raise TypeError("Can't size object; last value not set")
            res += (h - l) + 1

        return res

    def __str__(self):
        p = []
        for low, high in self.ranges:
            if low == high:
                if low is None:
                    p.append('*')
                else:
                    p.append(str(low))
            elif low is None:
                p.append('%d:*' % (high,))
            else:
                p.append('%d:%d' % (low, high))
        return ','.join(p)

    def __repr__(self):
        return '<MessageSet %s>' % (str(self),)

    def __eq__(self, other):
        if isinstance(other, MessageSet):
            return self.ranges == other.ranges
        return False


class LiteralString:
    def __init__(self, size, defered):
        self.size = size
        self.data = []
        self.defer = defered

    def write(self, data):
        self.size -= len(data)
        passon = None
        if self.size > 0:
            self.data.append(data)
        else:
            if self.size:
                data, passon = data[:self.size], data[self.size:]
            else:
                passon = ''
            if data:
                self.data.append(data)
        return passon

    def callback(self, line):
        """
        Call defered with data and rest of line
        """
        self.defer.callback((''.join(self.data), line))

class LiteralFile:
    _memoryFileLimit = 1024 * 1024 * 10

    def __init__(self, size, defered):
        self.size = size
        self.defer = defered
        if size > self._memoryFileLimit:
            self.data = tempfile.TemporaryFile()
        else:
            self.data = StringIO.StringIO()

    def write(self, data):
        self.size -= len(data)
        passon = None
        if self.size > 0:
            self.data.write(data)
        else:
            if self.size:
                data, passon = data[:self.size], data[self.size:]
            else:
                passon = ''
            if data:
                self.data.write(data)
        return passon

    def callback(self, line):
        """
        Call defered with data and rest of line
        """
        self.data.seek(0,0)
        self.defer.callback((self.data, line))


class WriteBuffer:
    """Buffer up a bunch of writes before sending them all to a transport at once.
    """
    def __init__(self, transport, size=8192):
        self.bufferSize = size
        self.transport = transport
        self._length = 0
        self._writes = []

    def write(self, s):
        self._length += len(s)
        self._writes.append(s)
        if self._length > self.bufferSize:
            self.flush()

    def flush(self):
        if self._writes:
            self.transport.writeSequence(self._writes)
            self._writes = []
            self._length = 0


class Command:
    _1_RESPONSES = ('CAPABILITY', 'FLAGS', 'LIST', 'LSUB', 'STATUS', 'SEARCH', 'NAMESPACE')
    _2_RESPONSES = ('EXISTS', 'EXPUNGE', 'FETCH', 'RECENT')
    _OK_RESPONSES = ('UIDVALIDITY', 'UNSEEN', 'READ-WRITE', 'READ-ONLY', 'UIDNEXT', 'PERMANENTFLAGS')
    defer = None

    def __init__(self, command, args=None, wantResponse=(),
                 continuation=None, *contArgs, **contKw):
        self.command = command
        self.args = args
        self.wantResponse = wantResponse
        self.continuation = lambda x: continuation(x, *contArgs, **contKw)
        self.lines = []

    def format(self, tag):
        if self.args is None:
            return ' '.join((tag, self.command))
        return ' '.join((tag, self.command, self.args))

    def finish(self, lastLine, unusedCallback):
        send = []
        unuse = []
        for L in self.lines:
            names = parseNestedParens(L)
            N = len(names)
            if (N >= 1 and names[0] in self._1_RESPONSES or
                N >= 2 and names[1] in self._2_RESPONSES or
                N >= 2 and names[0] == 'OK' and isinstance(names[1], types.ListType) and names[1][0] in self._OK_RESPONSES):
                send.append(names)
            else:
                unuse.append(names)
        d, self.defer = self.defer, None
        d.callback((send, lastLine))
        if unuse:
            unusedCallback(unuse)

class LOGINCredentials(cred.credentials.UsernamePassword):
    def __init__(self):
        self.challenges = ['Password\0', 'User Name\0']
        self.responses = ['password', 'username']
        cred.credentials.UsernamePassword.__init__(self, None, None)

    def getChallenge(self):
        return self.challenges.pop()

    def setResponse(self, response):
        setattr(self, self.responses.pop(), response)

    def moreChallenges(self):
        return bool(self.challenges)

class PLAINCredentials(cred.credentials.UsernamePassword):
    def __init__(self):
        cred.credentials.UsernamePassword.__init__(self, None, None)

    def getChallenge(self):
        return ''

    def setResponse(self, response):
        parts = response[:-1].split('\0', 1)
        if len(parts) != 2:
            raise IllegalClientResponse("Malformed Response - wrong number of parts")
        self.username, self.password = parts

    def moreChallenges(self):
        return False

class IMAP4Exception(Exception):
    def __init__(self, *args):
        Exception.__init__(self, *args)

class IllegalClientResponse(IMAP4Exception): pass

class IllegalOperation(IMAP4Exception): pass

class IllegalMailboxEncoding(IMAP4Exception): pass

class IMailboxListener(Interface):
    """Interface for objects interested in mailbox events"""

    def modeChanged(writeable):
        """Indicates that the write status of a mailbox has changed.

        @type writeable: C{bool}
        @param writeable: A true value if write is now allowed, false
        otherwise.
        """

    def flagsChanged(newFlags):
        """Indicates that the flags of one or more messages have changed.

        @type newFlags: C{dict}
        @param newFlags: A mapping of message identifiers to tuples of flags
        now set on that message.
        """

    def newMessages(exists, recent):
        """Indicates that the number of messages in a mailbox has changed.

        @type exists: C{int} or C{None}
        @param exists: The total number of messages now in this mailbox.
        If the total number of messages has not changed, this should be
        C{None}.

        @type recent: C{int}
        @param recent: The number of messages now flagged \\Recent.
        If the number of recent messages has not changed, this should be
        C{None}.
        """

class IMAP4Server(basic.LineReceiver, policies.TimeoutMixin):
    """
    Protocol implementation for an IMAP4rev1 server.

    The server can be in any of four states:
        - Non-authenticated
        - Authenticated
        - Selected
        - Logout
    """
    implements(IMailboxListener)

    # Identifier for this server software
    IDENT = 'Twisted IMAP4rev1 Ready'

    # Number of seconds before idle timeout
    # Initially 1 minute.  Raised to 30 minutes after login.
    timeOut = 60

    POSTAUTH_TIMEOUT = 60 * 30

    # Whether STARTTLS has been issued successfully yet or not.
    startedTLS = False

    # Whether our transport supports TLS
    canStartTLS = False

    # Mapping of tags to commands we have received
    tags = None

    # The object which will handle logins for us
    portal = None

    # The account object for this connection
    account = None

    # Logout callback
    _onLogout = None

    # The currently selected mailbox
    mbox = None

    # Command data to be processed when literal data is received
    _pendingLiteral = None

    # Maximum length to accept for a "short" string literal
    _literalStringLimit = 4096

    # IChallengeResponse factories for AUTHENTICATE command
    challengers = None

    state = 'unauth'

    parseState = 'command'

    def __init__(self, chal = None, contextFactory = None, scheduler = None):
        if chal is None:
            chal = {}
        self.challengers = chal
        self.ctx = contextFactory
        if scheduler is None:
            scheduler = iterateInReactor
        self._scheduler = scheduler
        self._queuedAsync = []

    def capabilities(self):
        cap = {'AUTH': self.challengers.keys()}
        if self.ctx and self.canStartTLS:
            if not self.startedTLS and interfaces.ISSLTransport(self.transport, None) is None:
                cap['LOGINDISABLED'] = None
                cap['STARTTLS'] = None
        cap['NAMESPACE'] = None
        cap['IDLE'] = None
        return cap

    def connectionMade(self):
        self.tags = {}
        self.canStartTLS = interfaces.ITLSTransport(self.transport, None) is not None
        self.setTimeout(self.timeOut)
        self.sendServerGreeting()

    def connectionLost(self, reason):
        self.setTimeout(None)
        if self._onLogout:
            self._onLogout()
            self._onLogout = None

    def timeoutConnection(self):
        self.sendLine('* BYE Autologout; connection idle too long')
        self.transport.loseConnection()
        if self.mbox:
            self.mbox.removeListener(self)
            cmbx = ICloseableMailbox(self.mbox, None)
            if cmbx is not None:
                maybeDeferred(cmbx.close).addErrback(log.err)
            self.mbox = None
        self.state = 'timeout'

    def rawDataReceived(self, data):
        self.resetTimeout()
        passon = self._pendingLiteral.write(data)
        if passon is not None:
            self.setLineMode(passon)

    # Avoid processing commands while buffers are being dumped to
    # our transport
    blocked = None

    def _unblock(self):
        commands = self.blocked
        self.blocked = None
        while commands and self.blocked is None:
            self.lineReceived(commands.pop(0))
        if self.blocked is not None:
            self.blocked.extend(commands)

#    def sendLine(self, line):
#        print 'C:', repr(line)
#        return basic.LineReceiver.sendLine(self, line)

    def lineReceived(self, line):
#        print 'S:', repr(line)
        if self.blocked is not None:
            self.blocked.append(line)
            return

        self.resetTimeout()

        f = getattr(self, 'parse_' + self.parseState)
        try:
            f(line)
        except Exception, e:
            self.sendUntaggedResponse('BAD Server error: ' + str(e))
            log.err()

    def parse_command(self, line):
        args = line.split(None, 2)
        rest = None
        if len(args) == 3:
            tag, cmd, rest = args
        elif len(args) == 2:
            tag, cmd = args
        elif len(args) == 1:
            tag = args[0]
            self.sendBadResponse(tag, 'Missing command')
            return None
        else:
            self.sendBadResponse(None, 'Null command')
            return None

        cmd = cmd.upper()
        try:
            return self.dispatchCommand(tag, cmd, rest)
        except IllegalClientResponse, e:
            self.sendBadResponse(tag, 'Illegal syntax: ' + str(e))
        except IllegalOperation, e:
            self.sendNegativeResponse(tag, 'Illegal operation: ' + str(e))
        except IllegalMailboxEncoding, e:
            self.sendNegativeResponse(tag, 'Illegal mailbox name: ' + str(e))

    def parse_pending(self, line):
        d = self._pendingLiteral
        self._pendingLiteral = None
        self.parseState = 'command'
        d.callback(line)

    def dispatchCommand(self, tag, cmd, rest, uid=None):
        f = self.lookupCommand(cmd)
        if f:
            fn = f[0]
            parseargs = f[1:]
            self.__doCommand(tag, fn, [self, tag], parseargs, rest, uid)
        else:
            self.sendBadResponse(tag, 'Unsupported command')

    def lookupCommand(self, cmd):
        return getattr(self, '_'.join((self.state, cmd.upper())), None)

    def __doCommand(self, tag, handler, args, parseargs, line, uid):
        for (i, arg) in enumerate(parseargs):
            if callable(arg):
                parseargs = parseargs[i+1:]
                maybeDeferred(arg, self, line).addCallback(
                    self.__cbDispatch, tag, handler, args,
                    parseargs, uid).addErrback(self.__ebDispatch, tag)
                return
            else:
                args.append(arg)

        if line:
            # Too many arguments
            raise IllegalClientResponse("Too many arguments for command: " + repr(line))

        if uid is not None:
            handler(uid=uid, *args)
        else:
            handler(*args)

    def __cbDispatch(self, (arg, rest), tag, fn, args, parseargs, uid):
        args.append(arg)
        self.__doCommand(tag, fn, args, parseargs, rest, uid)

    def __ebDispatch(self, failure, tag):
        if failure.check(IllegalClientResponse):
            self.sendBadResponse(tag, 'Illegal syntax: ' + str(failure.value))
        elif failure.check(IllegalOperation):
            self.sendNegativeResponse(tag, 'Illegal operation: ' +
                                      str(failure.value))
        elif failure.check(IllegalMailboxEncoding):
            self.sendNegativeResponse(tag, 'Illegal mailbox name: ' +
                                      str(failure.value))
        else:
            self.sendBadResponse(tag, 'Server error: ' + str(failure.value))
            log.err(failure)

    def _stringLiteral(self, size):
        if size > self._literalStringLimit:
            raise IllegalClientResponse(
                "Literal too long! I accept at most %d octets" %
                (self._literalStringLimit,))
        d = defer.Deferred()
        self.parseState = 'pending'
        self._pendingLiteral = LiteralString(size, d)
        self.sendContinuationRequest('Ready for %d octets of text' % size)
        self.setRawMode()
        return d

    def _fileLiteral(self, size):
        d = defer.Deferred()
        self.parseState = 'pending'
        self._pendingLiteral = LiteralFile(size, d)
        self.sendContinuationRequest('Ready for %d octets of data' % size)
        self.setRawMode()
        return d

    def arg_astring(self, line):
        """
        Parse an astring from the line, return (arg, rest), possibly
        via a deferred (to handle literals)
        """
        line = line.strip()
        if not line:
            raise IllegalClientResponse("Missing argument")
        d = None
        arg, rest = None, None
        if line[0] == '"':
            try:
                spam, arg, rest = line.split('"',2)
                rest = rest[1:] # Strip space
            except ValueError:
                raise IllegalClientResponse("Unmatched quotes")
        elif line[0] == '{':
            # literal
            if line[-1] != '}':
                raise IllegalClientResponse("Malformed literal")
            try:
                size = int(line[1:-1])
            except ValueError:
                raise IllegalClientResponse("Bad literal size: " + line[1:-1])
            d = self._stringLiteral(size)
        else:
            arg = line.split(' ',1)
            if len(arg) == 1:
                arg.append('')
            arg, rest = arg
        return d or (arg, rest)

    # ATOM: Any CHAR except ( ) { % * " \ ] CTL SP (CHAR is 7bit)
    atomre = re.compile(r'(?P<atom>[^\](){%*"\\\x00-\x20\x80-\xff]+)( (?P<rest>.*$)|$)')

    def arg_atom(self, line):
        """
        Parse an atom from the line
        """
        if not line:
            raise IllegalClientResponse("Missing argument")
        m = self.atomre.match(line)
        if m:
            return m.group('atom'), m.group('rest')
        else:
            raise IllegalClientResponse("Malformed ATOM")

    def arg_plist(self, line):
        """
        Parse a (non-nested) parenthesised list from the line
        """
        if not line:
            raise IllegalClientResponse("Missing argument")

        if line[0] != "(":
            raise IllegalClientResponse("Missing parenthesis")

        i = line.find(")")

        if i == -1:
            raise IllegalClientResponse("Mismatched parenthesis")

        return (parseNestedParens(line[1:i],0), line[i+2:])

    def arg_literal(self, line):
        """
        Parse a literal from the line
        """
        if not line:
            raise IllegalClientResponse("Missing argument")

        if line[0] != '{':
            raise IllegalClientResponse("Missing literal")

        if line[-1] != '}':
            raise IllegalClientResponse("Malformed literal")

        try:
            size = int(line[1:-1])
        except ValueError:
            raise IllegalClientResponse("Bad literal size: " + line[1:-1])

        return self._fileLiteral(size)

    def arg_searchkeys(self, line):
        """
        searchkeys
        """
        query = parseNestedParens(line)
        # XXX Should really use list of search terms and parse into
        # a proper tree

        return (query, '')

    def arg_seqset(self, line):
        """
        sequence-set
        """
        rest = ''
        arg = line.split(' ',1)
        if len(arg) == 2:
            rest = arg[1]
        arg = arg[0]

        try:
            return (parseIdList(arg), rest)
        except IllegalIdentifierError, e:
            raise IllegalClientResponse("Bad message number " + str(e))

    def arg_fetchatt(self, line):
        """
        fetch-att
        """
        p = _FetchParser()
        p.parseString(line)
        return (p.result, '')

    def arg_flaglist(self, line):
        """
        Flag part of store-att-flag
        """
        flags = []
        if line[0] == '(':
            if line[-1] != ')':
                raise IllegalClientResponse("Mismatched parenthesis")
            line = line[1:-1]

        while line:
            m = self.atomre.search(line)
            if not m:
                raise IllegalClientResponse("Malformed flag")
            if line[0] == '\\' and m.start() == 1:
                flags.append('\\' + m.group('atom'))
            elif m.start() == 0:
                flags.append(m.group('atom'))
            else:
                raise IllegalClientResponse("Malformed flag")
            line = m.group('rest')

        return (flags, '')

    def arg_line(self, line):
        """
        Command line of UID command
        """
        return (line, '')

    def opt_plist(self, line):
        """
        Optional parenthesised list
        """
        if line.startswith('('):
            return self.arg_plist(line)
        else:
            return (None, line)

    def opt_datetime(self, line):
        """
        Optional date-time string
        """
        if line.startswith('"'):
            try:
                spam, date, rest = line.split('"',2)
            except IndexError:
                raise IllegalClientResponse("Malformed date-time")
            return (date, rest[1:])
        else:
            return (None, line)

    def opt_charset(self, line):
        """
        Optional charset of SEARCH command
        """
        if line[:7].upper() == 'CHARSET':
            arg = line.split(' ',2)
            if len(arg) == 1:
                raise IllegalClientResponse("Missing charset identifier")
            if len(arg) == 2:
                arg.append('')
            spam, arg, rest = arg
            return (arg, rest)
        else:
            return (None, line)

    def sendServerGreeting(self):
        msg = '[CAPABILITY %s] %s' % (' '.join(self.listCapabilities()), self.IDENT)
        self.sendPositiveResponse(message=msg)

    def sendBadResponse(self, tag = None, message = ''):
        self._respond('BAD', tag, message)

    def sendPositiveResponse(self, tag = None, message = ''):
        self._respond('OK', tag, message)

    def sendNegativeResponse(self, tag = None, message = ''):
        self._respond('NO', tag, message)

    def sendUntaggedResponse(self, message, async=False):
        if not async or (self.blocked is None):
            self._respond(message, None, None)
        else:
            self._queuedAsync.append(message)

    def sendContinuationRequest(self, msg = 'Ready for additional command text'):
        if msg:
            self.sendLine('+ ' + msg)
        else:
            self.sendLine('+')

    def _respond(self, state, tag, message):
        if state in ('OK', 'NO', 'BAD') and self._queuedAsync:
            lines = self._queuedAsync
            self._queuedAsync = []
            for msg in lines:
                self._respond(msg, None, None)
        if not tag:
            tag = '*'
        if message:
            self.sendLine(' '.join((tag, state, message)))
        else:
            self.sendLine(' '.join((tag, state)))

    def listCapabilities(self):
        caps = ['IMAP4rev1']
        for c, v in self.capabilities().iteritems():
            if v is None:
                caps.append(c)
            elif len(v):
                caps.extend([('%s=%s' % (c, cap)) for cap in v])
        return caps

    def do_CAPABILITY(self, tag):
        self.sendUntaggedResponse('CAPABILITY ' + ' '.join(self.listCapabilities()))
        self.sendPositiveResponse(tag, 'CAPABILITY completed')

    unauth_CAPABILITY = (do_CAPABILITY,)
    auth_CAPABILITY = unauth_CAPABILITY
    select_CAPABILITY = unauth_CAPABILITY
    logout_CAPABILITY = unauth_CAPABILITY

    def do_LOGOUT(self, tag):
        self.sendUntaggedResponse('BYE Nice talking to you')
        self.sendPositiveResponse(tag, 'LOGOUT successful')
        self.transport.loseConnection()

    unauth_LOGOUT = (do_LOGOUT,)
    auth_LOGOUT = unauth_LOGOUT
    select_LOGOUT = unauth_LOGOUT
    logout_LOGOUT = unauth_LOGOUT

    def do_NOOP(self, tag):
        self.sendPositiveResponse(tag, 'NOOP No operation performed')

    unauth_NOOP = (do_NOOP,)
    auth_NOOP = unauth_NOOP
    select_NOOP = unauth_NOOP
    logout_NOOP = unauth_NOOP

    def do_AUTHENTICATE(self, tag, args):
        args = args.upper().strip()
        if args not in self.challengers:
            self.sendNegativeResponse(tag, 'AUTHENTICATE method unsupported')
        else:
            self.authenticate(self.challengers[args](), tag)

    unauth_AUTHENTICATE = (do_AUTHENTICATE, arg_atom)

    def authenticate(self, chal, tag):
        if self.portal is None:
            self.sendNegativeResponse(tag, 'Temporary authentication failure')
            return

        self._setupChallenge(chal, tag)

    def _setupChallenge(self, chal, tag):
        try:
            challenge = chal.getChallenge()
        except Exception, e:
            self.sendBadResponse(tag, 'Server error: ' + str(e))
        else:
            coded = base64.encodestring(challenge)[:-1]
            self.parseState = 'pending'
            self._pendingLiteral = defer.Deferred()
            self.sendContinuationRequest(coded)
            self._pendingLiteral.addCallback(self.__cbAuthChunk, chal, tag)
            self._pendingLiteral.addErrback(self.__ebAuthChunk, tag)

    def __cbAuthChunk(self, result, chal, tag):
        try:
            uncoded = base64.decodestring(result)
        except binascii.Error:
            raise IllegalClientResponse("Malformed Response - not base64")

        chal.setResponse(uncoded)
        if chal.moreChallenges():
            self._setupChallenge(chal, tag)
        else:
            self.portal.login(chal, None, IAccount).addCallbacks(
                self.__cbAuthResp,
                self.__ebAuthResp,
                (tag,), None, (tag,), None
            )

    def __cbAuthResp(self, (iface, avatar, logout), tag):
        assert iface is IAccount, "IAccount is the only supported interface"
        self.account = avatar
        self.state = 'auth'
        self._onLogout = logout
        self.sendPositiveResponse(tag, 'Authentication successful')
        self.setTimeout(self.POSTAUTH_TIMEOUT)

    def __ebAuthResp(self, failure, tag):
        if failure.check(cred.error.UnauthorizedLogin):
            self.sendNegativeResponse(tag, 'Authentication failed: unauthorized')
        elif failure.check(cred.error.UnhandledCredentials):
            self.sendNegativeResponse(tag, 'Authentication failed: server misconfigured')
        else:
            self.sendBadResponse(tag, 'Server error: login failed unexpectedly')
            log.err(failure)

    def __ebAuthChunk(self, failure, tag):
        self.sendNegativeResponse(tag, 'Authentication failed: ' + str(failure.value))

    def do_STARTTLS(self, tag):
        if self.startedTLS:
            self.sendNegativeResponse(tag, 'TLS already negotiated')
        elif self.ctx and self.canStartTLS:
            self.sendPositiveResponse(tag, 'Begin TLS negotiation now')
            self.transport.startTLS(self.ctx)
            self.startedTLS = True
            self.challengers = self.challengers.copy()
            if 'LOGIN' not in self.challengers:
                self.challengers['LOGIN'] = LOGINCredentials
            if 'PLAIN' not in self.challengers:
                self.challengers['PLAIN'] = PLAINCredentials
        else:
            self.sendNegativeResponse(tag, 'TLS not available')

    unauth_STARTTLS = (do_STARTTLS,)

    def do_LOGIN(self, tag, user, passwd):
        if 'LOGINDISABLED' in self.capabilities():
            self.sendBadResponse(tag, 'LOGIN is disabled before STARTTLS')
            return

        maybeDeferred(self.authenticateLogin, user, passwd
            ).addCallback(self.__cbLogin, tag
            ).addErrback(self.__ebLogin, tag
            )

    unauth_LOGIN = (do_LOGIN, arg_astring, arg_astring)

    def authenticateLogin(self, user, passwd):
        """Lookup the account associated with the given parameters

        Override this method to define the desired authentication behavior.

        The default behavior is to defer authentication to C{self.portal}
        if it is not None, or to deny the login otherwise.

        @type user: C{str}
        @param user: The username to lookup

        @type passwd: C{str}
        @param passwd: The password to login with
        """
        if self.portal:
            return self.portal.login(
                cred.credentials.UsernamePassword(user, passwd),
                None, IAccount
            )
        raise cred.error.UnauthorizedLogin()

    def __cbLogin(self, (iface, avatar, logout), tag):
        if iface is not IAccount:
            self.sendBadResponse(tag, 'Server error: login returned unexpected value')
            log.err("__cbLogin called with %r, IAccount expected" % (iface,))
        else:
            self.account = avatar
            self._onLogout = logout
            self.sendPositiveResponse(tag, 'LOGIN succeeded')
            self.state = 'auth'
            self.setTimeout(self.POSTAUTH_TIMEOUT)

    def __ebLogin(self, failure, tag):
        if failure.check(cred.error.UnauthorizedLogin):
            self.sendNegativeResponse(tag, 'LOGIN failed')
        else:
            self.sendBadResponse(tag, 'Server error: ' + str(failure.value))
            log.err(failure)

    def do_NAMESPACE(self, tag):
        personal = public = shared = None
        np = INamespacePresenter(self.account, None)
        if np is not None:
            personal = np.getPersonalNamespaces()
            public = np.getSharedNamespaces()
            shared = np.getSharedNamespaces()
        self.sendUntaggedResponse('NAMESPACE ' + collapseNestedLists([personal, public, shared]))
        self.sendPositiveResponse(tag, "NAMESPACE command completed")

    auth_NAMESPACE = (do_NAMESPACE,)
    select_NAMESPACE = auth_NAMESPACE

    def _parseMbox(self, name):
        if isinstance(name, unicode):
            return name
        try:
            return name.decode('imap4-utf-7')
        except:
            log.err()
            raise IllegalMailboxEncoding(name)

    def _selectWork(self, tag, name, rw, cmdName):
        if self.mbox:
            self.mbox.removeListener(self)
            cmbx = ICloseableMailbox(self.mbox, None)
            if cmbx is not None:
                maybeDeferred(cmbx.close).addErrback(log.err)
            self.mbox = None
            self.state = 'auth'

        name = self._parseMbox(name)
        maybeDeferred(self.account.select, self._parseMbox(name), rw
            ).addCallback(self._cbSelectWork, cmdName, tag
            ).addErrback(self._ebSelectWork, cmdName, tag
            )

    def _ebSelectWork(self, failure, cmdName, tag):
        self.sendBadResponse(tag, "%s failed: Server error" % (cmdName,))
        log.err(failure)

    def _cbSelectWork(self, mbox, cmdName, tag):
        if mbox is None:
            self.sendNegativeResponse(tag, 'No such mailbox')
            return
        if '\\noselect' in [s.lower() for s in mbox.getFlags()]:
            self.sendNegativeResponse(tag, 'Mailbox cannot be selected')
            return

        flags = mbox.getFlags()
        self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS')
        self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT')
        self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags))
        self.sendPositiveResponse(None, '[UIDVALIDITY %d]' % mbox.getUIDValidity())

        s = mbox.isWriteable() and 'READ-WRITE' or 'READ-ONLY'
        mbox.addListener(self)
        self.sendPositiveResponse(tag, '[%s] %s successful' % (s, cmdName))
        self.state = 'select'
        self.mbox = mbox

    auth_SELECT = ( _selectWork, arg_astring, 1, 'SELECT' )
    select_SELECT = auth_SELECT

    auth_EXAMINE = ( _selectWork, arg_astring, 0, 'EXAMINE' )
    select_EXAMINE = auth_EXAMINE


    def do_IDLE(self, tag):
        self.sendContinuationRequest(None)
        self.parseTag = tag
        self.lastState = self.parseState
        self.parseState = 'idle'

    def parse_idle(self, *args):
        self.parseState = self.lastState
        del self.lastState
        self.sendPositiveResponse(self.parseTag, "IDLE terminated")
        del self.parseTag

    select_IDLE = ( do_IDLE, )
    auth_IDLE = select_IDLE


    def do_CREATE(self, tag, name):
        name = self._parseMbox(name)
        try:
            result = self.account.create(name)
        except MailboxException, c:
            self.sendNegativeResponse(tag, str(c))
        except:
            self.sendBadResponse(tag, "Server error encountered while creating mailbox")
            log.err()
        else:
            if result:
                self.sendPositiveResponse(tag, 'Mailbox created')
            else:
                self.sendNegativeResponse(tag, 'Mailbox not created')

    auth_CREATE = (do_CREATE, arg_astring)
    select_CREATE = auth_CREATE

    def do_DELETE(self, tag, name):
        name = self._parseMbox(name)
        if name.lower() == 'inbox':
            self.sendNegativeResponse(tag, 'You cannot delete the inbox')
            return
        try:
            self.account.delete(name)
        except MailboxException, m:
            self.sendNegativeResponse(tag, str(m))
        except:
            self.sendBadResponse(tag, "Server error encountered while deleting mailbox")
            log.err()
        else:
            self.sendPositiveResponse(tag, 'Mailbox deleted')

    auth_DELETE = (do_DELETE, arg_astring)
    select_DELETE = auth_DELETE

    def do_RENAME(self, tag, oldname, newname):
        oldname, newname = [self._parseMbox(n) for n in oldname, newname]
        if oldname.lower() == 'inbox' or newname.lower() == 'inbox':
            self.sendNegativeResponse(tag, 'You cannot rename the inbox, or rename another mailbox to inbox.')
            return
        try:
            self.account.rename(oldname, newname)
        except TypeError:
            self.sendBadResponse(tag, 'Invalid command syntax')
        except MailboxException, m:
            self.sendNegativeResponse(tag, str(m))
        except:
            self.sendBadResponse(tag, "Server error encountered while renaming mailbox")
            log.err()
        else:
            self.sendPositiveResponse(tag, 'Mailbox renamed')

    auth_RENAME = (do_RENAME, arg_astring, arg_astring)
    select_RENAME = auth_RENAME

    def do_SUBSCRIBE(self, tag, name):
        name = self._parseMbox(name)
        try:
            self.account.subscribe(name)
        except MailboxException, m:
            self.sendNegativeResponse(tag, str(m))
        except:
            self.sendBadResponse(tag, "Server error encountered while subscribing to mailbox")
            log.err()
        else:
            self.sendPositiveResponse(tag, 'Subscribed')

    auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring)
    select_SUBSCRIBE = auth_SUBSCRIBE

    def do_UNSUBSCRIBE(self, tag, name):
        name = self._parseMbox(name)
        try:
            self.account.unsubscribe(name)
        except MailboxException, m:
            self.sendNegativeResponse(tag, str(m))
        except:
            self.sendBadResponse(tag, "Server error encountered while unsubscribing from mailbox")
            log.err()
        else:
            self.sendPositiveResponse(tag, 'Unsubscribed')

    auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring)
    select_UNSUBSCRIBE = auth_UNSUBSCRIBE

    def _listWork(self, tag, ref, mbox, sub, cmdName):
        mbox = self._parseMbox(mbox)
        maybeDeferred(self.account.listMailboxes, ref, mbox
            ).addCallback(self._cbListWork, tag, sub, cmdName
            ).addErrback(self._ebListWork, tag
            )

    def _cbListWork(self, mailboxes, tag, sub, cmdName):
        for (name, box) in mailboxes:
            if not sub or self.account.isSubscribed(name):
                flags = box.getFlags()
                delim = box.getHierarchicalDelimiter()
                resp = (DontQuoteMe(cmdName), map(DontQuoteMe, flags), delim, name.encode('imap4-utf-7'))
                self.sendUntaggedResponse(collapseNestedLists(resp))
        self.sendPositiveResponse(tag, '%s completed' % (cmdName,))

    def _ebListWork(self, failure, tag):
        self.sendBadResponse(tag, "Server error encountered while listing mailboxes.")
        log.err(failure)

    auth_LIST = (_listWork, arg_astring, arg_astring, 0, 'LIST')
    select_LIST = auth_LIST

    auth_LSUB = (_listWork, arg_astring, arg_astring, 1, 'LSUB')
    select_LSUB = auth_LSUB

    def do_STATUS(self, tag, mailbox, names):
        mailbox = self._parseMbox(mailbox)
        maybeDeferred(self.account.select, mailbox, 0
            ).addCallback(self._cbStatusGotMailbox, tag, mailbox, names
            ).addErrback(self._ebStatusGotMailbox, tag
            )

    def _cbStatusGotMailbox(self, mbox, tag, mailbox, names):
        if mbox:
            maybeDeferred(mbox.requestStatus, names).addCallbacks(
                self.__cbStatus, self.__ebStatus,
                (tag, mailbox), None, (tag, mailbox), None
            )
        else:
            self.sendNegativeResponse(tag, "Could not open mailbox")

    def _ebStatusGotMailbox(self, failure, tag):
        self.sendBadResponse(tag, "Server error encountered while opening mailbox.")
        log.err(failure)

    auth_STATUS = (do_STATUS, arg_astring, arg_plist)
    select_STATUS = auth_STATUS

    def __cbStatus(self, status, tag, box):
        line = ' '.join(['%s %s' % x for x in status.iteritems()])
        self.sendUntaggedResponse('STATUS %s (%s)' % (box, line))
        self.sendPositiveResponse(tag, 'STATUS complete')

    def __ebStatus(self, failure, tag, box):
        self.sendBadResponse(tag, 'STATUS %s failed: %s' % (box, str(failure.value)))

    def do_APPEND(self, tag, mailbox, flags, date, message):
        mailbox = self._parseMbox(mailbox)
        maybeDeferred(self.account.select, mailbox
            ).addCallback(self._cbAppendGotMailbox, tag, flags, date, message
            ).addErrback(self._ebAppendGotMailbox, tag
            )

    def _cbAppendGotMailbox(self, mbox, tag, flags, date, message):
        if not mbox:
            self.sendNegativeResponse(tag, '[TRYCREATE] No such mailbox')
            return

        d = mbox.addMessage(message, flags, date)
        d.addCallback(self.__cbAppend, tag, mbox)
        d.addErrback(self.__ebAppend, tag)

    def _ebAppendGotMailbox(self, failure, tag):
        self.sendBadResponse(tag, "Server error encountered while opening mailbox.")
        log.err(failure)

    auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime,
                   arg_literal)
    select_APPEND = auth_APPEND

    def __cbAppend(self, result, tag, mbox):
        self.sendUntaggedResponse('%d EXISTS' % mbox.getMessageCount())
        self.sendPositiveResponse(tag, 'APPEND complete')

    def __ebAppend(self, failure, tag):
        self.sendBadResponse(tag, 'APPEND failed: ' + str(failure.value))

    def do_CHECK(self, tag):
        d = self.checkpoint()
        if d is None:
            self.__cbCheck(None, tag)
        else:
            d.addCallbacks(
                self.__cbCheck,
                self.__ebCheck,
                callbackArgs=(tag,),
                errbackArgs=(tag,)
            )
    select_CHECK = (do_CHECK,)

    def __cbCheck(self, result, tag):
        self.sendPositiveResponse(tag, 'CHECK completed')

    def __ebCheck(self, failure, tag):
        self.sendBadResponse(tag, 'CHECK failed: ' + str(failure.value))

    def checkpoint(self):
        """Called when the client issues a CHECK command.

        This should perform any checkpoint operations required by the server.
        It may be a long running operation, but may not block.  If it returns
        a deferred, the client will only be informed of success (or failure)
        when the deferred's callback (or errback) is invoked.
        """
        return None

    def do_CLOSE(self, tag):
        d = None
        if self.mbox.isWriteable():
            d = maybeDeferred(self.mbox.expunge)
        cmbx = ICloseableMailbox(self.mbox, None)
        if cmbx is not None:
            if d is not None:
                d.addCallback(lambda result: cmbx.close())
            else:
                d = maybeDeferred(cmbx.close)
        if d is not None:
            d.addCallbacks(self.__cbClose, self.__ebClose, (tag,), None, (tag,), None)
        else:
            self.__cbClose(None, tag)

    select_CLOSE = (do_CLOSE,)

    def __cbClose(self, result, tag):
        self.sendPositiveResponse(tag, 'CLOSE completed')
        self.mbox.removeListener(self)
        self.mbox = None
        self.state = 'auth'

    def __ebClose(self, failure, tag):
        self.sendBadResponse(tag, 'CLOSE failed: ' + str(failure.value))

    def do_EXPUNGE(self, tag):
        if self.mbox.isWriteable():
            maybeDeferred(self.mbox.expunge).addCallbacks(
                self.__cbExpunge, self.__ebExpunge, (tag,), None, (tag,), None
            )
        else:
            self.sendNegativeResponse(tag, 'EXPUNGE ignored on read-only mailbox')

    select_EXPUNGE = (do_EXPUNGE,)

    def __cbExpunge(self, result, tag):
        for e in result:
            self.sendUntaggedResponse('%d EXPUNGE' % e)
        self.sendPositiveResponse(tag, 'EXPUNGE completed')

    def __ebExpunge(self, failure, tag):
        self.sendBadResponse(tag, 'EXPUNGE failed: ' + str(failure.value))
        log.err(failure)

    def do_SEARCH(self, tag, charset, query, uid=0):
        sm = ISearchableMailbox(self.mbox, None)
        if sm is not None:
            maybeDeferred(sm.search, query, uid=uid).addCallbacks(
                self.__cbSearch, self.__ebSearch,
                (tag, self.mbox, uid), None, (tag,), None
            )
        else:
            s = parseIdList('1:*')
            maybeDeferred(self.mbox.fetch, s, uid=uid).addCallbacks(
                self.__cbManualSearch, self.__ebSearch,
                (tag, self.mbox, query, uid), None, (tag,), None
            )

    select_SEARCH = (do_SEARCH, opt_charset, arg_searchkeys)

    def __cbSearch(self, result, tag, mbox, uid):
        if uid:
            result = map(mbox.getUID, result)
        ids = ' '.join([str(i) for i in result])
        self.sendUntaggedResponse('SEARCH ' + ids)
        self.sendPositiveResponse(tag, 'SEARCH completed')

    def __cbManualSearch(self, result, tag, mbox, query, uid, searchResults = None):
        if searchResults is None:
            searchResults = []
        i = 0
        for (i, (id, msg)) in zip(range(5), result):
            if self.searchFilter(query, id, msg):
                if uid:
                    searchResults.append(str(msg.getUID()))
                else:
                    searchResults.append(str(id))
        if i == 4:
            from twisted.internet import reactor
            reactor.callLater(0, self.__cbManualSearch, result, tag, mbox, query, uid, searchResults)
        else:
            if searchResults:
                self.sendUntaggedResponse('SEARCH ' + ' '.join(searchResults))
            self.sendPositiveResponse(tag, 'SEARCH completed')

    def searchFilter(self, query, id, msg):
        while query:
            if not self.singleSearchStep(query, id, msg):
                return False
        return True

    def singleSearchStep(self, query, id, msg):
        q = query.pop(0)
        if isinstance(q, list):
            if not self.searchFilter(q, id, msg):
                return False
        else:
            c = q.upper()
            f = getattr(self, 'search_' + c)
            if f:
                if not f(query, id, msg):
                    return False
            else:
                # IMAP goes *out of its way* to be complex
                # Sequence sets to search should be specified
                # with a command, like EVERYTHING ELSE.
                try:
                    m = parseIdList(c)
                except:
                    log.err('Unknown search term: ' + c)
                else:
                    if id not in m:
                        return False
        return True

    def search_ALL(self, query, id, msg):
        return True

    def search_ANSWERED(self, query, id, msg):
        return '\\Answered' in msg.getFlags()

    def search_BCC(self, query, id, msg):
        bcc = msg.getHeaders(False, 'bcc').get('bcc', '')
        return bcc.lower().find(query.pop(0).lower()) != -1

    def search_BEFORE(self, query, id, msg):
        date = parseTime(query.pop(0))
        return rfc822.parsedate(msg.getInternalDate()) < date

    def search_BODY(self, query, id, msg):
        body = query.pop(0).lower()
        return text.strFile(body, msg.getBodyFile(), False)

    def search_CC(self, query, id, msg):
        cc = msg.getHeaders(False, 'cc').get('cc', '')
        return cc.lower().find(query.pop(0).lower()) != -1

    def search_DELETED(self, query, id, msg):
        return '\\Deleted' in msg.getFlags()

    def search_DRAFT(self, query, id, msg):
        return '\\Draft' in msg.getFlags()

    def search_FLAGGED(self, query, id, msg):
        return '\\Flagged' in msg.getFlags()

    def search_FROM(self, query, id, msg):
        fm = msg.getHeaders(False, 'from').get('from', '')
        return fm.lower().find(query.pop(0).lower()) != -1

    def search_HEADER(self, query, id, msg):
        hdr = query.pop(0).lower()
        hdr = msg.getHeaders(False, hdr).get(hdr, '')
        return hdr.lower().find(query.pop(0).lower()) != -1

    def search_KEYWORD(self, query, id, msg):
        query.pop(0)
        return False

    def search_LARGER(self, query, id, msg):
        return int(query.pop(0)) < msg.getSize()

    def search_NEW(self, query, id, msg):
        return '\\Recent' in msg.getFlags() and '\\Seen' not in msg.getFlags()

    def search_NOT(self, query, id, msg):
        return not self.singleSearchStep(query, id, msg)

    def search_OLD(self, query, id, msg):
        return '\\Recent' not in msg.getFlags()

    def search_ON(self, query, id, msg):
        date = parseTime(query.pop(0))
        return rfc822.parsedate(msg.getInternalDate()) == date

    def search_OR(self, query, id, msg):
        a = self.singleSearchStep(query, id, msg)
        b = self.singleSearchStep(query, id, msg)
        return a or b

    def search_RECENT(self, query, id, msg):
        return '\\Recent' in msg.getFlags()

    def search_SEEN(self, query, id, msg):
        return '\\Seen' in msg.getFlags()

    def search_SENTBEFORE(self, query, id, msg):
        date = msg.getHeader(False, 'date').get('date', '')
        date = rfc822.parsedate(date)
        return date < parseTime(query.pop(0))

    def search_SENTON(self, query, id, msg):
        date = msg.getHeader(False, 'date').get('date', '')
        date = rfc822.parsedate(date)
        return date[:3] == parseTime(query.pop(0))[:3]

    def search_SENTSINCE(self, query, id, msg):
        date = msg.getHeader(False, 'date').get('date', '')
        date = rfc822.parsedate(date)
        return date > parseTime(query.pop(0))

    def search_SINCE(self, query, id, msg):
        date = parseTime(query.pop(0))
        return rfc822.parsedate(msg.getInternalDate()) > date

    def search_SMALLER(self, query, id, msg):
        return int(query.pop(0)) > msg.getSize()

    def search_SUBJECT(self, query, id, msg):
        subj = msg.getHeaders(False, 'subject').get('subject', '')
        return subj.lower().find(query.pop(0).lower()) != -1

    def search_TEXT(self, query, id, msg):
        # XXX - This must search headers too
        body = query.pop(0).lower()
        return text.strFile(body, msg.getBodyFile(), False)

    def search_TO(self, query, id, msg):
        to = msg.getHeaders(False, 'to').get('to', '')
        return to.lower().find(query.pop(0).lower()) != -1

    def search_UID(self, query, id, msg):
        c = query.pop(0)
        m = parseIdList(c)
        return msg.getUID() in m

    def search_UNANSWERED(self, query, id, msg):
        return '\\Answered' not in msg.getFlags()

    def search_UNDELETED(self, query, id, msg):
        return '\\Deleted' not in msg.getFlags()

    def search_UNDRAFT(self, query, id, msg):
        return '\\Draft' not in msg.getFlags()

    def search_UNFLAGGED(self, query, id, msg):
        return '\\Flagged' not in msg.getFlags()

    def search_UNKEYWORD(self, query, id, msg):
        query.pop(0)
        return False

    def search_UNSEEN(self, query, id, msg):
        return '\\Seen' not in msg.getFlags()

    def __ebSearch(self, failure, tag):
        self.sendBadResponse(tag, 'SEARCH failed: ' + str(failure.value))
        log.err(failure)

    def do_FETCH(self, tag, messages, query, uid=0):
        if query:
            self._oldTimeout = self.setTimeout(None)
            maybeDeferred(self.mbox.fetch, messages, uid=uid
                ).addCallback(iter
                ).addCallback(self.__cbFetch, tag, query, uid
                ).addErrback(self.__ebFetch, tag
                )
        else:
            self.sendPositiveResponse(tag, 'FETCH complete')

    select_FETCH = (do_FETCH, arg_seqset, arg_fetchatt)

    def __cbFetch(self, results, tag, query, uid):
        if self.blocked is None:
            self.blocked = []
        try:
            id, msg = results.next()
        except StopIteration:
            # The idle timeout was suspended while we delivered results,
            # restore it now.
            self.setTimeout(self._oldTimeout)
            del self._oldTimeout

            # All results have been processed, deliver completion notification.

            # It's important to run this *after* resetting the timeout to "rig
            # a race" in some test code. writing to the transport will
            # synchronously call test code, which synchronously loses the
            # connection, calling our connectionLost method, which cancels the
            # timeout. We want to make sure that timeout is cancelled *after*
            # we reset it above, so that the final state is no timed
            # calls. This avoids reactor uncleanliness errors in the test
            # suite.
            # XXX: Perhaps loopback should be fixed to not call the user code
            # synchronously in transport.write?
            self.sendPositiveResponse(tag, 'FETCH completed')

            # Instance state is now consistent again (ie, it is as though
            # the fetch command never ran), so allow any pending blocked
            # commands to execute.
            self._unblock()
        else:
            self.spewMessage(id, msg, query, uid
                ).addCallback(lambda _: self.__cbFetch(results, tag, query, uid)
                ).addErrback(self.__ebSpewMessage
                )

    def __ebSpewMessage(self, failure):
        # This indicates a programming error.
        # There's no reliable way to indicate anything to the client, since we
        # may have already written an arbitrary amount of data in response to
        # the command.
        log.err(failure)
        self.transport.loseConnection()

    def spew_envelope(self, id, msg, _w=None, _f=None):
        if _w is None:
            _w = self.transport.write
        _w('ENVELOPE ' + collapseNestedLists([getEnvelope(msg)]))

    def spew_flags(self, id, msg, _w=None, _f=None):
        if _w is None:
            _w = self.transport.write
        _w('FLAGS ' + '(%s)' % (' '.join(msg.getFlags())))

    def spew_internaldate(self, id, msg, _w=None, _f=None):
        if _w is None:
            _w = self.transport.write
        idate = msg.getInternalDate()
        ttup = rfc822.parsedate_tz(idate)
        if ttup is None:
            log.msg("%d:%r: unpareseable internaldate: %r" % (id, msg, idate))
            raise IMAP4Exception("Internal failure generating INTERNALDATE")

        odate = time.strftime("%d-%b-%Y %H:%M:%S ", ttup[:9])
        if ttup[9] is None:
            odate = odate + "+0000"
        else:
            if ttup[9] >= 0:
                sign = "+"
            else:
                sign = "-"
            odate = odate + sign + string.zfill(str(((abs(ttup[9]) / 3600) * 100 + (abs(ttup[9]) % 3600) / 60)), 4)
        _w('INTERNALDATE ' + _quote(odate))

    def spew_rfc822header(self, id, msg, _w=None, _f=None):
        if _w is None:
            _w = self.transport.write
        hdrs = _formatHeaders(msg.getHeaders(True))
        _w('RFC822.HEADER ' + _literal(hdrs))

    def spew_rfc822text(self, id, msg, _w=None, _f=None):
        if _w is None:
            _w = self.transport.write
        _w('RFC822.TEXT ')
        _f()
        return FileProducer(msg.getBodyFile()
            ).beginProducing(self.transport
            )

    def spew_rfc822size(self, id, msg, _w=None, _f=None):
        if _w is None:
            _w = self.transport.write
        _w('RFC822.SIZE ' + str(msg.getSize()))

    def spew_rfc822(self, id, msg, _w=None, _f=None):
        if _w is None:
            _w = self.transport.write
        _w('RFC822 ')
        _f()
        mf = IMessageFile(msg, None)
        if mf is not None:
            return FileProducer(mf.open()
                ).beginProducing(self.transport
                )
        return MessageProducer(msg, None, self._scheduler
            ).beginProducing(self.transport
            )

    def spew_uid(self, id, msg, _w=None, _f=None):
        if _w is None:
            _w = self.transport.write
        _w('UID ' + str(msg.getUID()))

    def spew_bodystructure(self, id, msg, _w=None, _f=None):
        _w('BODYSTRUCTURE ' + collapseNestedLists([getBodyStructure(msg, True)]))

    def spew_body(self, part, id, msg, _w=None, _f=None):
        if _w is None:
            _w = self.transport.write
        for p in part.part:
            if msg.isMultipart():
                msg = msg.getSubPart(p)
            elif p > 0:
                # Non-multipart messages have an implicit first part but no
                # other parts - reject any request for any other part.
                raise TypeError("Requested subpart of non-multipart message")

        if part.header:
            hdrs = msg.getHeaders(part.header.negate, *part.header.fields)
            hdrs = _formatHeaders(hdrs)
            _w(str(part) + ' ' + _literal(hdrs))
        elif part.text:
            _w(str(part) + ' ')
            _f()
            return FileProducer(msg.getBodyFile()
                ).beginProducing(self.transport
                )
        elif part.mime:
            hdrs = _formatHeaders(msg.getHeaders(True))
            _w(str(part) + ' ' + _literal(hdrs))
        elif part.empty:
            _w(str(part) + ' ')
            _f()
            if part.part:
                return FileProducer(msg.getBodyFile()
                    ).beginProducing(self.transport
                    )
            else:
                mf = IMessageFile(msg, None)
                if mf is not None:
                    return FileProducer(mf.open()).beginProducing(self.transport)
                return MessageProducer(msg, None, self._scheduler).beginProducing(self.transport)

        else:
            _w('BODY ' + collapseNestedLists([getBodyStructure(msg)]))

    def spewMessage(self, id, msg, query, uid):
        wbuf = WriteBuffer(self.transport)
        write = wbuf.write
        flush = wbuf.flush
        def start():
            write('* %d FETCH (' % (id,))
        def finish():
            write(')\r\n')
        def space():
            write(' ')

        def spew():
            seenUID = False
            start()
            for part in query:
                if part.type == 'uid':
                    seenUID = True
                if part.type == 'body':
                    yield self.spew_body(part, id, msg, write, flush)
                else:
                    f = getattr(self, 'spew_' + part.type)
                    yield f(id, msg, write, flush)
                if part is not query[-1]:
                    space()
            if uid and not seenUID:
                space()
                yield self.spew_uid(id, msg, write, flush)
            finish()
            flush()
        return self._scheduler(spew())

    def __ebFetch(self, failure, tag):
        self.setTimeout(self._oldTimeout)
        del self._oldTimeout
        log.err(failure)
        self.sendBadResponse(tag, 'FETCH failed: ' + str(failure.value))

    def do_STORE(self, tag, messages, mode, flags, uid=0):
        mode = mode.upper()
        silent = mode.endswith('SILENT')
        if mode.startswith('+'):
            mode = 1
        elif mode.startswith('-'):
            mode = -1
        else:
            mode = 0

        maybeDeferred(self.mbox.store, messages, flags, mode, uid=uid).addCallbacks(
            self.__cbStore, self.__ebStore, (tag, self.mbox, uid, silent), None, (tag,), None
        )

    select_STORE = (do_STORE, arg_seqset, arg_atom, arg_flaglist)

    def __cbStore(self, result, tag, mbox, uid, silent):
        if result and not silent:
              for (k, v) in result.iteritems():
                  if uid:
                      uidstr = ' UID %d' % mbox.getUID(k)
                  else:
                      uidstr = ''
                  self.sendUntaggedResponse('%d FETCH (FLAGS (%s)%s)' %
                                            (k, ' '.join(v), uidstr))
        self.sendPositiveResponse(tag, 'STORE completed')

    def __ebStore(self, failure, tag):
        self.sendBadResponse(tag, 'Server error: ' + str(failure.value))

    def do_COPY(self, tag, messages, mailbox, uid=0):
        mailbox = self._parseMbox(mailbox)
        maybeDeferred(self.account.select, mailbox
            ).addCallback(self._cbCopySelectedMailbox, tag, messages, mailbox, uid
            ).addErrback(self._ebCopySelectedMailbox, tag
            )
    select_COPY = (do_COPY, arg_seqset, arg_astring)

    def _cbCopySelectedMailbox(self, mbox, tag, messages, mailbox, uid):
        if not mbox:
            self.sendNegativeResponse(tag, 'No such mailbox: ' + mailbox)
        else:
            maybeDeferred(self.mbox.fetch, messages, uid
                ).addCallback(self.__cbCopy, tag, mbox
                ).addCallback(self.__cbCopied, tag, mbox
                ).addErrback(self.__ebCopy, tag
                )

    def _ebCopySelectedMailbox(self, failure, tag):
        self.sendBadResponse(tag, 'Server error: ' + str(failure.value))

    def __cbCopy(self, messages, tag, mbox):
        # XXX - This should handle failures with a rollback or something
        addedDeferreds = []
        addedIDs = []
        failures = []

        fastCopyMbox = IMessageCopier(mbox, None)
        for (id, msg) in messages:
            if fastCopyMbox is not None:
                d = maybeDeferred(fastCopyMbox.copy, msg)
                addedDeferreds.append(d)
                continue

            # XXX - The following should be an implementation of IMessageCopier.copy
            # on an IMailbox->IMessageCopier adapter.

            flags = msg.getFlags()
            date = msg.getInternalDate()

            body = IMessageFile(msg, None)
            if body is not None:
                bodyFile = body.open()
                d = maybeDeferred(mbox.addMessage, bodyFile, flags, date)
            else:
                def rewind(f):
                    f.seek(0)
                    return f
                buffer = tempfile.TemporaryFile()
                d = MessageProducer(msg, buffer, self._scheduler
                    ).beginProducing(None
                    ).addCallback(lambda _, b=buffer, f=flags, d=date: mbox.addMessage(rewind(b), f, d)
                    )
            addedDeferreds.append(d)
        return defer.DeferredList(addedDeferreds)

    def __cbCopied(self, deferredIds, tag, mbox):
        ids = []
        failures = []
        for (status, result) in deferredIds:
            if status:
                ids.append(result)
            else:
                failures.append(result.value)
        if failures:
            self.sendNegativeResponse(tag, '[ALERT] Some messages were not copied')
        else:
            self.sendPositiveResponse(tag, 'COPY completed')

    def __ebCopy(self, failure, tag):
        self.sendBadResponse(tag, 'COPY failed:' + str(failure.value))
        log.err(failure)

    def do_UID(self, tag, command, line):
        command = command.upper()

        if command not in ('COPY', 'FETCH', 'STORE', 'SEARCH'):
            raise IllegalClientResponse(command)

        self.dispatchCommand(tag, command, line, uid=1)

    select_UID = (do_UID, arg_atom, arg_line)
    #
    # IMailboxListener implementation
    #
    def modeChanged(self, writeable):
        if writeable:
            self.sendUntaggedResponse(message='[READ-WRITE]', async=True)
        else:
            self.sendUntaggedResponse(message='[READ-ONLY]', async=True)

    def flagsChanged(self, newFlags):
        for (mId, flags) in newFlags.iteritems():
            msg = '%d FETCH (FLAGS (%s))' % (mId, ' '.join(flags))
            self.sendUntaggedResponse(msg, async=True)

    def newMessages(self, exists, recent):
        if exists is not None:
            self.sendUntaggedResponse('%d EXISTS' % exists, async=True)
        if recent is not None:
            self.sendUntaggedResponse('%d RECENT' % recent, async=True)


class UnhandledResponse(IMAP4Exception): pass

class NegativeResponse(IMAP4Exception): pass

class NoSupportedAuthentication(IMAP4Exception):
    def __init__(self, serverSupports, clientSupports):
        IMAP4Exception.__init__(self, 'No supported authentication schemes available')
        self.serverSupports = serverSupports
        self.clientSupports = clientSupports

    def __str__(self):
        return (IMAP4Exception.__str__(self)
            + ': Server supports %r, client supports %r'
            % (self.serverSupports, self.clientSupports))

class IllegalServerResponse(IMAP4Exception): pass

TIMEOUT_ERROR = error.TimeoutError()

class IMAP4Client(basic.LineReceiver, policies.TimeoutMixin):
    """IMAP4 client protocol implementation

    @ivar state: A string representing the state the connection is currently
    in.
    """
    implements(IMailboxListener)

    tags = None
    waiting = None
    queued = None
    tagID = 1
    state = None

    startedTLS = False

    # Number of seconds to wait before timing out a connection.
    # If the number is <= 0 no timeout checking will be performed.
    timeout = 0

    # Capabilities are not allowed to change during the session
    # So cache the first response and use that for all later
    # lookups
    _capCache = None

    _memoryFileLimit = 1024 * 1024 * 10

    # Authentication is pluggable.  This maps names to IClientAuthentication
    # objects.
    authenticators = None

    STATUS_CODES = ('OK', 'NO', 'BAD', 'PREAUTH', 'BYE')

    STATUS_TRANSFORMATIONS = {
        'MESSAGES': int, 'RECENT': int, 'UNSEEN': int
    }

    context = None

    def __init__(self, contextFactory = None):
        self.tags = {}
        self.queued = []
        self.authenticators = {}
        self.context = contextFactory

        self._tag = None
        self._parts = None
        self._lastCmd = None

    def registerAuthenticator(self, auth):
        """Register a new form of authentication

        When invoking the authenticate() method of IMAP4Client, the first
        matching authentication scheme found will be used.  The ordering is
        that in which the server lists support authentication schemes.

        @type auth: Implementor of C{IClientAuthentication}
        @param auth: The object to use to perform the client
        side of this authentication scheme.
        """
        self.authenticators[auth.getName().upper()] = auth

    def rawDataReceived(self, data):
        if self.timeout > 0:
            self.resetTimeout()

        self._pendingSize -= len(data)
        if self._pendingSize > 0:
            self._pendingBuffer.write(data)
        else:
            passon = ''
            if self._pendingSize < 0:
                data, passon = data[:self._pendingSize], data[self._pendingSize:]
            self._pendingBuffer.write(data)
            rest = self._pendingBuffer
            self._pendingBuffer = None
            self._pendingSize = None
            rest.seek(0, 0)
            self._parts.append(rest.read())
            self.setLineMode(passon.lstrip('\r\n'))

#    def sendLine(self, line):
#        print 'S:', repr(line)
#        return basic.LineReceiver.sendLine(self, line)

    def _setupForLiteral(self, rest, octets):
        self._pendingBuffer = self.messageFile(octets)
        self._pendingSize = octets
        if self._parts is None:
            self._parts = [rest, '\r\n']
        else:
            self._parts.extend([rest, '\r\n'])
        self.setRawMode()

    def connectionMade(self):
        if self.timeout > 0:
            self.setTimeout(self.timeout)

    def connectionLost(self, reason):
        """We are no longer connected"""
        if self.timeout > 0:
            self.setTimeout(None)
        if self.queued is not None:
            queued = self.queued
            self.queued = None
            for cmd in queued:
                cmd.defer.errback(reason)
        if self.tags is not None:
            tags = self.tags
            self.tags = None
            for cmd in tags.itervalues():
                if cmd is not None and cmd.defer is not None:
                    cmd.defer.errback(reason)


    def lineReceived(self, line):
        """
        Attempt to parse a single line from the server.

        @type line: C{str}
        @param line: The line from the server, without the line delimiter.

        @raise IllegalServerResponse: If the line or some part of the line
            does not represent an allowed message from the server at this time.
        """
#        print 'C: ' + repr(line)
        if self.timeout > 0:
            self.resetTimeout()

        lastPart = line.rfind('{')
        if lastPart != -1:
            lastPart = line[lastPart + 1:]
            if lastPart.endswith('}'):
                # It's a literal a-comin' in
                try:
                    octets = int(lastPart[:-1])
                except ValueError:
                    raise IllegalServerResponse(line)
                if self._parts is None:
                    self._tag, parts = line.split(None, 1)
                else:
                    parts = line
                self._setupForLiteral(parts, octets)
                return

        if self._parts is None:
            # It isn't a literal at all
            self._regularDispatch(line)
        else:
            # If an expression is in progress, no tag is required here
            # Since we didn't find a literal indicator, this expression
            # is done.
            self._parts.append(line)
            tag, rest = self._tag, ''.join(self._parts)
            self._tag = self._parts = None
            self.dispatchCommand(tag, rest)

    def timeoutConnection(self):
        if self._lastCmd and self._lastCmd.defer is not None:
            d, self._lastCmd.defer = self._lastCmd.defer, None
            d.errback(TIMEOUT_ERROR)

        if self.queued:
            for cmd in self.queued:
                if cmd.defer is not None:
                    d, cmd.defer = cmd.defer, d
                    d.errback(TIMEOUT_ERROR)

        self.transport.loseConnection()

    def _regularDispatch(self, line):
        parts = line.split(None, 1)
        if len(parts) != 2:
            parts.append('')
        tag, rest = parts
        self.dispatchCommand(tag, rest)

    def messageFile(self, octets):
        """Create a file to which an incoming message may be written.

        @type octets: C{int}
        @param octets: The number of octets which will be written to the file

        @rtype: Any object which implements C{write(string)} and
        C{seek(int, int)}
        @return: A file-like object
        """
        if octets > self._memoryFileLimit:
            return tempfile.TemporaryFile()
        else:
            return StringIO.StringIO()

    def makeTag(self):
        tag = '%0.4X' % self.tagID
        self.tagID += 1
        return tag

    def dispatchCommand(self, tag, rest):
        if self.state is None:
            f = self.response_UNAUTH
        else:
            f = getattr(self, 'response_' + self.state.upper(), None)
        if f:
            try:
                f(tag, rest)
            except:
                log.err()
                self.transport.loseConnection()
        else:
            log.err("Cannot dispatch: %s, %s, %s" % (self.state, tag, rest))
            self.transport.loseConnection()

    def response_UNAUTH(self, tag, rest):
        if self.state is None:
            # Server greeting, this is
            status, rest = rest.split(None, 1)
            if status.upper() == 'OK':
                self.state = 'unauth'
            elif status.upper() == 'PREAUTH':
                self.state = 'auth'
            else:
                # XXX - This is rude.
                self.transport.loseConnection()
                raise IllegalServerResponse(tag + ' ' + rest)

            b, e = rest.find('['), rest.find(']')
            if b != -1 and e != -1:
                self.serverGreeting(
                    self.__cbCapabilities(
                        ([parseNestedParens(rest[b + 1:e])], None)))
            else:
                self.serverGreeting(None)
        else:
            self._defaultHandler(tag, rest)

    def response_AUTH(self, tag, rest):
        self._defaultHandler(tag, rest)

    def _defaultHandler(self, tag, rest):
        if tag == '*' or tag == '+':
            if not self.waiting:
                self._extraInfo([parseNestedParens(rest)])
            else:
                cmd = self.tags[self.waiting]
                if tag == '+':
                    cmd.continuation(rest)
                else:
                    cmd.lines.append(rest)
        else:
            try:
                cmd = self.tags[tag]
            except KeyError:
                # XXX - This is rude.
                self.transport.loseConnection()
                raise IllegalServerResponse(tag + ' ' + rest)
            else:
                status, line = rest.split(None, 1)
                if status == 'OK':
                    # Give them this last line, too
                    cmd.finish(rest, self._extraInfo)
                else:
                    cmd.defer.errback(IMAP4Exception(line))
                del self.tags[tag]
                self.waiting = None
                self._flushQueue()

    def _flushQueue(self):
        if self.queued:
            cmd = self.queued.pop(0)
            t = self.makeTag()
            self.tags[t] = cmd
            self.sendLine(cmd.format(t))
            self.waiting = t

    def _extraInfo(self, lines):
        # XXX - This is terrible.
        # XXX - Also, this should collapse temporally proximate calls into single
        #       invocations of IMailboxListener methods, where possible.
        flags = {}
        recent = exists = None
        for response in lines:
            elements = len(response)
            if elements == 1 and response[0] == ['READ-ONLY']:
                self.modeChanged(False)
            elif elements == 1 and response[0] == ['READ-WRITE']:
                self.modeChanged(True)
            elif elements == 2 and response[1] == 'EXISTS':
                exists = int(response[0])
            elif elements == 2 and response[1] == 'RECENT':
                recent = int(response[0])
            elif elements == 3 and response[1] == 'FETCH':
                mId = int(response[0])
                values = self._parseFetchPairs(response[2])
                flags.setdefault(mId, []).extend(values.get('FLAGS', ()))
            else:
                log.msg('Unhandled unsolicited response: %s' % (response,))

        if flags:
            self.flagsChanged(flags)
        if recent is not None or exists is not None:
            self.newMessages(exists, recent)

    def sendCommand(self, cmd):
        cmd.defer = defer.Deferred()
        if self.waiting:
            self.queued.append(cmd)
            return cmd.defer
        t = self.makeTag()
        self.tags[t] = cmd
        self.sendLine(cmd.format(t))
        self.waiting = t
        self._lastCmd = cmd
        return cmd.defer

    def getCapabilities(self, useCache=1):
        """Request the capabilities available on this server.

        This command is allowed in any state of connection.

        @type useCache: C{bool}
        @param useCache: Specify whether to use the capability-cache or to
        re-retrieve the capabilities from the server.  Server capabilities
        should never change, so for normal use, this flag should never be
        false.

        @rtype: C{Deferred}
        @return: A deferred whose callback will be invoked with a
        dictionary mapping capability types to lists of supported
        mechanisms, or to None if a support list is not applicable.
        """
        if useCache and self._capCache is not None:
            return defer.succeed(self._capCache)
        cmd = 'CAPABILITY'
        resp = ('CAPABILITY',)
        d = self.sendCommand(Command(cmd, wantResponse=resp))
        d.addCallback(self.__cbCapabilities)
        return d

    def __cbCapabilities(self, (lines, tagline)):
        caps = {}
        for rest in lines:
            for cap in rest[1:]:
                parts = cap.split('=', 1)
                if len(parts) == 1:
                    category, value = parts[0], None
                else:
                    category, value = parts
                caps.setdefault(category, []).append(value)

        # Preserve a non-ideal API for backwards compatibility.  It would
        # probably be entirely sensible to have an object with a wider API than
        # dict here so this could be presented less insanely.
        for category in caps:
            if caps[category] == [None]:
                caps[category] = None
        self._capCache = caps
        return caps

    def logout(self):
        """Inform the server that we are done with the connection.

        This command is allowed in any state of connection.

        @rtype: C{Deferred}
        @return: A deferred whose callback will be invoked with None
        when the proper server acknowledgement has been received.
        """
        d = self.sendCommand(Command('LOGOUT', wantResponse=('BYE',)))
        d.addCallback(self.__cbLogout)
        return d

    def __cbLogout(self, (lines, tagline)):
        self.transport.loseConnection()
        # We don't particularly care what the server said
        return None


    def noop(self):
        """Perform no operation.

        This command is allowed in any state of connection.

        @rtype: C{Deferred}
        @return: A deferred whose callback will be invoked with a list
        of untagged status updates the server responds with.
        """
        d = self.sendCommand(Command('NOOP'))
        d.addCallback(self.__cbNoop)
        return d

    def __cbNoop(self, (lines, tagline)):
        # Conceivable, this is elidable.
        # It is, afterall, a no-op.
        return lines

    def startTLS(self, contextFactory=None):
        """
        Initiates a 'STARTTLS' request and negotiates the TLS / SSL
        Handshake.

        @param contextFactory: The TLS / SSL Context Factory to
        leverage.  If the contextFactory is None the IMAP4Client will
        either use the current TLS / SSL Context Factory or attempt to
        create a new one.

        @type contextFactory: C{ssl.ClientContextFactory}

        @return: A Deferred which fires when the transport has been
        secured according to the given contextFactory, or which fails
        if the transport cannot be secured.
        """
        assert not self.startedTLS, "Client and Server are currently communicating via TLS"

        if contextFactory is None:
            contextFactory = self._getContextFactory()

        if contextFactory is None:
            return defer.fail(IMAP4Exception(
                "IMAP4Client requires a TLS context to "
                "initiate the STARTTLS handshake"))

        if 'STARTTLS' not in self._capCache:
            return defer.fail(IMAP4Exception(
                "Server does not support secure communication "
                "via TLS / SSL"))

        tls = interfaces.ITLSTransport(self.transport, None)
        if tls is None:
            return defer.fail(IMAP4Exception(
                "IMAP4Client transport does not implement "
                "interfaces.ITLSTransport"))

        d = self.sendCommand(Command('STARTTLS'))
        d.addCallback(self._startedTLS, contextFactory)
        d.addCallback(lambda _: self.getCapabilities())
        return d


    def authenticate(self, secret):
        """Attempt to enter the authenticated state with the server

        This command is allowed in the Non-Authenticated state.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked if the authentication
        succeeds and whose errback will be invoked otherwise.
        """
        if self._capCache is None:
            d = self.getCapabilities()
        else:
            d = defer.succeed(self._capCache)
        d.addCallback(self.__cbAuthenticate, secret)
        return d

    def __cbAuthenticate(self, caps, secret):
        auths = caps.get('AUTH', ())
        for scheme in auths:
            if scheme.upper() in self.authenticators:
                cmd = Command('AUTHENTICATE', scheme, (),
                              self.__cbContinueAuth, scheme,
                              secret)
                return self.sendCommand(cmd)

        if self.startedTLS:
            return defer.fail(NoSupportedAuthentication(
                auths, self.authenticators.keys()))
        else:
            def ebStartTLS(err):
                err.trap(IMAP4Exception)
                # We couldn't negotiate TLS for some reason
                return defer.fail(NoSupportedAuthentication(
                    auths, self.authenticators.keys()))

            d = self.startTLS()
            d.addErrback(ebStartTLS)
            d.addCallback(lambda _: self.getCapabilities())
            d.addCallback(self.__cbAuthTLS, secret)
            return d


    def __cbContinueAuth(self, rest, scheme, secret):
        try:
            chal = base64.decodestring(rest + '\n')
        except binascii.Error:
            self.sendLine('*')
            raise IllegalServerResponse(rest)
            self.transport.loseConnection()
        else:
            auth = self.authenticators[scheme]
            chal = auth.challengeResponse(secret, chal)
            self.sendLine(base64.encodestring(chal).strip())

    def __cbAuthTLS(self, caps, secret):
        auths = caps.get('AUTH', ())
        for scheme in auths:
            if scheme.upper() in self.authenticators:
                cmd = Command('AUTHENTICATE', scheme, (),
                              self.__cbContinueAuth, scheme,
                              secret)
                return self.sendCommand(cmd)
        raise NoSupportedAuthentication(auths, self.authenticators.keys())


    def login(self, username, password):
        """Authenticate with the server using a username and password

        This command is allowed in the Non-Authenticated state.  If the
        server supports the STARTTLS capability and our transport supports
        TLS, TLS is negotiated before the login command is issued.

        A more secure way to log in is to use C{startTLS} or
        C{authenticate} or both.

        @type username: C{str}
        @param username: The username to log in with

        @type password: C{str}
        @param password: The password to log in with

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked if login is successful
        and whose errback is invoked otherwise.
        """
        d = maybeDeferred(self.getCapabilities)
        d.addCallback(self.__cbLoginCaps, username, password)
        return d

    def serverGreeting(self, caps):
        """Called when the server has sent us a greeting.

        @type caps: C{dict}
        @param caps: Capabilities the server advertised in its greeting.
        """

    def _getContextFactory(self):
        if self.context is not None:
            return self.context
        try:
            from twisted.internet import ssl
        except ImportError:
            return None
        else:
            context = ssl.ClientContextFactory()
            context.method = ssl.SSL.TLSv1_METHOD
            return context

    def __cbLoginCaps(self, capabilities, username, password):
        # If the server advertises STARTTLS, we might want to try to switch to TLS
        tryTLS = 'STARTTLS' in capabilities

        # If our transport supports switching to TLS, we might want to try to switch to TLS.
        tlsableTransport = interfaces.ITLSTransport(self.transport, None) is not None

        # If our transport is not already using TLS, we might want to try to switch to TLS.
        nontlsTransport = interfaces.ISSLTransport(self.transport, None) is None

        if not self.startedTLS and tryTLS and tlsableTransport and nontlsTransport:
            d = self.startTLS()

            d.addCallbacks(
                self.__cbLoginTLS,
                self.__ebLoginTLS,
                callbackArgs=(username, password),
                )
            return d
        else:
            if nontlsTransport:
                log.msg("Server has no TLS support. logging in over cleartext!")
            args = ' '.join((_quote(username), _quote(password)))
            return self.sendCommand(Command('LOGIN', args))

    def _startedTLS(self, result, context):
        self.transport.startTLS(context)
        self._capCache = None
        self.startedTLS = True
        return result

    def __cbLoginTLS(self, result, username, password):
        args = ' '.join((_quote(username), _quote(password)))
        return self.sendCommand(Command('LOGIN', args))

    def __ebLoginTLS(self, failure):
        log.err(failure)
        return failure

    def namespace(self):
        """Retrieve information about the namespaces available to this account

        This command is allowed in the Authenticated and Selected states.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with namespace
        information.  An example of this information is::

            [[['', '/']], [], []]

        which indicates a single personal namespace called '' with '/'
        as its hierarchical delimiter, and no shared or user namespaces.
        """
        cmd = 'NAMESPACE'
        resp = ('NAMESPACE',)
        d = self.sendCommand(Command(cmd, wantResponse=resp))
        d.addCallback(self.__cbNamespace)
        return d

    def __cbNamespace(self, (lines, last)):
        for parts in lines:
            if len(parts) == 4 and parts[0] == 'NAMESPACE':
                return [e or [] for e in parts[1:]]
        log.err("No NAMESPACE response to NAMESPACE command")
        return [[], [], []]

    def select(self, mailbox):
        """Select a mailbox

        This command is allowed in the Authenticated and Selected states.

        @type mailbox: C{str}
        @param mailbox: The name of the mailbox to select

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with mailbox
        information if the select is successful and whose errback is
        invoked otherwise.  Mailbox information consists of a dictionary
        with the following keys and values::

                FLAGS: A list of strings containing the flags settable on
                        messages in this mailbox.

                EXISTS: An integer indicating the number of messages in this
                        mailbox.

                RECENT: An integer indicating the number of \"recent\"
                        messages in this mailbox.

                UNSEEN: An integer indicating the number of messages not
                        flagged \\Seen in this mailbox.

                PERMANENTFLAGS: A list of strings containing the flags that
                        can be permanently set on messages in this mailbox.

                UIDVALIDITY: An integer uniquely identifying this mailbox.
        """
        cmd = 'SELECT'
        args = _prepareMailboxName(mailbox)
        resp = ('FLAGS', 'EXISTS', 'RECENT', 'UNSEEN', 'PERMANENTFLAGS', 'UIDVALIDITY')
        d = self.sendCommand(Command(cmd, args, wantResponse=resp))
        d.addCallback(self.__cbSelect, 1)
        return d

    def examine(self, mailbox):
        """Select a mailbox in read-only mode

        This command is allowed in the Authenticated and Selected states.

        @type mailbox: C{str}
        @param mailbox: The name of the mailbox to examine

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with mailbox
        information if the examine is successful and whose errback
        is invoked otherwise.  Mailbox information consists of a dictionary
        with the following keys and values::

            'FLAGS': A list of strings containing the flags settable on
                        messages in this mailbox.

            'EXISTS': An integer indicating the number of messages in this
                        mailbox.

            'RECENT': An integer indicating the number of \"recent\"
                        messages in this mailbox.

            'UNSEEN': An integer indicating the number of messages not
                        flagged \\Seen in this mailbox.

            'PERMANENTFLAGS': A list of strings containing the flags that
                        can be permanently set on messages in this mailbox.

            'UIDVALIDITY': An integer uniquely identifying this mailbox.
        """
        cmd = 'EXAMINE'
        args = _prepareMailboxName(mailbox)
        resp = ('FLAGS', 'EXISTS', 'RECENT', 'UNSEEN', 'PERMANENTFLAGS', 'UIDVALIDITY')
        d = self.sendCommand(Command(cmd, args, wantResponse=resp))
        d.addCallback(self.__cbSelect, 0)
        return d


    def _intOrRaise(self, value, phrase):
        """
        Parse C{value} as an integer and return the result or raise
        L{IllegalServerResponse} with C{phrase} as an argument if C{value}
        cannot be parsed as an integer.
        """
        try:
            return int(value)
        except ValueError:
            raise IllegalServerResponse(phrase)


    def __cbSelect(self, (lines, tagline), rw):
        """
        Handle lines received in response to a SELECT or EXAMINE command.

        See RFC 3501, section 6.3.1.
        """
        # In the absense of specification, we are free to assume:
        #   READ-WRITE access
        datum = {'READ-WRITE': rw}
        lines.append(parseNestedParens(tagline))
        for split in lines:
            if len(split) > 0 and split[0].upper() == 'OK':
                # Handle all the kinds of OK response.
                content = split[1]
                key = content[0].upper()
                if key == 'READ-ONLY':
                    datum['READ-WRITE'] = False
                elif key == 'READ-WRITE':
                    datum['READ-WRITE'] = True
                elif key == 'UIDVALIDITY':
                    datum['UIDVALIDITY'] = self._intOrRaise(
                        content[1], split)
                elif key == 'UNSEEN':
                    datum['UNSEEN'] = self._intOrRaise(content[1], split)
                elif key == 'UIDNEXT':
                    datum['UIDNEXT'] = self._intOrRaise(content[1], split)
                elif key == 'PERMANENTFLAGS':
                    datum['PERMANENTFLAGS'] = tuple(content[1])
                else:
                    log.err('Unhandled SELECT response (2): %s' % (split,))
            elif len(split) == 2:
                # Handle FLAGS, EXISTS, and RECENT
                if split[0].upper() == 'FLAGS':
                    datum['FLAGS'] = tuple(split[1])
                elif isinstance(split[1], str):
                    # Must make sure things are strings before treating them as
                    # strings since some other forms of response have nesting in
                    # places which results in lists instead.
                    if split[1].upper() == 'EXISTS':
                        datum['EXISTS'] = self._intOrRaise(split[0], split)
                    elif split[1].upper() == 'RECENT':
                        datum['RECENT'] = self._intOrRaise(split[0], split)
                    else:
                        log.err('Unhandled SELECT response (0): %s' % (split,))
                else:
                    log.err('Unhandled SELECT response (1): %s' % (split,))
            else:
                log.err('Unhandled SELECT response (4): %s' % (split,))
        return datum


    def create(self, name):
        """Create a new mailbox on the server

        This command is allowed in the Authenticated and Selected states.

        @type name: C{str}
        @param name: The name of the mailbox to create.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked if the mailbox creation
        is successful and whose errback is invoked otherwise.
        """
        return self.sendCommand(Command('CREATE', _prepareMailboxName(name)))

    def delete(self, name):
        """Delete a mailbox

        This command is allowed in the Authenticated and Selected states.

        @type name: C{str}
        @param name: The name of the mailbox to delete.

        @rtype: C{Deferred}
        @return: A deferred whose calblack is invoked if the mailbox is
        deleted successfully and whose errback is invoked otherwise.
        """
        return self.sendCommand(Command('DELETE', _prepareMailboxName(name)))

    def rename(self, oldname, newname):
        """Rename a mailbox

        This command is allowed in the Authenticated and Selected states.

        @type oldname: C{str}
        @param oldname: The current name of the mailbox to rename.

        @type newname: C{str}
        @param newname: The new name to give the mailbox.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked if the rename is
        successful and whose errback is invoked otherwise.
        """
        oldname = _prepareMailboxName(oldname)
        newname = _prepareMailboxName(newname)
        return self.sendCommand(Command('RENAME', ' '.join((oldname, newname))))

    def subscribe(self, name):
        """Add a mailbox to the subscription list

        This command is allowed in the Authenticated and Selected states.

        @type name: C{str}
        @param name: The mailbox to mark as 'active' or 'subscribed'

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked if the subscription
        is successful and whose errback is invoked otherwise.
        """
        return self.sendCommand(Command('SUBSCRIBE', _prepareMailboxName(name)))

    def unsubscribe(self, name):
        """Remove a mailbox from the subscription list

        This command is allowed in the Authenticated and Selected states.

        @type name: C{str}
        @param name: The mailbox to unsubscribe

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked if the unsubscription
        is successful and whose errback is invoked otherwise.
        """
        return self.sendCommand(Command('UNSUBSCRIBE', _prepareMailboxName(name)))

    def list(self, reference, wildcard):
        """List a subset of the available mailboxes

        This command is allowed in the Authenticated and Selected states.

        @type reference: C{str}
        @param reference: The context in which to interpret C{wildcard}

        @type wildcard: C{str}
        @param wildcard: The pattern of mailbox names to match, optionally
        including either or both of the '*' and '%' wildcards.  '*' will
        match zero or more characters and cross hierarchical boundaries.
        '%' will also match zero or more characters, but is limited to a
        single hierarchical level.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with a list of C{tuple}s,
        the first element of which is a C{tuple} of mailbox flags, the second
        element of which is the hierarchy delimiter for this mailbox, and the
        third of which is the mailbox name; if the command is unsuccessful,
        the deferred's errback is invoked instead.
        """
        cmd = 'LIST'
        args = '"%s" "%s"' % (reference, wildcard.encode('imap4-utf-7'))
        resp = ('LIST',)
        d = self.sendCommand(Command(cmd, args, wantResponse=resp))
        d.addCallback(self.__cbList, 'LIST')
        return d

    def lsub(self, reference, wildcard):
        """List a subset of the subscribed available mailboxes

        This command is allowed in the Authenticated and Selected states.

        The parameters and returned object are the same as for the C{list}
        method, with one slight difference: Only mailboxes which have been
        subscribed can be included in the resulting list.
        """
        cmd = 'LSUB'
        args = '"%s" "%s"' % (reference, wildcard.encode('imap4-utf-7'))
        resp = ('LSUB',)
        d = self.sendCommand(Command(cmd, args, wantResponse=resp))
        d.addCallback(self.__cbList, 'LSUB')
        return d

    def __cbList(self, (lines, last), command):
        results = []
        for parts in lines:
            if len(parts) == 4 and parts[0] == command:
                parts[1] = tuple(parts[1])
                results.append(tuple(parts[1:]))
        return results

    def status(self, mailbox, *names):
        """
        Retrieve the status of the given mailbox

        This command is allowed in the Authenticated and Selected states.

        @type mailbox: C{str}
        @param mailbox: The name of the mailbox to query

        @type *names: C{str}
        @param *names: The status names to query.  These may be any number of:
            C{'MESSAGES'}, C{'RECENT'}, C{'UIDNEXT'}, C{'UIDVALIDITY'}, and
            C{'UNSEEN'}.

        @rtype: C{Deferred}
        @return: A deferred which fires with with the status information if the
            command is successful and whose errback is invoked otherwise.  The
            status information is in the form of a C{dict}.  Each element of
            C{names} is a key in the dictionary.  The value for each key is the
            corresponding response from the server.
        """
        cmd = 'STATUS'
        args = "%s (%s)" % (_prepareMailboxName(mailbox), ' '.join(names))
        resp = ('STATUS',)
        d = self.sendCommand(Command(cmd, args, wantResponse=resp))
        d.addCallback(self.__cbStatus)
        return d

    def __cbStatus(self, (lines, last)):
        status = {}
        for parts in lines:
            if parts[0] == 'STATUS':
                items = parts[2]
                items = [items[i:i+2] for i in range(0, len(items), 2)]
                status.update(dict(items))
        for k in status.keys():
            t = self.STATUS_TRANSFORMATIONS.get(k)
            if t:
                try:
                    status[k] = t(status[k])
                except Exception, e:
                    raise IllegalServerResponse('(%s %s): %s' % (k, status[k], str(e)))
        return status

    def append(self, mailbox, message, flags = (), date = None):
        """Add the given message to the given mailbox.

        This command is allowed in the Authenticated and Selected states.

        @type mailbox: C{str}
        @param mailbox: The mailbox to which to add this message.

        @type message: Any file-like object
        @param message: The message to add, in RFC822 format.  Newlines
        in this file should be \\r\\n-style.

        @type flags: Any iterable of C{str}
        @param flags: The flags to associated with this message.

        @type date: C{str}
        @param date: The date to associate with this message.  This should
        be of the format DD-MM-YYYY HH:MM:SS +/-HHMM.  For example, in
        Eastern Standard Time, on July 1st 2004 at half past 1 PM,
        \"01-07-2004 13:30:00 -0500\".

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked when this command
        succeeds or whose errback is invoked if it fails.
        """
        message.seek(0, 2)
        L = message.tell()
        message.seek(0, 0)
        fmt = '%s (%s)%s {%d}'
        if date:
            date = ' "%s"' % date
        else:
            date = ''
        cmd = fmt % (
            _prepareMailboxName(mailbox), ' '.join(flags),
            date, L
        )
        d = self.sendCommand(Command('APPEND', cmd, (), self.__cbContinueAppend, message))
        return d

    def __cbContinueAppend(self, lines, message):
        s = basic.FileSender()
        return s.beginFileTransfer(message, self.transport, None
            ).addCallback(self.__cbFinishAppend)

    def __cbFinishAppend(self, foo):
        self.sendLine('')

    def check(self):
        """Tell the server to perform a checkpoint

        This command is allowed in the Selected state.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked when this command
        succeeds or whose errback is invoked if it fails.
        """
        return self.sendCommand(Command('CHECK'))

    def close(self):
        """Return the connection to the Authenticated state.

        This command is allowed in the Selected state.

        Issuing this command will also remove all messages flagged \\Deleted
        from the selected mailbox if it is opened in read-write mode,
        otherwise it indicates success by no messages are removed.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked when the command
        completes successfully or whose errback is invoked if it fails.
        """
        return self.sendCommand(Command('CLOSE'))


    def expunge(self):
        """Return the connection to the Authenticate state.

        This command is allowed in the Selected state.

        Issuing this command will perform the same actions as issuing the
        close command, but will also generate an 'expunge' response for
        every message deleted.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with a list of the
        'expunge' responses when this command is successful or whose errback
        is invoked otherwise.
        """
        cmd = 'EXPUNGE'
        resp = ('EXPUNGE',)
        d = self.sendCommand(Command(cmd, wantResponse=resp))
        d.addCallback(self.__cbExpunge)
        return d


    def __cbExpunge(self, (lines, last)):
        ids = []
        for parts in lines:
            if len(parts) == 2 and parts[1] == 'EXPUNGE':
                ids.append(self._intOrRaise(parts[0], parts))
        return ids


    def search(self, *queries, **kwarg):
        """Search messages in the currently selected mailbox

        This command is allowed in the Selected state.

        Any non-zero number of queries are accepted by this method, as
        returned by the C{Query}, C{Or}, and C{Not} functions.

        One keyword argument is accepted: if uid is passed in with a non-zero
        value, the server is asked to return message UIDs instead of message
        sequence numbers.

        @rtype: C{Deferred}
        @return: A deferred whose callback will be invoked with a list of all
        the message sequence numbers return by the search, or whose errback
        will be invoked if there is an error.
        """
        if kwarg.get('uid'):
            cmd = 'UID SEARCH'
        else:
            cmd = 'SEARCH'
        args = ' '.join(queries)
        d = self.sendCommand(Command(cmd, args, wantResponse=(cmd,)))
        d.addCallback(self.__cbSearch)
        return d


    def __cbSearch(self, (lines, end)):
        ids = []
        for parts in lines:
            if len(parts) > 0 and parts[0] == 'SEARCH':
                ids.extend([self._intOrRaise(p, parts) for p in parts[1:]])
        return ids


    def fetchUID(self, messages, uid=0):
        """Retrieve the unique identifier for one or more messages

        This command is allowed in the Selected state.

        @type messages: C{MessageSet} or C{str}
        @param messages: A message sequence set

        @type uid: C{bool}
        @param uid: Indicates whether the message sequence set is of message
        numbers or of unique message IDs.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with a dict mapping
        message sequence numbers to unique message identifiers, or whose
        errback is invoked if there is an error.
        """
        return self._fetch(messages, useUID=uid, uid=1)


    def fetchFlags(self, messages, uid=0):
        """Retrieve the flags for one or more messages

        This command is allowed in the Selected state.

        @type messages: C{MessageSet} or C{str}
        @param messages: The messages for which to retrieve flags.

        @type uid: C{bool}
        @param uid: Indicates whether the message sequence set is of message
        numbers or of unique message IDs.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with a dict mapping
        message numbers to lists of flags, or whose errback is invoked if
        there is an error.
        """
        return self._fetch(str(messages), useUID=uid, flags=1)


    def fetchInternalDate(self, messages, uid=0):
        """Retrieve the internal date associated with one or more messages

        This command is allowed in the Selected state.

        @type messages: C{MessageSet} or C{str}
        @param messages: The messages for which to retrieve the internal date.

        @type uid: C{bool}
        @param uid: Indicates whether the message sequence set is of message
        numbers or of unique message IDs.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with a dict mapping
        message numbers to date strings, or whose errback is invoked
        if there is an error.  Date strings take the format of
        \"day-month-year time timezone\".
        """
        return self._fetch(str(messages), useUID=uid, internaldate=1)


    def fetchEnvelope(self, messages, uid=0):
        """Retrieve the envelope data for one or more messages

        This command is allowed in the Selected state.

        @type messages: C{MessageSet} or C{str}
        @param messages: The messages for which to retrieve envelope data.

        @type uid: C{bool}
        @param uid: Indicates whether the message sequence set is of message
        numbers or of unique message IDs.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with a dict mapping
        message numbers to envelope data, or whose errback is invoked
        if there is an error.  Envelope data consists of a sequence of the
        date, subject, from, sender, reply-to, to, cc, bcc, in-reply-to,
        and message-id header fields.  The date, subject, in-reply-to, and
        message-id fields are strings, while the from, sender, reply-to,
        to, cc, and bcc fields contain address data.  Address data consists
        of a sequence of name, source route, mailbox name, and hostname.
        Fields which are not present for a particular address may be C{None}.
        """
        return self._fetch(str(messages), useUID=uid, envelope=1)


    def fetchBodyStructure(self, messages, uid=0):
        """Retrieve the structure of the body of one or more messages

        This command is allowed in the Selected state.

        @type messages: C{MessageSet} or C{str}
        @param messages: The messages for which to retrieve body structure
        data.

        @type uid: C{bool}
        @param uid: Indicates whether the message sequence set is of message
        numbers or of unique message IDs.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with a dict mapping
        message numbers to body structure data, or whose errback is invoked
        if there is an error.  Body structure data describes the MIME-IMB
        format of a message and consists of a sequence of mime type, mime
        subtype, parameters, content id, description, encoding, and size.
        The fields following the size field are variable: if the mime
        type/subtype is message/rfc822, the contained message's envelope
        information, body structure data, and number of lines of text; if
        the mime type is text, the number of lines of text.  Extension fields
        may also be included; if present, they are: the MD5 hash of the body,
        body disposition, body language.
        """
        return self._fetch(messages, useUID=uid, bodystructure=1)


    def fetchSimplifiedBody(self, messages, uid=0):
        """Retrieve the simplified body structure of one or more messages

        This command is allowed in the Selected state.

        @type messages: C{MessageSet} or C{str}
        @param messages: A message sequence set

        @type uid: C{bool}
        @param uid: Indicates whether the message sequence set is of message
        numbers or of unique message IDs.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with a dict mapping
        message numbers to body data, or whose errback is invoked
        if there is an error.  The simplified body structure is the same
        as the body structure, except that extension fields will never be
        present.
        """
        return self._fetch(messages, useUID=uid, body=1)


    def fetchMessage(self, messages, uid=0):
        """Retrieve one or more entire messages

        This command is allowed in the Selected state.

        @type messages: L{MessageSet} or C{str}
        @param messages: A message sequence set

        @type uid: C{bool}
        @param uid: Indicates whether the message sequence set is of message
        numbers or of unique message IDs.

        @rtype: L{Deferred}

        @return: A L{Deferred} which will fire with a C{dict} mapping message
            sequence numbers to C{dict}s giving message data for the
            corresponding message.  If C{uid} is true, the inner dictionaries
            have a C{'UID'} key mapped to a C{str} giving the UID for the
            message.  The text of the message is a C{str} associated with the
            C{'RFC822'} key in each dictionary.
        """
        return self._fetch(messages, useUID=uid, rfc822=1)


    def fetchHeaders(self, messages, uid=0):
        """Retrieve headers of one or more messages

        This command is allowed in the Selected state.

        @type messages: C{MessageSet} or C{str}
        @param messages: A message sequence set

        @type uid: C{bool}
        @param uid: Indicates whether the message sequence set is of message
        numbers or of unique message IDs.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with a dict mapping
        message numbers to dicts of message headers, or whose errback is
        invoked if there is an error.
        """
        return self._fetch(messages, useUID=uid, rfc822header=1)


    def fetchBody(self, messages, uid=0):
        """Retrieve body text of one or more messages

        This command is allowed in the Selected state.

        @type messages: C{MessageSet} or C{str}
        @param messages: A message sequence set

        @type uid: C{bool}
        @param uid: Indicates whether the message sequence set is of message
        numbers or of unique message IDs.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with a dict mapping
        message numbers to file-like objects containing body text, or whose
        errback is invoked if there is an error.
        """
        return self._fetch(messages, useUID=uid, rfc822text=1)


    def fetchSize(self, messages, uid=0):
        """Retrieve the size, in octets, of one or more messages

        This command is allowed in the Selected state.

        @type messages: C{MessageSet} or C{str}
        @param messages: A message sequence set

        @type uid: C{bool}
        @param uid: Indicates whether the message sequence set is of message
        numbers or of unique message IDs.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with a dict mapping
        message numbers to sizes, or whose errback is invoked if there is
        an error.
        """
        return self._fetch(messages, useUID=uid, rfc822size=1)


    def fetchFull(self, messages, uid=0):
        """Retrieve several different fields of one or more messages

        This command is allowed in the Selected state.  This is equivalent
        to issuing all of the C{fetchFlags}, C{fetchInternalDate},
        C{fetchSize}, C{fetchEnvelope}, and C{fetchSimplifiedBody}
        functions.

        @type messages: C{MessageSet} or C{str}
        @param messages: A message sequence set

        @type uid: C{bool}
        @param uid: Indicates whether the message sequence set is of message
        numbers or of unique message IDs.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with a dict mapping
        message numbers to dict of the retrieved data values, or whose
        errback is invoked if there is an error.  They dictionary keys
        are "flags", "date", "size", "envelope", and "body".
        """
        return self._fetch(
            messages, useUID=uid, flags=1, internaldate=1,
            rfc822size=1, envelope=1, body=1)


    def fetchAll(self, messages, uid=0):
        """Retrieve several different fields of one or more messages

        This command is allowed in the Selected state.  This is equivalent
        to issuing all of the C{fetchFlags}, C{fetchInternalDate},
        C{fetchSize}, and C{fetchEnvelope} functions.

        @type messages: C{MessageSet} or C{str}
        @param messages: A message sequence set

        @type uid: C{bool}
        @param uid: Indicates whether the message sequence set is of message
        numbers or of unique message IDs.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with a dict mapping
        message numbers to dict of the retrieved data values, or whose
        errback is invoked if there is an error.  They dictionary keys
        are "flags", "date", "size", and "envelope".
        """
        return self._fetch(
            messages, useUID=uid, flags=1, internaldate=1,
            rfc822size=1, envelope=1)


    def fetchFast(self, messages, uid=0):
        """Retrieve several different fields of one or more messages

        This command is allowed in the Selected state.  This is equivalent
        to issuing all of the C{fetchFlags}, C{fetchInternalDate}, and
        C{fetchSize} functions.

        @type messages: C{MessageSet} or C{str}
        @param messages: A message sequence set

        @type uid: C{bool}
        @param uid: Indicates whether the message sequence set is of message
        numbers or of unique message IDs.

        @rtype: C{Deferred}
        @return: A deferred whose callback is invoked with a dict mapping
        message numbers to dict of the retrieved data values, or whose
        errback is invoked if there is an error.  They dictionary keys are
        "flags", "date", and "size".
        """
        return self._fetch(
            messages, useUID=uid, flags=1, internaldate=1, rfc822size=1)


    def _parseFetchPairs(self, fetchResponseList):
        """
        Given the result of parsing a single I{FETCH} response, construct a
        C{dict} mapping response keys to response values.

        @param fetchResponseList: The result of parsing a I{FETCH} response
            with L{parseNestedParens} and extracting just the response data
            (that is, just the part that comes after C{"FETCH"}).  The form
            of this input (and therefore the output of this method) is very
            disagreable.  A valuable improvement would be to enumerate the
            possible keys (representing them as structured objects of some
            sort) rather than using strings and tuples of tuples of strings
            and so forth.  This would allow the keys to be documented more
            easily and would allow for a much simpler application-facing API
            (one not based on looking up somewhat hard to predict keys in a
            dict).  Since C{fetchResponseList} notionally represents a
            flattened sequence of pairs (identifying keys followed by their
            associated values), collapsing such complex elements of this
            list as C{["BODY", ["HEADER.FIELDS", ["SUBJECT"]]]} into a
            single object would also greatly simplify the implementation of
            this method.