--- /dev/null
+# A new Feed-style Parser
+
+from email import Errors, Message
+import re
+
+NLCRE = re.compile('\r\n|\r|\n')
+
+EMPTYSTRING = ''
+NL = '\n'
+
+NeedMoreData = object()
+
+class FeedableLumpOfText:
+ "A file-like object that can have new data loaded into it"
+
+ def __init__(self):
+ self._partial = ''
+ self._done = False
+ # _pending is a list of lines, in reverse order
+ self._pending = []
+
+ def readline(self):
+ """ Return a line of data.
+
+ If data has been pushed back with unreadline(), the most recently
+ returned unreadline()d data will be returned.
+ """
+ if not self._pending:
+ if self._done:
+ return ''
+ return NeedMoreData
+ return self._pending.pop()
+
+ def unreadline(self, line):
+ """ Push a line back into the object.
+ """
+ self._pending.append(line)
+
+ def peekline(self):
+ """ Non-destructively look at the next line """
+ if not self._pending:
+ if self._done:
+ return ''
+ return NeedMoreData
+ return self._pending[-1]
+
+
+ # for r in self._input.readuntil(regexp):
+ # if r is NeedMoreData:
+ # yield NeedMoreData
+ # preamble, matchobj = r
+ def readuntil(self, matchre, afterblank=False, includematch=False):
+ """ Read a line at a time until we get the specified RE.
+
+ Returns the text up to (and including, if includematch is true) the
+ matched text, and the RE match object. If afterblank is true,
+ there must be a blank line before the matched text. Moves current
+ filepointer to the line following the matched line. If we reach
+ end-of-file, return what we've got so far, and return None as the
+ RE match object.
+ """
+ prematch = []
+ blankseen = 0
+ while 1:
+ if not self._pending:
+ if self._done:
+ # end of file
+ yield EMPTYSTRING.join(prematch), None
+ else:
+ yield NeedMoreData
+ continue
+ line = self._pending.pop()
+ if afterblank:
+ if NLCRE.match(line):
+ blankseen = 1
+ continue
+ else:
+ blankseen = 0
+ m = matchre.match(line)
+ if (m and not afterblank) or (m and afterblank and blankseen):
+ if includematch:
+ prematch.append(line)
+ yield EMPTYSTRING.join(prematch), m
+ prematch.append(line)
+
+
+ NLatend = re.compile('(\r\n|\r|\n)$').match
+ NLCRE_crack = re.compile('(\r\n|\r|\n)')
+
+ def push(self, data):
+ """ Push some new data into this object """
+ # Handle any previous leftovers
+ data, self._partial = self._partial+data, ''
+ # Crack into lines, but leave the newlines on the end of each
+ lines = self.NLCRE_crack.split(data)
+ # The *ahem* interesting behaviour of re.split when supplied
+ # groups means that the last element is the data after the
+ # final RE. In the case of a NL/CR terminated string, this is
+ # the empty string.
+ self._partial = lines.pop()
+ o = []
+ for i in range(len(lines) / 2):
+ o.append(EMPTYSTRING.join([lines[i*2], lines[i*2+1]]))
+ self.pushlines(o)
+
+ def pushlines(self, lines):
+ """ Push a list of new lines into the object """
+ # Reverse and insert at the front of _pending
+ self._pending[:0] = lines[::-1]
+
+ def end(self):
+ """ There is no more data """
+ self._done = True
+
+ def is_done(self):
+ return self._done
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ l = self.readline()
+ if l == '':
+ raise StopIteration
+ return l
+
+class FeedParser:
+ "A feed-style parser of email. copy docstring here"
+
+ def __init__(self, _class=Message.Message):
+ "fnord fnord fnord"
+ self._class = _class
+ self._input = FeedableLumpOfText()
+ self._root = None
+ self._objectstack = []
+ self._parse = self._parsegen().next
+
+ def end(self):
+ self._input.end()
+ self._call_parse()
+ return self._root
+
+ def feed(self, data):
+ self._input.push(data)
+ self._call_parse()
+
+ def _call_parse(self):
+ try:
+ self._parse()
+ except StopIteration:
+ pass
+
+ headerRE = re.compile(r'^(From |[-\w]{2,}:|[\t ])')
+
+ def _parse_headers(self,headerlist):
+ # Passed a list of strings that are the headers for the
+ # current object
+ lastheader = ''
+ lastvalue = []
+
+
+ for lineno, line in enumerate(headerlist):
+ # Check for continuation
+ if line[0] in ' \t':
+ if not lastheader:
+ raise Errors.HeaderParseError('First line must not be a continuation')
+ lastvalue.append(line)
+ continue
+
+ if lastheader:
+ # XXX reconsider the joining of folded lines
+ self._cur[lastheader] = EMPTYSTRING.join(lastvalue).rstrip()
+ lastheader, lastvalue = '', []
+
+ # Check for Unix-From
+ if line.startswith('From '):
+ if lineno == 0:
+ self._cur.set_unixfrom(line)
+ continue
+ elif lineno == len(headerlist) - 1:
+ # Something looking like a unix-from at the end - it's
+ # probably the first line of the body
+ self._input.unreadline(line)
+ return
+ else:
+ # Weirdly placed unix-from line. Ignore it.
+ continue
+
+ i = line.find(':')
+ if i < 0:
+ # The older parser had various special-cases here. We've
+ # already handled them
+ raise Errors.HeaderParseError(
+ "Not a header, not a continuation: ``%s''" % line)
+ lastheader = line[:i]
+ lastvalue = [line[i+1:].lstrip()]
+
+ if lastheader:
+ # XXX reconsider the joining of folded lines
+ self._cur[lastheader] = EMPTYSTRING.join(lastvalue).rstrip()
+
+
+ def _parsegen(self):
+ # Parse any currently available text
+ self._new_sub_object()
+ self._root = self._cur
+ completing = False
+ last = None
+
+ for line in self._input:
+ if line is NeedMoreData:
+ yield None # Need More Data
+ continue
+ self._input.unreadline(line)
+ if not completing:
+ headers = []
+ # Now collect all headers.
+ for line in self._input:
+ if line is NeedMoreData:
+ yield None # Need More Data
+ continue
+ if not self.headerRE.match(line):
+ self._parse_headers(headers)
+ # A message/rfc822 has no body and no internal
+ # boundary.
+ if self._cur.get_content_maintype() == "message":
+ self._new_sub_object()
+ completing = False
+ headers = []
+ continue
+ if line.strip():
+ # No blank line between headers and body.
+ # Push this line back, it's the first line of
+ # the body.
+ self._input.unreadline(line)
+ break
+ else:
+ headers.append(line)
+ else:
+ # We're done with the data and are still inside the headers
+ self._parse_headers(headers)
+
+ # Now we're dealing with the body
+ boundary = self._cur.get_boundary()
+ isdigest = (self._cur.get_content_type() == 'multipart/digest')
+ if boundary and not self._cur._finishing:
+ separator = '--' + boundary
+ self._cur._boundaryRE = re.compile(
+ r'(?P<sep>' + re.escape(separator) +
+ r')(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)$')
+ for r in self._input.readuntil(self._cur._boundaryRE):
+ if r is NeedMoreData:
+ yield NeedMoreData
+ else:
+ preamble, matchobj = r
+ break
+ if not matchobj:
+ # Broken - we hit the end of file. Just set the body
+ # to the text.
+ if completing:
+ self._attach_trailer(last, preamble)
+ else:
+ self._attach_preamble(self._cur, preamble)
+ # XXX move back to the parent container.
+ self._pop_container()
+ completing = True
+ continue
+ if preamble:
+ if completing:
+ preamble = preamble[:-len(matchobj.group('linesep'))]
+ self._attach_trailer(last, preamble)
+ else:
+ self._attach_preamble(self._cur, preamble)
+ elif not completing:
+ # The module docs specify an empty preamble is None, not ''
+ self._cur.preamble = None
+ # If we _are_ completing, the last object gets no payload
+
+ if matchobj.group('end'):
+ # That was the end boundary tag. Bounce back to the
+ # parent container
+ last = self._pop_container()
+ self._input.unreadline(matchobj.group('linesep'))
+ completing = True
+ continue
+
+ # A number of MTAs produced by a nameless large company
+ # we shall call "SicroMoft" produce repeated boundary
+ # lines.
+ while True:
+ line = self._input.peekline()
+ if line is NeedMoreData:
+ yield None
+ continue
+ if self._cur._boundaryRE.match(line):
+ self._input.readline()
+ else:
+ break
+
+ self._new_sub_object()
+
+ completing = False
+ if isdigest:
+ self._cur.set_default_type('message/rfc822')
+ continue
+ else:
+ # non-multipart or after end-boundary
+ if last is not self._root:
+ last = self._pop_container()
+ if self._cur.get_content_maintype() == "message":
+ # We double-pop to leave the RFC822 object
+ self._pop_container()
+ completing = True
+ elif self._cur._boundaryRE and last <> self._root:
+ completing = True
+ else:
+ # Non-multipart top level, or in the trailer of the
+ # top level multipart
+ while not self._input.is_done():
+ yield None
+ data = list(self._input)
+ body = EMPTYSTRING.join(data)
+ self._attach_trailer(last, body)
+
+
+ def _attach_trailer(self, obj, trailer):
+ #import pdb ; pdb.set_trace()
+ if obj.get_content_maintype() in ( "multipart", "message" ):
+ obj.epilogue = trailer
+ else:
+ obj.set_payload(trailer)
+
+ def _attach_preamble(self, obj, trailer):
+ if obj.get_content_maintype() in ( "multipart", "message" ):
+ obj.preamble = trailer
+ else:
+ obj.set_payload(trailer)
+
+
+ def _new_sub_object(self):
+ new = self._class()
+ #print "pushing", self._objectstack, repr(new)
+ if self._objectstack:
+ self._objectstack[-1].attach(new)
+ self._objectstack.append(new)
+ new._boundaryRE = None
+ new._finishing = False
+ self._cur = new
+
+ def _pop_container(self):
+ # Move the pointer to the container of the current object.
+ # Returns the (old) current object
+ #import pdb ; pdb.set_trace()
+ #print "popping", self._objectstack
+ last = self._objectstack.pop()
+ if self._objectstack:
+ self._cur = self._objectstack[-1]
+ else:
+ self._cur._finishing = True
+ return last
+
+
projects / python / commitdiff