]> granicus.if.org Git - python/commitdiff
Mass checkin of universal newline support.
authorJack Jansen <jack.jansen@cwi.nl>
Sun, 14 Apr 2002 20:12:41 +0000 (20:12 +0000)
committerJack Jansen <jack.jansen@cwi.nl>
Sun, 14 Apr 2002 20:12:41 +0000 (20:12 +0000)
Highlights: import and friends will understand any of \r, \n and \r\n
as end of line. Python file input will do the same if you use mode 'U'.
Everything can be disabled by configuring with --without-universal-newlines.

See PEP278 for details.

15 files changed:
Include/fileobject.h
Lib/linecache.py
Lib/py_compile.py
Mac/Python/macmain.c
Objects/fileobject.c
Parser/pgenmain.c
Parser/tokenizer.c
Python/bltinmodule.c
Python/errors.c
Python/import.c
Python/traceback.c
README
configure
configure.in
pyconfig.h.in

index 0876cd29a880d963441281eddaa0e0776b2861c9..d696475f947231026527d84c3263aecc5a722636 100644 (file)
@@ -14,8 +14,13 @@ typedef struct {
        PyObject *f_mode;
        int (*f_close)(FILE *);
        int f_softspace; /* Flag used by 'print' command */
-       int f_binary; /* Flag which indicates whether the file is
+       int f_binary; /* Flag which indicates whether the file is open
                         open in binary (1) or test (0) mode */
+#ifdef WITH_UNIVERSAL_NEWLINES
+       int f_univ_newline;     /* Handle any newline convention */
+       int f_newlinetypes;     /* Types of newlines seen */
+       int f_skipnextlf;       /* Skip next \n */
+#endif
 } PyFileObject;
 
 extern DL_IMPORT(PyTypeObject) PyFile_Type;
@@ -40,6 +45,19 @@ extern DL_IMPORT(int) PyObject_AsFileDescriptor(PyObject *);
 */
 extern DL_IMPORT(const char *) Py_FileSystemDefaultEncoding;
 
+#ifdef WITH_UNIVERSAL_NEWLINES
+/* Routines to replace fread() and fgets() which accept any of \r, \n
+   or \r\n as line terminators.
+*/
+#define PY_STDIOTEXTMODE "b"
+char *Py_UniversalNewlineFgets(char *, int, FILE*, PyObject *);
+size_t Py_UniversalNewlineFread(void *, size_t, FILE *, PyObject *);
+#else
+#define PY_STDIOTEXTMODE ""
+#define Py_UniversalNewlineFgets(buf, len, fp, obj) (fgets((buf), (len), (fp)))
+#define Py_UniversalNewlineFread(buf, len, fp, obj) \
+               (fread((buf), 1, (len), (fp)))
+#endif /* WITH_UNIVERSAL_NEWLINES */
 #ifdef __cplusplus
 }
 #endif
index cd3e50d27edde9e426fffe1e70a4554d013c6126..9953034d5150870f7a49c24883be2fe8dc0bc21d 100644 (file)
@@ -90,7 +90,7 @@ def updatecache(filename):
 ##          print '*** Cannot stat', filename, ':', msg
             return []
     try:
-        fp = open(fullname, 'r')
+        fp = open(fullname, 'rU')
         lines = fp.readlines()
         fp.close()
     except IOError, msg:
index 48c4afc36239f9ff5b0522956dd08b6bb63234ce..afeaae12c6f1f56801401cdc98d40997a37476c1 100644 (file)
@@ -44,7 +44,7 @@ def compile(file, cfile=None, dfile=None):
 
     """
     import os, marshal, __builtin__
-    f = open(file)
+    f = open(file, 'U')
     try:
         timestamp = long(os.fstat(f.fileno())[8])
     except AttributeError:
index 8b38a2c8b30f83064b4cf62910d33b2da8bcb3d8..a4f7dc4dc3392c7aebaf2e7f0c1dbe518cfe14be 100644 (file)
@@ -611,7 +611,7 @@ Py_Main(int argc, char **argv, char *filename)
                        Py_GetVersion(), Py_GetPlatform(), COPYRIGHT);
        
        if (filename != NULL) {
-               if ((fp = fopen(filename, "r")) == NULL) {
+               if ((fp = fopen(filename, "r" PY_STDIOTEXTMODE)) == NULL) {
                        fprintf(stderr, "%s: can't open file '%s'\n",
                                argv[0], filename);
                        PyMac_Exit(2);
@@ -630,7 +630,7 @@ Py_Main(int argc, char **argv, char *filename)
        PySys_SetArgv(argc, argv);
 
        if (filename == NULL && isatty((int)fileno(fp))) {
-               FILE *fp = fopen(STARTUP, "r");
+               FILE *fp = fopen(STARTUP, "r" PY_STDIOTEXTMODE);
                if (fp != NULL) {
                        (void) PyRun_SimpleFile(fp, STARTUP);
                        PyErr_Clear();
index 54d040de22af76caf6658891e45e5a69a7f21f0b..152ba1a470ac3a70e6ac2cf3f7d010854c7f5816 100644 (file)
 #include <errno.h>
 #endif
 
+#ifdef HAVE_GETC_UNLOCKED
+#define GETC(f) getc_unlocked(f)
+#define FLOCKFILE(f) flockfile(f)
+#define FUNLOCKFILE(f) funlockfile(f)
+#else
+#define GETC(f) getc(f)
+#define FLOCKFILE(f)
+#define FUNLOCKFILE(f)
+#endif
+
+#ifdef WITH_UNIVERSAL_NEWLINES
+/* Bits in f_newlinetypes */
+#define NEWLINE_UNKNOWN        0       /* No newline seen, yet */
+#define NEWLINE_CR 1           /* \r newline seen */
+#define NEWLINE_LF 2           /* \n newline seen */
+#define NEWLINE_CRLF 4         /* \r\n newline seen */
+#endif
 
 FILE *
 PyFile_AsFile(PyObject *f)
@@ -99,6 +116,11 @@ fill_file_fields(PyFileObject *f, FILE *fp, char *name, char *mode,
        f->f_close = close;
        f->f_softspace = 0;
        f->f_binary = strchr(mode,'b') != NULL;
+#ifdef WITH_UNIVERSAL_NEWLINES
+       f->f_univ_newline = (strchr(mode, 'U') != NULL);
+       f->f_newlinetypes = NEWLINE_UNKNOWN;
+       f->f_skipnextlf = 0;
+#endif
 
        if (f->f_name == NULL || f->f_mode == NULL)
                return NULL;
@@ -134,6 +156,17 @@ open_the_file(PyFileObject *f, char *name, char *mode)
 #endif
        {
                Py_BEGIN_ALLOW_THREADS
+#ifdef WITH_UNIVERSAL_NEWLINES
+               if (strcmp(mode, "U") == 0 || strcmp(mode, "rU") == 0)
+                       mode = "rb";
+#else
+               /* Compatibility: specifying U in a Python without universal
+               ** newlines is allowed, and the file is opened as a normal text
+               ** file.
+               */
+               if (strcmp(mode, "U") == 0 || strcmp(mode, "rU") == 0)
+                       mode = "r";
+#endif
                f->f_fp = fopen(name, mode);
                Py_END_ALLOW_THREADS
        }
@@ -394,6 +427,9 @@ file_seek(PyFileObject *f, PyObject *args)
                clearerr(f->f_fp);
                return NULL;
        }
+#ifdef WITH_UNIVERSAL_NEWLINES
+       f->f_skipnextlf = 0;
+#endif
        Py_INCREF(Py_None);
        return Py_None;
 }
@@ -534,6 +570,16 @@ file_tell(PyFileObject *f)
                clearerr(f->f_fp);
                return NULL;
        }
+#ifdef WITH_UNIVERSAL_NEWLINES
+       if (f->f_skipnextlf) {
+               int c;
+               c = GETC(f->f_fp);
+               if (c == '\n') {
+                       pos++;
+                       f->f_skipnextlf = 0;
+               } else if (c != EOF) ungetc(c, f->f_fp);
+       }
+#endif
 #if !defined(HAVE_LARGEFILE_SUPPORT)
        return PyInt_FromLong(pos);
 #else
@@ -665,8 +711,8 @@ file_read(PyFileObject *f, PyObject *args)
        for (;;) {
                Py_BEGIN_ALLOW_THREADS
                errno = 0;
-               chunksize = fread(BUF(v) + bytesread, 1,
-                                 buffersize - bytesread, f->f_fp);
+               chunksize = Py_UniversalNewlineFread(BUF(v) + bytesread,
+                                 buffersize - bytesread, f->f_fp, (PyObject *)f);
                Py_END_ALLOW_THREADS
                if (chunksize == 0) {
                        if (!ferror(f->f_fp))
@@ -705,7 +751,7 @@ file_readinto(PyFileObject *f, PyObject *args)
        while (ntodo > 0) {
                Py_BEGIN_ALLOW_THREADS
                errno = 0;
-               nnow = fread(ptr+ndone, 1, ntodo, f->f_fp);
+               nnow = Py_UniversalNewlineFread(ptr+ndone, ntodo, f->f_fp, (PyObject *)f);
                Py_END_ALLOW_THREADS
                if (nnow == 0) {
                        if (!ferror(f->f_fp))
@@ -934,16 +980,6 @@ getline_via_fgets(FILE *fp)
    <= 0: read arbitrary line
 */
 
-#ifdef HAVE_GETC_UNLOCKED
-#define GETC(f) getc_unlocked(f)
-#define FLOCKFILE(f) flockfile(f)
-#define FUNLOCKFILE(f) funlockfile(f)
-#else
-#define GETC(f) getc(f)
-#define FLOCKFILE(f)
-#define FUNLOCKFILE(f)
-#endif
-
 static PyObject *
 get_line(PyFileObject *f, int n)
 {
@@ -954,9 +990,18 @@ get_line(PyFileObject *f, int n)
        size_t used_v_size;     /* # used slots in buffer */
        size_t increment;       /* amount to increment the buffer */
        PyObject *v;
+#ifdef WITH_UNIVERSAL_NEWLINES
+       int newlinetypes = f->f_newlinetypes;
+       int skipnextlf = f->f_skipnextlf;
+       int univ_newline = f->f_univ_newline;
+#endif
 
-#ifdef USE_FGETS_IN_GETLINE
+#if defined(USE_FGETS_IN_GETLINE)
+#ifdef WITH_UNIVERSAL_NEWLINES
+       if (n <= 0 && !univ_newline )
+#else
        if (n <= 0)
+#endif
                return getline_via_fgets(fp);
 #endif
        total_v_size = n > 0 ? n : 100;
@@ -969,12 +1014,45 @@ get_line(PyFileObject *f, int n)
        for (;;) {
                Py_BEGIN_ALLOW_THREADS
                FLOCKFILE(fp);
+#ifdef WITH_UNIVERSAL_NEWLINES
+               if (univ_newline) {
+                       c = 'x'; /* Shut up gcc warning */
+                       while ( buf != end && (c = GETC(fp)) != EOF ) {
+                               if (skipnextlf ) {
+                                       skipnextlf = 0;
+                                       if (c == '\n') {
+                                               /* Seeing a \n here with skipnextlf true
+                                               ** means we saw a \r before.
+                                               */
+                                               newlinetypes |= NEWLINE_CRLF;
+                                               c = GETC(fp);
+                                               if (c == EOF) break;
+                                       } else {
+                                               newlinetypes |= NEWLINE_CR;
+                                       }
+                               }
+                               if (c == '\r') {
+                                       skipnextlf = 1;
+                                       c = '\n';
+                               } else if ( c == '\n')
+                                       newlinetypes |= NEWLINE_LF;
+                               *buf++ = c;
+                               if (c == '\n') break;
+                       }
+                       if ( c == EOF && skipnextlf )
+                               newlinetypes |= NEWLINE_CR;
+               } else /* If not universal newlines use the normal loop */
+#endif
                while ((c = GETC(fp)) != EOF &&
                       (*buf++ = c) != '\n' &&
                        buf != end)
                        ;
                FUNLOCKFILE(fp);
                Py_END_ALLOW_THREADS
+#ifdef WITH_UNIVERSAL_NEWLINES
+               f->f_newlinetypes = newlinetypes;
+               f->f_skipnextlf = skipnextlf;
+#endif
                if (c == '\n')
                        break;
                if (c == EOF) {
@@ -1150,8 +1228,8 @@ file_readlines(PyFileObject *f, PyObject *args)
                else {
                        Py_BEGIN_ALLOW_THREADS
                        errno = 0;
-                       nread = fread(buffer+nfilled, 1,
-                                     buffersize-nfilled, f->f_fp);
+                       nread = Py_UniversalNewlineFread(buffer+nfilled, 
+                               buffersize-nfilled, f->f_fp, (PyObject *)f);
                        Py_END_ALLOW_THREADS
                        shortread = (nread < buffersize-nfilled);
                }
@@ -1188,7 +1266,8 @@ file_readlines(PyFileObject *f, PyObject *args)
                        }
                        else {
                                /* Grow the big buffer */
-                               _PyString_Resize(&big_buffer, buffersize);
+                               if ( _PyString_Resize(&big_buffer, buffersize) < 0 )
+                                       goto error;
                                buffer = PyString_AS_STRING(big_buffer);
                        }
                        continue;
@@ -1503,9 +1582,40 @@ get_closed(PyFileObject *f, void *closure)
 {
        return PyBool_FromLong((long)(f->f_fp == 0));
 }
+#ifdef WITH_UNIVERSAL_NEWLINES
+static PyObject *
+get_newlines(PyFileObject *f, void *closure)
+{
+       switch (f->f_newlinetypes) {
+       case NEWLINE_UNKNOWN:
+               Py_INCREF(Py_None);
+               return Py_None;
+       case NEWLINE_CR:
+               return PyString_FromString("\r");
+       case NEWLINE_LF:
+               return PyString_FromString("\n");
+       case NEWLINE_CR|NEWLINE_LF:
+               return Py_BuildValue("(ss)", "\r", "\n");
+       case NEWLINE_CRLF:
+               return PyString_FromString("\r\n");
+       case NEWLINE_CR|NEWLINE_CRLF:
+               return Py_BuildValue("(ss)", "\r", "\r\n");
+       case NEWLINE_LF|NEWLINE_CRLF:
+               return Py_BuildValue("(ss)", "\n", "\r\n");
+       case NEWLINE_CR|NEWLINE_LF|NEWLINE_CRLF:
+               return Py_BuildValue("(sss)", "\r", "\n", "\r\n");
+       default:
+               PyErr_Format(PyExc_SystemError, "Unknown newlines value 0x%x\n", f->f_newlinetypes);
+               return NULL;
+       }
+}
+#endif
 
 static PyGetSetDef file_getsetlist[] = {
        {"closed", (getter)get_closed, NULL, "True if the file is closed"},
+#ifdef WITH_UNIVERSAL_NEWLINES
+       {"newlines", (getter)get_newlines, NULL, "end-of-line convention used in this file"},
+#endif
        {0},
 };
 
@@ -1805,3 +1915,170 @@ int PyObject_AsFileDescriptor(PyObject *o)
        }
        return fd;
 }
+
+#ifdef WITH_UNIVERSAL_NEWLINES
+/* From here on we need access to the real fgets and fread */
+#undef fgets
+#undef fread
+
+/*
+** Py_UniversalNewlineFgets is an fgets variation that understands
+** all of \r, \n and \r\n conventions.
+** The stream should be opened in binary mode.
+** If fobj is NULL the routine always does newline conversion, and
+** it may peek one char ahead to gobble the second char in \r\n.
+** If fobj is non-NULL it must be a PyFileObject. In this case there
+** is no readahead but in stead a flag is used to skip a following
+** \n on the next read. Also, if the file is open in binary mode
+** the whole conversion is skipped. Finally, the routine keeps track of
+** the different types of newlines seen.
+** Note that we need no error handling: fgets() treats error and eof
+** identically.
+*/
+char *
+Py_UniversalNewlineFgets(char *buf, int n, FILE *stream, PyObject *fobj)
+{
+       char *p = buf;
+       int c;
+       int newlinetypes = 0;
+       int skipnextlf = 0;
+       int univ_newline = 1;
+       
+       if (fobj) {
+               if (!PyFile_Check(fobj)) {
+                       errno = ENXIO;  /* What can you do... */
+                       return NULL;
+               }
+               univ_newline = ((PyFileObject *)fobj)->f_univ_newline;
+               if ( !univ_newline )
+                       return fgets(buf, n, stream);
+               newlinetypes = ((PyFileObject *)fobj)->f_newlinetypes;
+               skipnextlf = ((PyFileObject *)fobj)->f_skipnextlf;
+       }
+       FLOCKFILE(stream);
+       c = 'x'; /* Shut up gcc warning */
+       while (--n > 0 && (c = GETC(stream)) != EOF ) {
+               if (skipnextlf ) {
+                       skipnextlf = 0;
+                       if (c == '\n') {
+                               /* Seeing a \n here with skipnextlf true
+                               ** means we saw a \r before.
+                               */
+                               newlinetypes |= NEWLINE_CRLF;
+                               c = GETC(stream);
+                               if (c == EOF) break;
+                       } else {
+                               /*
+                               ** Note that c == EOF also brings us here,
+                               ** so we're okay if the last char in the file
+                               ** is a CR.
+                               */
+                               newlinetypes |= NEWLINE_CR;
+                       }
+               }
+               if (c == '\r') {
+                       /* A \r is translated into a \n, and we skip
+                       ** an adjacent \n, if any. We don't set the
+                       ** newlinetypes flag until we've seen the next char.
+                       */
+                       skipnextlf = 1;
+                       c = '\n';
+               } else if ( c == '\n') {
+                       newlinetypes |= NEWLINE_LF;
+               }
+               *p++ = c;
+               if (c == '\n') break;
+       }
+       if ( c == EOF && skipnextlf )
+               newlinetypes |= NEWLINE_CR;
+       FUNLOCKFILE(stream);
+       *p = '\0';
+       if (fobj) {
+               ((PyFileObject *)fobj)->f_newlinetypes = newlinetypes;
+               ((PyFileObject *)fobj)->f_skipnextlf = skipnextlf;
+       } else if ( skipnextlf ) {
+               /* If we have no file object we cannot save the
+               ** skipnextlf flag. We have to readahead, which
+               ** will cause a pause if we're reading from an
+               ** interactive stream, but that is very unlikely
+               ** unless we're doing something silly like
+               ** execfile("/dev/tty").
+               */
+               c = GETC(stream);
+               if ( c != '\n' )
+                       ungetc(c, stream);
+       }
+       if (p == buf)
+               return NULL;
+       return buf;
+}
+
+/*
+** Py_UniversalNewlineFread is an fread variation that understands
+** all of \r, \n and \r\n conventions.
+** The stream should be opened in binary mode.
+** fobj must be a PyFileObject. In this case there
+** is no readahead but in stead a flag is used to skip a following
+** \n on the next read. Also, if the file is open in binary mode
+** the whole conversion is skipped. Finally, the routine keeps track of
+** the different types of newlines seen.
+*/
+size_t
+Py_UniversalNewlineFread(void *buf, size_t n,
+                        FILE *stream, PyObject *fobj)
+{
+       char *src = buf, *dst = buf, c;
+       int nread, ntodo=n;
+       int newlinetypes, skipnextlf, univ_newline;
+       
+       if (!fobj || !PyFile_Check(fobj)) {
+               errno = ENXIO;  /* What can you do... */
+               return -1;
+       }
+       univ_newline = ((PyFileObject *)fobj)->f_univ_newline;
+       if ( !univ_newline )
+               return fread(buf, 1, n, stream);
+       newlinetypes = ((PyFileObject *)fobj)->f_newlinetypes;
+       skipnextlf = ((PyFileObject *)fobj)->f_skipnextlf;
+       while (ntodo > 0) {
+               if (ferror(stream))
+                       break;
+               nread = fread(dst, 1, ntodo, stream);
+               src = dst;
+               if (nread <= 0) {
+                       if (skipnextlf)
+                               newlinetypes |= NEWLINE_CR;
+                       break;
+               }
+               ntodo -= nread;
+               while ( nread-- ) {
+                       c = *src++;
+                       if (c == '\r') {
+                               /* Save CR as LF and set flag to skip next newline
+                               */
+                               *dst++ = '\n';
+                               skipnextlf = 1;
+                       } else if (skipnextlf && c == '\n') {
+                               /* Skip an LF, and remember that we saw CR LF
+                               */
+                               skipnextlf = 0;
+                               newlinetypes |= NEWLINE_CRLF;
+                       } else {
+                               /* Normal char to be stored in buffer. Also update
+                               ** the newlinetypes flag if either this is an LF
+                               ** or the previous char was a CR.
+                               */
+                               if (c == '\n')
+                                       newlinetypes |= NEWLINE_LF;
+                               else if (skipnextlf)
+                                       newlinetypes |= NEWLINE_CR;
+                               *dst++ = c;
+                               skipnextlf = 0;
+                       }
+               }
+       }
+       ((PyFileObject *)fobj)->f_newlinetypes = newlinetypes;
+       ((PyFileObject *)fobj)->f_skipnextlf = skipnextlf;
+       return dst - (char *)buf;
+}
+#endif
index 2a284bc08a54e4d78b33da1567f6284bfd3a0eb5..d25cbd4f8f52f63288456a06412f5b8b065dd0b2 100644 (file)
@@ -13,6 +13,7 @@
    - check for duplicate definitions of names (instead of fatal err)
 */
 
+#include "Python.h"
 #include "pgenheaders.h"
 #include "grammar.h"
 #include "node.h"
@@ -182,6 +183,16 @@ PyOS_Readline(char *prompt)
        return PyMem_REALLOC(p, n+1);
 }
 
+#ifdef WITH_UNIVERSAL_NEWLINES
+/* No-nonsense fgets */
+char *
+Py_UniversalNewlineFgets(char *buf, int n, FILE *stream, PyObject *fobj)
+{
+       return fgets(buf, n, stream);
+}
+#endif
+
+
 #include <stdarg.h>
 
 void
index 324d9b6548c77b7fabe03ace2bf3b5f56b3a3ebb..b4e0fbf7e58d9446e02d3f5af619b42b0191f2fa 100644 (file)
@@ -1,6 +1,7 @@
 
 /* Tokenizer implementation */
 
+#include "Python.h"
 #include "pgenheaders.h"
 
 #include <ctype.h>
@@ -245,8 +246,8 @@ tok_nextc(register struct tok_state *tok)
                                        }
                                        tok->end = tok->buf + BUFSIZ;
                                }
-                               if (fgets(tok->buf, (int)(tok->end - tok->buf),
-                                         tok->fp) == NULL) {
+                               if (Py_UniversalNewlineFgets(tok->buf, (int)(tok->end - tok->buf),
+                                         tok->fp, NULL) == NULL) {
                                        tok->done = E_EOF;
                                        done = 1;
                                }
@@ -284,9 +285,9 @@ tok_nextc(register struct tok_state *tok)
                                tok->end = tok->buf + newsize;
                                tok->start = curstart < 0 ? NULL :
                                             tok->buf + curstart;
-                               if (fgets(tok->inp,
+                               if (Py_UniversalNewlineFgets(tok->inp,
                                               (int)(tok->end - tok->inp),
-                                              tok->fp) == NULL) {
+                                              tok->fp, NULL) == NULL) {
                                        /* Last line does not end in \n,
                                           fake one */
                                        strcpy(tok->inp, "\n");
index 35536d99d2d6df321514b0f633e89bee67cebcc0..680152d510b5215c514c141bf31e208354bb8fd9 100644 (file)
@@ -594,7 +594,7 @@ builtin_execfile(PyObject *self, PyObject *args)
 
         if (exists) {
                Py_BEGIN_ALLOW_THREADS
-               fp = fopen(filename, "r");
+               fp = fopen(filename, "r" PY_STDIOTEXTMODE);
                Py_END_ALLOW_THREADS
 
                if (fp == NULL) {
index 3869b1cc5c3b4a3bf430ca4171ed425400bd71be..265e5bb48e3df9ed70d5bcd9de9cfa7f3a91fb11 100644 (file)
@@ -646,14 +646,14 @@ PyErr_ProgramText(char *filename, int lineno)
 
        if (filename == NULL || lineno <= 0)
                return NULL;
-       fp = fopen(filename, "r");
+       fp = fopen(filename, "r" PY_STDIOTEXTMODE);
        if (fp == NULL)
                return NULL;
        for (i = 0; i < lineno; i++) {
                char *pLastChar = &linebuf[sizeof(linebuf) - 2];
                do {
                        *pLastChar = '\0';
-                       if (fgets(linebuf, sizeof linebuf, fp) == NULL)
+                       if (Py_UniversalNewlineFgets(linebuf, sizeof linebuf, fp, NULL) == NULL)
                                break;
                        /* fgets read *something*; if it didn't get as
                           far as pLastChar, it must have found a newline
index 3c8750671e5d7741b6527a63584b223ea247d564..3775f886e3144fc331f99b0194342bd6ee86efba 100644 (file)
@@ -81,15 +81,15 @@ struct filedescr * _PyImport_Filetab = NULL;
 
 #ifdef RISCOS
 static const struct filedescr _PyImport_StandardFiletab[] = {
-       {"/py", "r", PY_SOURCE},
+       {"/py", "r" PY_STDIOTEXTMODE, PY_SOURCE},
        {"/pyc", "rb", PY_COMPILED},
        {0, 0}
 };
 #else
 static const struct filedescr _PyImport_StandardFiletab[] = {
-       {".py", "r", PY_SOURCE},
+       {".py", "r" PY_STDIOTEXTMODE, PY_SOURCE},
 #ifdef MS_WIN32
-       {".pyw", "r", PY_SOURCE},
+       {".pyw", "r" PY_STDIOTEXTMODE, PY_SOURCE},
 #endif
        {".pyc", "rb", PY_COMPILED},
        {0, 0}
index 52f3202ad5d5ec1bf3d65812fd8a3b49fd34f002..de918f9a203e6304ceca12bd9c628ab9564b51b3 100644 (file)
@@ -155,7 +155,7 @@ tb_displayline(PyObject *f, char *filename, int lineno, char *name)
        /* This is needed by Emacs' compile command */
 #define FMT "  File \"%.500s\", line %d, in %.500s\n"
 #endif
-       xfp = fopen(filename, "r");
+       xfp = fopen(filename, "r" PY_STDIOTEXTMODE);
        if (xfp == NULL) {
                /* Search tail of filename in sys.path before giving up */
                PyObject *path;
@@ -186,7 +186,7 @@ tb_displayline(PyObject *f, char *filename, int lineno, char *name)
                                        if (len > 0 && namebuf[len-1] != SEP)
                                                namebuf[len++] = SEP;
                                        strcpy(namebuf+len, tail);
-                                       xfp = fopen(namebuf, "r");
+                                       xfp = fopen(namebuf, "r" PY_STDIOTEXTMODE);
                                        if (xfp != NULL) {
                                                filename = namebuf;
                                                break;
@@ -203,7 +203,7 @@ tb_displayline(PyObject *f, char *filename, int lineno, char *name)
                char* pLastChar = &linebuf[sizeof(linebuf)-2];
                do {
                        *pLastChar = '\0';
-                       if (fgets(linebuf, sizeof linebuf, xfp) == NULL)
+                       if (Py_UniversalNewlineFgets(linebuf, sizeof linebuf, xfp, NULL) == NULL)
                                break;
                        /* fgets read *something*; if it didn't get as
                           far as pLastChar, it must have found a newline
diff --git a/README b/README
index d5c5d58483661b04768b2197948cdbf1cf2f0606..38117a7534c8abbb7d16256912ed812092a272d8 100644 (file)
--- a/README
+++ b/README
@@ -811,6 +811,13 @@ Modules/getpath.o.
 --with-pydebug:  Enable additional debugging code to help track down
        memory management problems.  This allows printing a list of all
        live objects when the interpreter terminates.
+       
+--with(out)-universal-newlines: enable reading of text files with
+       foreign newline convention (default: enabled). In other words,
+       any of \r, \n or \r\n is acceptable as end-of-line character.
+       If enabled import and execfile will automatically accept any newline
+       in files. Python code can open a file with open(file, 'U') to
+       read it in universal newline mode.
 
 
 Building for multiple architectures (using the VPATH feature)
index c9c9bd2f06f96da9ec90b63735ee54b2c507d0d8..9d3fb08b296b9c239135fa83aa6324fc11f95e6f 100755 (executable)
--- a/configure
+++ b/configure
@@ -844,6 +844,7 @@ Optional Packages:
   --with(out)-thread=DIRECTORY  deprecated; use --with(out)-threads
   --with-pth                      use GNU pth threading libraries
   --with(out)-cycle-gc            disable/enable garbage collection
+  --with(out)-universal-newlines            disable/enable foreign newlines
   --with(out)-pymalloc            disable/enable specialized mallocs
   --with-wctype-functions         use wctype.h functions
   --with-sgi-dl=DIRECTORY         IRIX 4 dynamic linking
 echo "$as_me:$LINENO: result: $with_cycle_gc" >&5
 echo "${ECHO_T}$with_cycle_gc" >&6
 
+# Check for universal newline support
+echo "$as_me:$LINENO: checking for --with-universal-newline" >&5
+echo $ECHO_N "checking for --with-universal-newline... $ECHO_C" >&6
+
+# Check whether --with-universal-newlines or --without-universal-newlines was given.
+if test "${with_universal_newlines+set}" = set; then
+  withval="$with_universal_newlines"
+
+fi;
+
+if test -z "$with_universal_newlines"
+then with_universal_newlines="yes"
+fi
+if test "$with_universal_newlines" != "no"
+then
+    cat >>confdefs.h <<\_ACEOF
+#define WITH_UNIVERSAL_NEWLINES 1
+_ACEOF
+
+fi
+echo "$as_me:$LINENO: result: $with_universal_newlines" >&5
+echo "${ECHO_T}$with_universal_newlines" >&6
+
 # Check for Python-specific malloc support
 echo "$as_me:$LINENO: checking for --with-pymalloc" >&5
 echo $ECHO_N "checking for --with-pymalloc... $ECHO_C" >&6
index 8d1339a00747321dee66e0bd0785d0d7f3d2da10..d3211a0e9b580affaa0adaf3fda4904bc78f452d 100644 (file)
@@ -1445,6 +1445,20 @@ then
 fi
 AC_MSG_RESULT($with_cycle_gc)
 
+# Check for universal newline support
+AC_MSG_CHECKING(for --with-universal-newline)
+AC_ARG_WITH(universal-newlines,
+[  --with(out)-universal-newlines            disable/enable foreign newlines])
+
+if test -z "$with_universal_newlines"
+then with_universal_newlines="yes"
+fi
+if test "$with_universal_newlines" != "no"
+then
+    AC_DEFINE(WITH_UNIVERSAL_NEWLINES)
+fi
+AC_MSG_RESULT($with_universal_newlines)
+
 # Check for Python-specific malloc support
 AC_MSG_CHECKING(for --with-pymalloc)
 AC_ARG_WITH(pymalloc,
index dbc03be2ea9830146690d5c35f8a3655e1b5e07d..0bd548596a7a2c1434dd4dd216a74918030b35af 100644 (file)
 /* Define if you want to compile in cycle garbage collection. */
 #undef WITH_CYCLE_GC
 
+/* Define if you want to read files with foreign newlines. */
+#undef WITH_UNIVERSAL_NEWLINES
+
 /* Define if you want to emulate SGI (IRIX 4) dynamic linking. This is
    rumoured to work on VAX (Ultrix), Sun3 (SunOS 3.4), Sequent Symmetry
    (Dynix), and Atari ST. This requires the 'dl-dld' library,