bpo-17852: Maintain a list of BufferedWriter objects. Flush them on exit. (#1908)

* Maintain a list of BufferedWriter objects.  Flush them on exit.

In Python 3, the buffer and the underlying file object are separate
and so the order in which objects are finalized matters.  This is
unlike Python 2 where the file and buffer were a single object and
finalization was done for both at the same time.  In Python 3, if
the file is finalized and closed before the buffer then the data in
the buffer is lost.

This change adds a doubly linked list of open file buffers.  An atexit
hook ensures they are flushed before proceeding with interpreter
shutdown.  This is addition does not remove the need to properly close
files as there are other reasons why buffered data could get lost during
finalization.

Initial patch by Armin Rigo.

* Use weakref.WeakSet instead of WeakKeyDictionary.

* Simplify buffered double-linked list types.

* In _flush_all_writers(), suppress errors from flush().

* Remove NEWS entry, use blurb.

diff --git a/Lib/_pyio.py b/Lib/_pyio.py
index 4653847bcb18e9e7472ded828e2e3b750a66da1d..3aa2b24c040aaec61b74e15d920d46cb8fd1ecf5 100644 (file)
--- a/Lib/_pyio.py
+++ b/Lib/_pyio.py
@@ -1185,6 +1185,7 @@ class BufferedWriter(_BufferedIOMixin):
         self.buffer_size = buffer_size
         self._write_buf = bytearray()
         self._write_lock = Lock()
+        _register_writer(self)
 
     def writable(self):
         return self.raw.writable()
@@ -2574,3 +2575,26 @@ class StringIO(TextIOWrapper):
     def detach(self):
         # This doesn't make sense on StringIO.
         self._unsupported("detach")
+
+
+# ____________________________________________________________
+
+import atexit, weakref
+
+_all_writers = weakref.WeakSet()
+
+def _register_writer(w):
+    # keep weak-ref to buffered writer
+    _all_writers.add(w)
+
+def _flush_all_writers():
+    # Ensure all buffered writers are flushed before proceeding with
+    # normal shutdown.  Otherwise, if the underlying file objects get
+    # finalized before the buffered writer wrapping it then any buffered
+    # data will be lost.
+    for w in _all_writers:
+        try:
+            w.flush()
+        except:
+            pass
+atexit.register(_flush_all_writers)

diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-04-12-46-25.bpo-17852.OxAtCg.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-04-12-46-25.bpo-17852.OxAtCg.rst
new file mode 100644 (file)
index 0000000..185664c
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-04-12-46-25.bpo-17852.OxAtCg.rst
@@ -0,0 +1,2 @@
+Maintain a list of open buffered files, flush them before exiting the
+interpreter.  Based on a patch from Armin Rigo.

diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c
index f0621f4d4ab4bba7c3c98a939b73feba1bea4d1c..5db44f970d22ba6d5300ad267ee7baca741576b6 100644 (file)
--- a/Modules/_io/_iomodule.c
+++ b/Modules/_io/_iomodule.c
@@ -766,6 +766,8 @@ PyInit__io(void)
         !(_PyIO_empty_bytes = PyBytes_FromStringAndSize(NULL, 0)))
         goto fail;
 
+    _Py_PyAtExit(_PyIO_atexit_flush);
+
     state->initialized = 1;
 
     return m;

diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h
index db8403774ead23dabf0efe781082a1a49019bf74..1dce5dada4e23a014451bb4e58be047ebaa87721 100644 (file)
--- a/Modules/_io/_iomodule.h
+++ b/Modules/_io/_iomodule.h
@@ -183,3 +183,5 @@ extern PyObject *_PyIO_empty_str;
 extern PyObject *_PyIO_empty_bytes;
 
 extern PyTypeObject _PyBytesIOBuffer_Type;
+
+extern void _PyIO_atexit_flush(void);

diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c
index 189b1cd8442827fc08144fdaea0cf6bf699f700c..50c87c1746d6f21021e698948dc57e66fc0c742a 100644 (file)
--- a/Modules/_io/bufferedio.c
+++ b/Modules/_io/bufferedio.c
@@ -196,7 +196,7 @@ bufferediobase_write(PyObject *self, PyObject *args)
 }
 
 
-typedef struct {
+typedef struct _buffered {
     PyObject_HEAD
 
     PyObject *raw;
@@ -240,8 +240,18 @@ typedef struct {
 
     PyObject *dict;
     PyObject *weakreflist;
+
+    /* a doubly-linked chained list of "buffered" objects that need to
+       be flushed when the process exits */
+    struct _buffered *next, *prev;
 } buffered;
 
+/* the actual list of buffered objects */
+static buffered buffer_list_end = {
+    .next = &buffer_list_end,
+    .prev = &buffer_list_end
+};
+
 /*
     Implementation notes:
 
@@ -386,6 +396,15 @@ _enter_buffered_busy(buffered *self)
         (self->buffer_size * (size / self->buffer_size)))
 
 
+static void
+remove_from_linked_list(buffered *self)
+{
+    self->next->prev = self->prev;
+    self->prev->next = self->next;
+    self->prev = NULL;
+    self->next = NULL;
+}
+
 static void
 buffered_dealloc(buffered *self)
 {
@@ -394,6 +413,8 @@ buffered_dealloc(buffered *self)
         return;
     _PyObject_GC_UNTRACK(self);
     self->ok = 0;
+    if (self->next != NULL)
+        remove_from_linked_list(self);
     if (self->weakreflist != NULL)
         PyObject_ClearWeakRefs((PyObject *)self);
     Py_CLEAR(self->raw);
@@ -1817,10 +1838,33 @@ _io_BufferedWriter___init___impl(buffered *self, PyObject *raw,
     self->fast_closed_checks = (Py_TYPE(self) == &PyBufferedWriter_Type &&
                                 Py_TYPE(raw) == &PyFileIO_Type);
 
+    if (self->next == NULL) {
+        self->prev = &buffer_list_end;
+        self->next = buffer_list_end.next;
+        buffer_list_end.next->prev = self;
+        buffer_list_end.next = self;
+    }
+
     self->ok = 1;
     return 0;
 }
 
+/*
+* Ensure all buffered writers are flushed before proceeding with
+* normal shutdown.  Otherwise, if the underlying file objects get
+* finalized before the buffered writer wrapping it then any buffered
+* data will be lost.
+*/
+void _PyIO_atexit_flush(void)
+{
+    while (buffer_list_end.next != &buffer_list_end) {
+        buffered *buf = buffer_list_end.next;
+        remove_from_linked_list(buf);
+        buffered_flush(buf, NULL);
+        PyErr_Clear();
+    }
+}
+
 static Py_ssize_t
 _bufferedwriter_raw_write(buffered *self, char *start, Py_ssize_t len)
 {