darn! I converted half of the files the wrong way.
authorBenjamin Peterson <benjamin@python.org>
Fri, 13 Jun 2008 19:20:48 +0000 (19:20 +0000)
committerBenjamin Peterson <benjamin@python.org>
Fri, 13 Jun 2008 19:20:48 +0000 (19:20 +0000)
12 files changed:
Lib/multiprocessing/__init__.py
Lib/multiprocessing/connection.py
Lib/multiprocessing/forking.py
Lib/multiprocessing/heap.py
Lib/multiprocessing/managers.py
Lib/multiprocessing/pool.py
Lib/multiprocessing/process.py
Lib/multiprocessing/queues.py
Lib/multiprocessing/reduction.py
Lib/multiprocessing/sharedctypes.py
Lib/multiprocessing/synchronize.py
Lib/multiprocessing/util.py

index decb2ada17438ced42d0ebdbe6bb6e35054cc480..f96505630ae5487a4a326bd8907adb0d75f2edeb 100644 (file)
-#\r
-# Package analogous to 'threading.py' but using processes\r
-#\r
-# multiprocessing/__init__.py\r
-#\r
-# This package is intended to duplicate the functionality (and much of\r
-# the API) of threading.py but uses processes instead of threads.  A\r
-# subpackage 'multiprocessing.dummy' has the same API but is a simple\r
-# wrapper for 'threading'.\r
-#\r
-# Try calling `multiprocessing.doc.main()` to read the html\r
-# documentation in in a webbrowser.\r
-#\r
-#\r
-# Copyright (c) 2006-2008, R Oudkerk\r
-# All rights reserved.\r
-#\r
-# Redistribution and use in source and binary forms, with or without\r
-# modification, are permitted provided that the following conditions\r
-# are met:\r
-#\r
-# 1. Redistributions of source code must retain the above copyright\r
-#    notice, this list of conditions and the following disclaimer.\r
-# 2. Redistributions in binary form must reproduce the above copyright\r
-#    notice, this list of conditions and the following disclaimer in the\r
-#    documentation and/or other materials provided with the distribution.\r
-# 3. Neither the name of author nor the names of any contributors may be\r
-#    used to endorse or promote products derived from this software\r
-#    without specific prior written permission.\r
-#\r
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND\r
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\r
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\r
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE\r
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\r
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS\r
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\r
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY\r
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF\r
-#\r
-\r
-__version__ = '0.70a1'\r
-\r
-__all__ = [\r
-    'Process', 'current_process', 'active_children', 'freeze_support',\r
-    'Manager', 'Pipe', 'cpu_count', 'log_to_stderr', 'get_logger',\r
-    'allow_connection_pickling', 'BufferTooShort', 'TimeoutError',\r
-    'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition',\r
-    'Event', 'Queue', 'JoinableQueue', 'Pool', 'Value', 'Array',\r
-    'RawValue', 'RawArray'\r
-    ]\r
-\r
-__author__ = 'R. Oudkerk (r.m.oudkerk@gmail.com)'\r
-\r
-#\r
-# Imports\r
-#\r
-\r
-import os\r
-import sys\r
-\r
-from multiprocessing.process import Process, current_process, active_children\r
-\r
-#\r
-# Exceptions\r
-#\r
-\r
-class ProcessError(Exception):\r
-    pass\r
-\r
-class BufferTooShort(ProcessError):\r
-    pass\r
-\r
-class TimeoutError(ProcessError):\r
-    pass\r
-\r
-class AuthenticationError(ProcessError):\r
-    pass\r
-\r
-# This is down here because _multiprocessing uses BufferTooShort\r
-import _multiprocessing\r
-\r
-#\r
-# Definitions not depending on native semaphores\r
-#\r
-\r
-def Manager():\r
-    '''\r
-    Returns a manager associated with a running server process\r
-\r
-    The managers methods such as `Lock()`, `Condition()` and `Queue()`\r
-    can be used to create shared objects.\r
-    '''\r
-    from multiprocessing.managers import SyncManager\r
-    m = SyncManager()\r
-    m.start()\r
-    return m\r
-\r
-def Pipe(duplex=True):\r
-    '''\r
-    Returns two connection object connected by a pipe\r
-    '''\r
-    from multiprocessing.connection import Pipe\r
-    return Pipe(duplex)\r
-\r
-def cpu_count():\r
-    '''\r
-    Returns the number of CPUs in the system\r
-    '''\r
-    if sys.platform == 'win32':\r
-        try:\r
-            num = int(os.environ['NUMBER_OF_PROCESSORS'])\r
-        except (ValueError, KeyError):\r
-            num = 0\r
-    elif sys.platform == 'darwin':\r
-        try:\r
-            num = int(os.popen('sysctl -n hw.ncpu').read())\r
-        except ValueError:\r
-            num = 0\r
-    else:\r
-        try:\r
-            num = os.sysconf('SC_NPROCESSORS_ONLN')\r
-        except (ValueError, OSError, AttributeError):\r
-            num = 0\r
-\r
-    if num >= 1:\r
-        return num\r
-    else:\r
-        raise NotImplementedError('cannot determine number of cpus')\r
-\r
-def freeze_support():\r
-    '''\r
-    Check whether this is a fake forked process in a frozen executable.\r
-    If so then run code specified by commandline and exit.\r
-    '''\r
-    if sys.platform == 'win32' and getattr(sys, 'frozen', False):\r
-        from multiprocessing.forking import freeze_support\r
-        freeze_support()\r
-\r
-def get_logger():\r
-    '''\r
-    Return package logger -- if it does not already exist then it is created\r
-    '''\r
-    from multiprocessing.util import get_logger\r
-    return get_logger()\r
-\r
-def log_to_stderr(level=None):\r
-    '''\r
-    Turn on logging and add a handler which prints to stderr\r
-    '''\r
-    from multiprocessing.util import log_to_stderr\r
-    return log_to_stderr(level)\r
-\r
-def allow_connection_pickling():\r
-    '''\r
-    Install support for sending connections and sockets between processes\r
-    '''\r
-    from multiprocessing import reduction\r
-\r
-#\r
-# Definitions depending on native semaphores\r
-#\r
-\r
-def Lock():\r
-    '''\r
-    Returns a non-recursive lock object\r
-    '''\r
-    from multiprocessing.synchronize import Lock\r
-    return Lock()\r
-\r
-def RLock():\r
-    '''\r
-    Returns a recursive lock object\r
-    '''\r
-    from multiprocessing.synchronize import RLock\r
-    return RLock()\r
-\r
-def Condition(lock=None):\r
-    '''\r
-    Returns a condition object\r
-    '''\r
-    from multiprocessing.synchronize import Condition\r
-    return Condition(lock)\r
-\r
-def Semaphore(value=1):\r
-    '''\r
-    Returns a semaphore object\r
-    '''\r
-    from multiprocessing.synchronize import Semaphore\r
-    return Semaphore(value)\r
-\r
-def BoundedSemaphore(value=1):\r
-    '''\r
-    Returns a bounded semaphore object\r
-    '''\r
-    from multiprocessing.synchronize import BoundedSemaphore\r
-    return BoundedSemaphore(value)\r
-\r
-def Event():\r
-    '''\r
-    Returns an event object\r
-    '''\r
-    from multiprocessing.synchronize import Event\r
-    return Event()\r
-\r
-def Queue(maxsize=0):\r
-    '''\r
-    Returns a queue object\r
-    '''\r
-    from multiprocessing.queues import Queue\r
-    return Queue(maxsize)\r
-\r
-def JoinableQueue(maxsize=0):\r
-    '''\r
-    Returns a queue object\r
-    '''\r
-    from multiprocessing.queues import JoinableQueue\r
-    return JoinableQueue(maxsize)\r
-\r
-def Pool(processes=None, initializer=None, initargs=()):\r
-    '''\r
-    Returns a process pool object\r
-    '''\r
-    from multiprocessing.pool import Pool\r
-    return Pool(processes, initializer, initargs)\r
-\r
-def RawValue(typecode_or_type, *args):\r
-    '''\r
-    Returns a shared object\r
-    '''\r
-    from multiprocessing.sharedctypes import RawValue\r
-    return RawValue(typecode_or_type, *args)\r
-\r
-def RawArray(typecode_or_type, size_or_initializer):\r
-    '''\r
-    Returns a shared array\r
-    '''\r
-    from multiprocessing.sharedctypes import RawArray\r
-    return RawArray(typecode_or_type, size_or_initializer)\r
-\r
-def Value(typecode_or_type, *args, **kwds):\r
-    '''\r
-    Returns a synchronized shared object\r
-    '''\r
-    from multiprocessing.sharedctypes import Value\r
-    return Value(typecode_or_type, *args, **kwds)\r
-\r
-def Array(typecode_or_type, size_or_initializer, **kwds):\r
-    '''\r
-    Returns a synchronized shared array\r
-    '''\r
-    from multiprocessing.sharedctypes import Array\r
-    return Array(typecode_or_type, size_or_initializer, **kwds)\r
-\r
-#\r
-#\r
-#\r
-\r
-if sys.platform == 'win32':\r
-\r
-    def set_executable(executable):\r
-        '''\r
-        Sets the path to a python.exe or pythonw.exe binary used to run\r
-        child processes on Windows instead of sys.executable.\r
-        Useful for people embedding Python.\r
-        '''\r
-        from multiprocessing.forking import set_executable\r
-        set_executable(executable)\r
-\r
-    __all__ += ['set_executable']\r
+#
+# Package analogous to 'threading.py' but using processes
+#
+# multiprocessing/__init__.py
+#
+# This package is intended to duplicate the functionality (and much of
+# the API) of threading.py but uses processes instead of threads.  A
+# subpackage 'multiprocessing.dummy' has the same API but is a simple
+# wrapper for 'threading'.
+#
+# Try calling `multiprocessing.doc.main()` to read the html
+# documentation in in a webbrowser.
+#
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+#
+
+__version__ = '0.70a1'
+
+__all__ = [
+    'Process', 'current_process', 'active_children', 'freeze_support',
+    'Manager', 'Pipe', 'cpu_count', 'log_to_stderr', 'get_logger',
+    'allow_connection_pickling', 'BufferTooShort', 'TimeoutError',
+    'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition',
+    'Event', 'Queue', 'JoinableQueue', 'Pool', 'Value', 'Array',
+    'RawValue', 'RawArray'
+    ]
+
+__author__ = 'R. Oudkerk (r.m.oudkerk@gmail.com)'
+
+#
+# Imports
+#
+
+import os
+import sys
+
+from multiprocessing.process import Process, current_process, active_children
+
+#
+# Exceptions
+#
+
+class ProcessError(Exception):
+    pass
+
+class BufferTooShort(ProcessError):
+    pass
+
+class TimeoutError(ProcessError):
+    pass
+
+class AuthenticationError(ProcessError):
+    pass
+
+# This is down here because _multiprocessing uses BufferTooShort
+import _multiprocessing
+
+#
+# Definitions not depending on native semaphores
+#
+
+def Manager():
+    '''
+    Returns a manager associated with a running server process
+
+    The managers methods such as `Lock()`, `Condition()` and `Queue()`
+    can be used to create shared objects.
+    '''
+    from multiprocessing.managers import SyncManager
+    m = SyncManager()
+    m.start()
+    return m
+
+def Pipe(duplex=True):
+    '''
+    Returns two connection object connected by a pipe
+    '''
+    from multiprocessing.connection import Pipe
+    return Pipe(duplex)
+
+def cpu_count():
+    '''
+    Returns the number of CPUs in the system
+    '''
+    if sys.platform == 'win32':
+        try:
+            num = int(os.environ['NUMBER_OF_PROCESSORS'])
+        except (ValueError, KeyError):
+            num = 0
+    elif sys.platform == 'darwin':
+        try:
+            num = int(os.popen('sysctl -n hw.ncpu').read())
+        except ValueError:
+            num = 0
+    else:
+        try:
+            num = os.sysconf('SC_NPROCESSORS_ONLN')
+        except (ValueError, OSError, AttributeError):
+            num = 0
+
+    if num >= 1:
+        return num
+    else:
+        raise NotImplementedError('cannot determine number of cpus')
+
+def freeze_support():
+    '''
+    Check whether this is a fake forked process in a frozen executable.
+    If so then run code specified by commandline and exit.
+    '''
+    if sys.platform == 'win32' and getattr(sys, 'frozen', False):
+        from multiprocessing.forking import freeze_support
+        freeze_support()
+
+def get_logger():
+    '''
+    Return package logger -- if it does not already exist then it is created
+    '''
+    from multiprocessing.util import get_logger
+    return get_logger()
+
+def log_to_stderr(level=None):
+    '''
+    Turn on logging and add a handler which prints to stderr
+    '''
+    from multiprocessing.util import log_to_stderr
+    return log_to_stderr(level)
+
+def allow_connection_pickling():
+    '''
+    Install support for sending connections and sockets between processes
+    '''
+    from multiprocessing import reduction
+
+#
+# Definitions depending on native semaphores
+#
+
+def Lock():
+    '''
+    Returns a non-recursive lock object
+    '''
+    from multiprocessing.synchronize import Lock
+    return Lock()
+
+def RLock():
+    '''
+    Returns a recursive lock object
+    '''
+    from multiprocessing.synchronize import RLock
+    return RLock()
+
+def Condition(lock=None):
+    '''
+    Returns a condition object
+    '''
+    from multiprocessing.synchronize import Condition
+    return Condition(lock)
+
+def Semaphore(value=1):
+    '''
+    Returns a semaphore object
+    '''
+    from multiprocessing.synchronize import Semaphore
+    return Semaphore(value)
+
+def BoundedSemaphore(value=1):
+    '''
+    Returns a bounded semaphore object
+    '''
+    from multiprocessing.synchronize import BoundedSemaphore
+    return BoundedSemaphore(value)
+
+def Event():
+    '''
+    Returns an event object
+    '''
+    from multiprocessing.synchronize import Event
+    return Event()
+
+def Queue(maxsize=0):
+    '''
+    Returns a queue object
+    '''
+    from multiprocessing.queues import Queue
+    return Queue(maxsize)
+
+def JoinableQueue(maxsize=0):
+    '''
+    Returns a queue object
+    '''
+    from multiprocessing.queues import JoinableQueue
+    return JoinableQueue(maxsize)
+
+def Pool(processes=None, initializer=None, initargs=()):
+    '''
+    Returns a process pool object
+    '''
+    from multiprocessing.pool import Pool
+    return Pool(processes, initializer, initargs)
+
+def RawValue(typecode_or_type, *args):
+    '''
+    Returns a shared object
+    '''
+    from multiprocessing.sharedctypes import RawValue
+    return RawValue(typecode_or_type, *args)
+
+def RawArray(typecode_or_type, size_or_initializer):
+    '''
+    Returns a shared array
+    '''
+    from multiprocessing.sharedctypes import RawArray
+    return RawArray(typecode_or_type, size_or_initializer)
+
+def Value(typecode_or_type, *args, **kwds):
+    '''
+    Returns a synchronized shared object
+    '''
+    from multiprocessing.sharedctypes import Value
+    return Value(typecode_or_type, *args, **kwds)
+
+def Array(typecode_or_type, size_or_initializer, **kwds):
+    '''
+    Returns a synchronized shared array
+    '''
+    from multiprocessing.sharedctypes import Array
+    return Array(typecode_or_type, size_or_initializer, **kwds)
+
+#
+#
+#
+
+if sys.platform == 'win32':
+
+    def set_executable(executable):
+        '''
+        Sets the path to a python.exe or pythonw.exe binary used to run
+        child processes on Windows instead of sys.executable.
+        Useful for people embedding Python.
+        '''
+        from multiprocessing.forking import set_executable
+        set_executable(executable)
+
+    __all__ += ['set_executable']
index f5a3301fbdb11f7a628dbf000ee2330454035c64..cfb7201b4d7a4805a2be2753edc2fc5030f94dd4 100644 (file)
-#\r
-# A higher level module for using sockets (or Windows named pipes)\r
-#\r
-# multiprocessing/connection.py\r
-#\r
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt\r
-#\r
-\r
-__all__ = [ 'Client', 'Listener', 'Pipe' ]\r
-\r
-import os\r
-import sys\r
-import socket\r
-import time\r
-import tempfile\r
-import itertools\r
-\r
-import _multiprocessing\r
-from multiprocessing import current_process\r
-from multiprocessing.util import get_temp_dir, Finalize, sub_debug, debug\r
-from multiprocessing.forking import duplicate, close\r
-\r
-\r
-#\r
-#\r
-#\r
-\r
-BUFSIZE = 8192\r
-\r
-_mmap_counter = itertools.count()\r
-\r
-default_family = 'AF_INET'\r
-families = ['AF_INET']\r
-\r
-if hasattr(socket, 'AF_UNIX'):\r
-    default_family = 'AF_UNIX'\r
-    families += ['AF_UNIX']\r
-\r
-if sys.platform == 'win32':\r
-    default_family = 'AF_PIPE'\r
-    families += ['AF_PIPE']\r
-\r
-#\r
-#\r
-#\r
-\r
-def arbitrary_address(family):\r
-    '''\r
-    Return an arbitrary free address for the given family\r
-    '''\r
-    if family == 'AF_INET':\r
-        return ('localhost', 0)\r
-    elif family == 'AF_UNIX':\r
-        return tempfile.mktemp(prefix='listener-', dir=get_temp_dir())\r
-    elif family == 'AF_PIPE':\r
-        return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %\r
-                               (os.getpid(), _mmap_counter.next()))\r
-    else:\r
-        raise ValueError('unrecognized family')\r
-\r
-\r
-def address_type(address):\r
-    '''\r
-    Return the types of the address\r
-\r
-    This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'\r
-    '''\r
-    if type(address) == tuple:\r
-        return 'AF_INET'\r
-    elif type(address) is str and address.startswith('\\\\'):\r
-        return 'AF_PIPE'\r
-    elif type(address) is str:\r
-        return 'AF_UNIX'\r
-    else:\r
-        raise ValueError('address type of %r unrecognized' % address)\r
-\r
-#\r
-# Public functions\r
-#\r
-\r
-class Listener(object):\r
-    '''\r
-    Returns a listener object.\r
-\r
-    This is a wrapper for a bound socket which is 'listening' for\r
-    connections, or for a Windows named pipe.\r
-    '''\r
-    def __init__(self, address=None, family=None, backlog=1, authkey=None):\r
-        family = family or (address and address_type(address)) \\r
-                 or default_family\r
-        address = address or arbitrary_address(family)\r
-\r
-        if family == 'AF_PIPE':\r
-            self._listener = PipeListener(address, backlog)\r
-        else:\r
-            self._listener = SocketListener(address, family, backlog)\r
-\r
-        if authkey is not None and not isinstance(authkey, bytes):\r
-            raise TypeError, 'authkey should be a byte string'\r
-\r
-        self._authkey = authkey\r
-\r
-    def accept(self):\r
-        '''\r
-        Accept a connection on the bound socket or named pipe of `self`.\r
-\r
-        Returns a `Connection` object.\r
-        '''\r
-        c = self._listener.accept()\r
-        if self._authkey:\r
-            deliver_challenge(c, self._authkey)\r
-            answer_challenge(c, self._authkey)\r
-        return c\r
-\r
-    def close(self):\r
-        '''\r
-        Close the bound socket or named pipe of `self`.\r
-        '''\r
-        return self._listener.close()\r
-\r
-    address = property(lambda self: self._listener._address)\r
-    last_accepted = property(lambda self: self._listener._last_accepted)\r
-\r
-\r
-def Client(address, family=None, authkey=None):\r
-    '''\r
-    Returns a connection to the address of a `Listener`\r
-    '''\r
-    family = family or address_type(address)\r
-    if family == 'AF_PIPE':\r
-        c = PipeClient(address)\r
-    else:\r
-        c = SocketClient(address)\r
-\r
-    if authkey is not None and not isinstance(authkey, bytes):\r
-        raise TypeError, 'authkey should be a byte string'\r
-\r
-    if authkey is not None:\r
-        answer_challenge(c, authkey)\r
-        deliver_challenge(c, authkey)\r
-\r
-    return c\r
-\r
-\r
-if sys.platform != 'win32':\r
-\r
-    def Pipe(duplex=True):\r
-        '''\r
-        Returns pair of connection objects at either end of a pipe\r
-        '''\r
-        if duplex:\r
-            s1, s2 = socket.socketpair()\r
-            c1 = _multiprocessing.Connection(os.dup(s1.fileno()))\r
-            c2 = _multiprocessing.Connection(os.dup(s2.fileno()))\r
-            s1.close()\r
-            s2.close()\r
-        else:\r
-            fd1, fd2 = os.pipe()\r
-            c1 = _multiprocessing.Connection(fd1, writable=False)\r
-            c2 = _multiprocessing.Connection(fd2, readable=False)\r
-\r
-        return c1, c2\r
-\r
-else:\r
-\r
-    from ._multiprocessing import win32\r
-\r
-    def Pipe(duplex=True):\r
-        '''\r
-        Returns pair of connection objects at either end of a pipe\r
-        '''\r
-        address = arbitrary_address('AF_PIPE')\r
-        if duplex:\r
-            openmode = win32.PIPE_ACCESS_DUPLEX\r
-            access = win32.GENERIC_READ | win32.GENERIC_WRITE\r
-            obsize, ibsize = BUFSIZE, BUFSIZE\r
-        else:\r
-            openmode = win32.PIPE_ACCESS_INBOUND\r
-            access = win32.GENERIC_WRITE\r
-            obsize, ibsize = 0, BUFSIZE\r
-\r
-        h1 = win32.CreateNamedPipe(\r
-            address, openmode,\r
-            win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |\r
-            win32.PIPE_WAIT,\r
-            1, obsize, ibsize, win32.NMPWAIT_WAIT_FOREVER, win32.NULL\r
-            )\r
-        h2 = win32.CreateFile(\r
-            address, access, 0, win32.NULL, win32.OPEN_EXISTING, 0, win32.NULL\r
-            )\r
-        win32.SetNamedPipeHandleState(\r
-            h2, win32.PIPE_READMODE_MESSAGE, None, None\r
-            )\r
-\r
-        try:\r
-            win32.ConnectNamedPipe(h1, win32.NULL)\r
-        except WindowsError, e:\r
-            if e.args[0] != win32.ERROR_PIPE_CONNECTED:\r
-                raise\r
-\r
-        c1 = _multiprocessing.PipeConnection(h1, writable=duplex)\r
-        c2 = _multiprocessing.PipeConnection(h2, readable=duplex)\r
-\r
-        return c1, c2\r
-\r
-#\r
-# Definitions for connections based on sockets\r
-#\r
-\r
-class SocketListener(object):\r
-    '''\r
-    Represtation of a socket which is bound to an address and listening\r
-    '''\r
-    def __init__(self, address, family, backlog=1):\r
-        self._socket = socket.socket(getattr(socket, family))\r
-        self._socket.bind(address)\r
-        self._socket.listen(backlog)\r
-        address = self._socket.getsockname()\r
-        if type(address) is tuple:\r
-            address = (socket.getfqdn(address[0]),) + address[1:]\r
-        self._address = address\r
-        self._family = family\r
-        self._last_accepted = None\r
-\r
-        sub_debug('listener bound to address %r', self._address)\r
-\r
-        if family == 'AF_UNIX':\r
-            self._unlink = Finalize(\r
-                self, os.unlink, args=(self._address,), exitpriority=0\r
-                )\r
-        else:\r
-            self._unlink = None\r
-\r
-    def accept(self):\r
-        s, self._last_accepted = self._socket.accept()\r
-        fd = duplicate(s.fileno())\r
-        conn = _multiprocessing.Connection(fd)\r
-        s.close()\r
-        return conn\r
-\r
-    def close(self):\r
-        self._socket.close()\r
-        if self._unlink is not None:\r
-            self._unlink()\r
-\r
-\r
-def SocketClient(address):\r
-    '''\r
-    Return a connection object connected to the socket given by `address`\r
-    '''\r
-    family = address_type(address)\r
-    s = socket.socket( getattr(socket, family) )\r
-\r
-    while 1:\r
-        try:\r
-            s.connect(address)\r
-        except socket.error, e:\r
-            if e.args[0] != 10061:    # 10061 => connection refused\r
-                debug('failed to connect to address %s', address)\r
-                raise\r
-            time.sleep(0.01)\r
-        else:\r
-            break\r
-    else:\r
-        raise\r
-\r
-    fd = duplicate(s.fileno())\r
-    conn = _multiprocessing.Connection(fd)\r
-    s.close()\r
-    return conn\r
-\r
-#\r
-# Definitions for connections based on named pipes\r
-#\r
-\r
-if sys.platform == 'win32':\r
-\r
-    class PipeListener(object):\r
-        '''\r
-        Representation of a named pipe\r
-        '''\r
-        def __init__(self, address, backlog=None):\r
-            self._address = address\r
-            handle = win32.CreateNamedPipe(\r
-                address, win32.PIPE_ACCESS_DUPLEX,\r
-                win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |\r
-                win32.PIPE_WAIT,\r
-                win32.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,\r
-                win32.NMPWAIT_WAIT_FOREVER, win32.NULL\r
-                )\r
-            self._handle_queue = [handle]\r
-            self._last_accepted = None\r
-\r
-            sub_debug('listener created with address=%r', self._address)\r
-\r
-            self.close = Finalize(\r
-                self, PipeListener._finalize_pipe_listener,\r
-                args=(self._handle_queue, self._address), exitpriority=0\r
-                )\r
-\r
-        def accept(self):\r
-            newhandle = win32.CreateNamedPipe(\r
-                self._address, win32.PIPE_ACCESS_DUPLEX,\r
-                win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |\r
-                win32.PIPE_WAIT,\r
-                win32.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,\r
-                win32.NMPWAIT_WAIT_FOREVER, win32.NULL\r
-                )\r
-            self._handle_queue.append(newhandle)\r
-            handle = self._handle_queue.pop(0)\r
-            try:\r
-                win32.ConnectNamedPipe(handle, win32.NULL)\r
-            except WindowsError, e:\r
-                if e.args[0] != win32.ERROR_PIPE_CONNECTED:\r
-                    raise\r
-            return _multiprocessing.PipeConnection(handle)\r
-\r
-        @staticmethod\r
-        def _finalize_pipe_listener(queue, address):\r
-            sub_debug('closing listener with address=%r', address)\r
-            for handle in queue:\r
-                close(handle)\r
-\r
-    def PipeClient(address):\r
-        '''\r
-        Return a connection object connected to the pipe given by `address`\r
-        '''\r
-        while 1:\r
-            try:\r
-                win32.WaitNamedPipe(address, 1000)\r
-                h = win32.CreateFile(\r
-                    address, win32.GENERIC_READ | win32.GENERIC_WRITE,\r
-                    0, win32.NULL, win32.OPEN_EXISTING, 0, win32.NULL\r
-                    )\r
-            except WindowsError, e:\r
-                if e.args[0] not in (win32.ERROR_SEM_TIMEOUT,\r
-                                     win32.ERROR_PIPE_BUSY):\r
-                    raise\r
-            else:\r
-                break\r
-        else:\r
-            raise\r
-\r
-        win32.SetNamedPipeHandleState(\r
-            h, win32.PIPE_READMODE_MESSAGE, None, None\r
-            )\r
-        return _multiprocessing.PipeConnection(h)\r
-\r
-#\r
-# Authentication stuff\r
-#\r
-\r
-MESSAGE_LENGTH = 20\r
-\r
-CHALLENGE = '#CHALLENGE#'\r
-WELCOME = '#WELCOME#'\r
-FAILURE = '#FAILURE#'\r
-\r
-if sys.version_info >= (3, 0):         # XXX can use bytes literals in 2.6/3.0\r
-    CHALLENGE = CHALLENGE.encode('ascii')\r
-    WELCOME = WELCOME.encode('ascii')\r
-    FAILURE = FAILURE.encode('ascii')\r
-\r
-def deliver_challenge(connection, authkey):\r
-    import hmac\r
-    assert isinstance(authkey, bytes)\r
-    message = os.urandom(MESSAGE_LENGTH)\r
-    connection.send_bytes(CHALLENGE + message)\r
-    digest = hmac.new(authkey, message).digest()\r
-    response = connection.recv_bytes(256)        # reject large message\r
-    if response == digest:\r
-        connection.send_bytes(WELCOME)\r
-    else:\r
-        connection.send_bytes(FAILURE)\r
-        raise AuthenticationError('digest received was wrong')\r
-\r
-def answer_challenge(connection, authkey):\r
-    import hmac\r
-    assert isinstance(authkey, bytes)\r
-    message = connection.recv_bytes(256)         # reject large message\r
-    assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message\r
-    message = message[len(CHALLENGE):]\r
-    digest = hmac.new(authkey, message).digest()\r
-    connection.send_bytes(digest)\r
-    response = connection.recv_bytes(256)        # reject large message\r
-    if response != WELCOME:\r
-        raise AuthenticationError('digest sent was rejected')\r
-\r
-#\r
-# Support for using xmlrpclib for serialization\r
-#\r
-\r
-class ConnectionWrapper(object):\r
-    def __init__(self, conn, dumps, loads):\r
-        self._conn = conn\r
-        self._dumps = dumps\r
-        self._loads = loads\r
-        for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'):\r
-            obj = getattr(conn, attr)\r
-            setattr(self, attr, obj)\r
-    def send(self, obj):\r
-        s = self._dumps(obj)\r
-        self._conn.send_bytes(s)\r
-    def recv(self):\r
-        s = self._conn.recv_bytes()\r
-        return self._loads(s)\r
-\r
-def _xml_dumps(obj):\r
-    return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf8')\r
-\r
-def _xml_loads(s):\r
-    (obj,), method = xmlrpclib.loads(s.decode('utf8'))\r
-    return obj\r
-\r
-class XmlListener(Listener):\r
-    def accept(self):\r
-        global xmlrpclib\r
-        import xmlrpclib\r
-        obj = Listener.accept(self)\r
-        return ConnectionWrapper(obj, _xml_dumps, _xml_loads)\r
-\r
-def XmlClient(*args, **kwds):\r
-    global xmlrpclib\r
-    import xmlrpclib\r
-    return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)\r
+#
+# A higher level module for using sockets (or Windows named pipes)
+#
+# multiprocessing/connection.py
+#
+# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+#
+
+__all__ = [ 'Client', 'Listener', 'Pipe' ]
+
+import os
+import sys
+import socket
+import time
+import tempfile
+import itertools
+
+import _multiprocessing
+from multiprocessing import current_process
+from multiprocessing.util import get_temp_dir, Finalize, sub_debug, debug
+from multiprocessing.forking import duplicate, close
+
+
+#
+#
+#
+
+BUFSIZE = 8192
+
+_mmap_counter = itertools.count()
+
+default_family = 'AF_INET'
+families = ['AF_INET']
+
+if hasattr(socket, 'AF_UNIX'):
+    default_family = 'AF_UNIX'
+    families += ['AF_UNIX']
+
+if sys.platform == 'win32':
+    default_family = 'AF_PIPE'
+    families += ['AF_PIPE']
+
+#
+#
+#
+
+def arbitrary_address(family):
+    '''
+    Return an arbitrary free address for the given family
+    '''
+    if family == 'AF_INET':
+        return ('localhost', 0)
+    elif family == 'AF_UNIX':
+        return tempfile.mktemp(prefix='listener-', dir=get_temp_dir())
+    elif family == 'AF_PIPE':
+        return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
+                               (os.getpid(), _mmap_counter.next()))
+    else:
+        raise ValueError('unrecognized family')
+
+
+def address_type(address):
+    '''
+    Return the types of the address
+
+    This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'
+    '''
+    if type(address) == tuple:
+        return 'AF_INET'
+    elif type(address) is str and address.startswith('\\\\'):
+        return 'AF_PIPE'
+    elif type(address) is str:
+        return 'AF_UNIX'
+    else:
+        raise ValueError('address type of %r unrecognized' % address)
+
+#
+# Public functions
+#
+
+class Listener(object):
+    '''
+    Returns a listener object.
+
+    This is a wrapper for a bound socket which is 'listening' for
+    connections, or for a Windows named pipe.
+    '''
+    def __init__(self, address=None, family=None, backlog=1, authkey=None):
+        family = family or (address and address_type(address)) \
+                 or default_family
+        address = address or arbitrary_address(family)
+
+        if family == 'AF_PIPE':
+            self._listener = PipeListener(address, backlog)
+        else:
+            self._listener = SocketListener(address, family, backlog)
+
+        if authkey is not None and not isinstance(authkey, bytes):
+            raise TypeError, 'authkey should be a byte string'
+
+        self._authkey = authkey
+
+    def accept(self):
+        '''
+        Accept a connection on the bound socket or named pipe of `self`.
+
+        Returns a `Connection` object.
+        '''
+        c = self._listener.accept()
+        if self._authkey:
+            deliver_challenge(c, self._authkey)
+            answer_challenge(c, self._authkey)
+        return c
+
+    def close(self):
+        '''
+        Close the bound socket or named pipe of `self`.
+        '''
+        return self._listener.close()
+
+    address = property(lambda self: self._listener._address)
+    last_accepted = property(lambda self: self._listener._last_accepted)
+
+
+def Client(address, family=None, authkey=None):
+    '''
+    Returns a connection to the address of a `Listener`
+    '''
+    family = family or address_type(address)
+    if family == 'AF_PIPE':
+        c = PipeClient(address)
+    else:
+        c = SocketClient(address)
+
+    if authkey is not None and not isinstance(authkey, bytes):
+        raise TypeError, 'authkey should be a byte string'
+
+    if authkey is not None:
+        answer_challenge(c, authkey)
+        deliver_challenge(c, authkey)
+
+    return c
+
+
+if sys.platform != 'win32':
+
+    def Pipe(duplex=True):
+        '''
+        Returns pair of connection objects at either end of a pipe
+        '''
+        if duplex:
+            s1, s2 = socket.socketpair()
+            c1 = _multiprocessing.Connection(os.dup(s1.fileno()))
+            c2 = _multiprocessing.Connection(os.dup(s2.fileno()))
+            s1.close()
+            s2.close()
+        else:
+            fd1, fd2 = os.pipe()
+            c1 = _multiprocessing.Connection(fd1, writable=False)
+            c2 = _multiprocessing.Connection(fd2, readable=False)
+
+        return c1, c2
+
+else:
+
+    from ._multiprocessing import win32
+
+    def Pipe(duplex=True):
+        '''
+        Returns pair of connection objects at either end of a pipe
+        '''
+        address = arbitrary_address('AF_PIPE')
+        if duplex:
+            openmode = win32.PIPE_ACCESS_DUPLEX
+            access = win32.GENERIC_READ | win32.GENERIC_WRITE
+            obsize, ibsize = BUFSIZE, BUFSIZE
+        else:
+            openmode = win32.PIPE_ACCESS_INBOUND
+            access = win32.GENERIC_WRITE
+            obsize, ibsize = 0, BUFSIZE
+
+        h1 = win32.CreateNamedPipe(
+            address, openmode,
+            win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |
+            win32.PIPE_WAIT,
+            1, obsize, ibsize, win32.NMPWAIT_WAIT_FOREVER, win32.NULL
+            )
+        h2 = win32.CreateFile(
+            address, access, 0, win32.NULL, win32.OPEN_EXISTING, 0, win32.NULL
+            )
+        win32.SetNamedPipeHandleState(
+            h2, win32.PIPE_READMODE_MESSAGE, None, None
+            )
+
+        try:
+            win32.ConnectNamedPipe(h1, win32.NULL)
+        except WindowsError, e:
+            if e.args[0] != win32.ERROR_PIPE_CONNECTED:
+                raise
+
+        c1 = _multiprocessing.PipeConnection(h1, writable=duplex)
+        c2 = _multiprocessing.PipeConnection(h2, readable=duplex)
+
+        return c1, c2
+
+#
+# Definitions for connections based on sockets
+#
+
+class SocketListener(object):
+    '''
+    Represtation of a socket which is bound to an address and listening
+    '''
+    def __init__(self, address, family, backlog=1):
+        self._socket = socket.socket(getattr(socket, family))
+        self._socket.bind(address)
+        self._socket.listen(backlog)
+        address = self._socket.getsockname()
+        if type(address) is tuple:
+            address = (socket.getfqdn(address[0]),) + address[1:]
+        self._address = address
+        self._family = family
+        self._last_accepted = None
+
+        sub_debug('listener bound to address %r', self._address)
+
+        if family == 'AF_UNIX':
+            self._unlink = Finalize(
+                self, os.unlink, args=(self._address,), exitpriority=0
+                )
+        else:
+            self._unlink = None
+
+    def accept(self):
+        s, self._last_accepted = self._socket.accept()
+        fd = duplicate(s.fileno())
+        conn = _multiprocessing.Connection(fd)
+        s.close()
+        return conn
+
+    def close(self):
+        self._socket.close()
+        if self._unlink is not None:
+            self._unlink()
+
+
+def SocketClient(address):
+    '''
+    Return a connection object connected to the socket given by `address`
+    '''
+    family = address_type(address)
+    s = socket.socket( getattr(socket, family) )
+
+    while 1:
+        try:
+            s.connect(address)
+        except socket.error, e:
+            if e.args[0] != 10061:    # 10061 => connection refused
+                debug('failed to connect to address %s', address)
+                raise
+            time.sleep(0.01)
+        else:
+            break
+    else:
+        raise
+
+    fd = duplicate(s.fileno())
+    conn = _multiprocessing.Connection(fd)
+    s.close()
+    return conn
+
+#
+# Definitions for connections based on named pipes
+#
+
+if sys.platform == 'win32':
+
+    class PipeListener(object):
+        '''
+        Representation of a named pipe
+        '''
+        def __init__(self, address, backlog=None):
+            self._address = address
+            handle = win32.CreateNamedPipe(
+                address, win32.PIPE_ACCESS_DUPLEX,
+                win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |
+                win32.PIPE_WAIT,
+                win32.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
+                win32.NMPWAIT_WAIT_FOREVER, win32.NULL
+                )
+            self._handle_queue = [handle]
+            self._last_accepted = None
+
+            sub_debug('listener created with address=%r', self._address)
+
+            self.close = Finalize(
+                self, PipeListener._finalize_pipe_listener,
+                args=(self._handle_queue, self._address), exitpriority=0
+                )
+
+        def accept(self):
+            newhandle = win32.CreateNamedPipe(
+                self._address, win32.PIPE_ACCESS_DUPLEX,
+                win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |
+                win32.PIPE_WAIT,
+                win32.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
+                win32.NMPWAIT_WAIT_FOREVER, win32.NULL
+                )
+            self._handle_queue.append(newhandle)
+            handle = self._handle_queue.pop(0)
+            try:
+                win32.ConnectNamedPipe(handle, win32.NULL)
+            except WindowsError, e:
+                if e.args[0] != win32.ERROR_PIPE_CONNECTED:
+                    raise
+            return _multiprocessing.PipeConnection(handle)
+
+        @staticmethod
+        def _finalize_pipe_listener(queue, address):
+            sub_debug('closing listener with address=%r', address)
+            for handle in queue:
+                close(handle)
+
+    def PipeClient(address):
+        '''
+        Return a connection object connected to the pipe given by `address`
+        '''
+        while 1:
+            try:
+                win32.WaitNamedPipe(address, 1000)
+                h = win32.CreateFile(
+                    address, win32.GENERIC_READ | win32.GENERIC_WRITE,
+                    0, win32.NULL, win32.OPEN_EXISTING, 0, win32.NULL
+                    )
+            except WindowsError, e:
+                if e.args[0] not in (win32.ERROR_SEM_TIMEOUT,
+                                     win32.ERROR_PIPE_BUSY):
+                    raise
+            else:
+                break
+        else:
+            raise
+
+        win32.SetNamedPipeHandleState(
+            h, win32.PIPE_READMODE_MESSAGE, None, None
+            )
+        return _multiprocessing.PipeConnection(h)
+
+#
+# Authentication stuff
+#
+
+MESSAGE_LENGTH = 20
+
+CHALLENGE = '#CHALLENGE#'
+WELCOME = '#WELCOME#'
+FAILURE = '#FAILURE#'
+
+if sys.version_info >= (3, 0):         # XXX can use bytes literals in 2.6/3.0
+    CHALLENGE = CHALLENGE.encode('ascii')
+    WELCOME = WELCOME.encode('ascii')
+    FAILURE = FAILURE.encode('ascii')
+
+def deliver_challenge(connection, authkey):
+    import hmac
+    assert isinstance(authkey, bytes)
+    message = os.urandom(MESSAGE_LENGTH)
+    connection.send_bytes(CHALLENGE + message)
+    digest = hmac.new(authkey, message).digest()
+    response = connection.recv_bytes(256)        # reject large message
+    if response == digest:
+        connection.send_bytes(WELCOME)
+    else:
+        connection.send_bytes(FAILURE)
+        raise AuthenticationError('digest received was wrong')
+
+def answer_challenge(connection, authkey):
+    import hmac
+    assert isinstance(authkey, bytes)
+    message = connection.recv_bytes(256)         # reject large message
+    assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
+    message = message[len(CHALLENGE):]
+    digest = hmac.new(authkey, message).digest()
+    connection.send_bytes(digest)
+    response = connection.recv_bytes(256)        # reject large message
+    if response != WELCOME:
+        raise AuthenticationError('digest sent was rejected')
+
+#
+# Support for using xmlrpclib for serialization
+#
+
+class ConnectionWrapper(object):
+    def __init__(self, conn, dumps, loads):
+        self._conn = conn
+        self._dumps = dumps
+        self._loads = loads
+        for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'):
+            obj = getattr(conn, attr)
+            setattr(self, attr, obj)
+    def send(self, obj):
+        s = self._dumps(obj)
+        self._conn.send_bytes(s)
+    def recv(self):
+        s = self._conn.recv_bytes()
+        return self._loads(s)
+
+def _xml_dumps(obj):
+    return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf8')
+
+def _xml_loads(s):
+    (obj,), method = xmlrpclib.loads(s.decode('utf8'))
+    return obj
+
+class XmlListener(Listener):
+    def accept(self):
+        global xmlrpclib
+        import xmlrpclib
+        obj = Listener.accept(self)
+        return ConnectionWrapper(obj, _xml_dumps, _xml_loads)
+
+def XmlClient(*args, **kwds):
+    global xmlrpclib
+    import xmlrpclib
+    return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)
index 6107f07a41eb45321ab19048c91458661ad6272d..43e3e830556ed07b26c7200735efb29044e36ef1 100644 (file)
-#\r
-# Module for starting a process object using os.fork() or CreateProcess()\r
-#\r
-# multiprocessing/forking.py\r
-#\r
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt\r
-#\r
-\r
-import os\r
-import sys\r
-import signal\r
-\r
-from multiprocessing import util, process\r
-\r
-__all__ = ['Popen', 'assert_spawning', 'exit', 'duplicate', 'close']\r
-\r
-#\r
-# Check that the current thread is spawning a child process\r
-#\r
-\r
-def assert_spawning(self):\r
-    if not Popen.thread_is_spawning():\r
-        raise RuntimeError(\r
-            '%s objects should only be shared between processes'\r
-            ' through inheritance' % type(self).__name__\r
-            )\r
-\r
-#\r
-# Unix\r
-#\r
-\r
-if sys.platform != 'win32':\r
-    import time\r
-\r
-    exit = os._exit\r
-    duplicate = os.dup\r
-    close = os.close\r
-\r
-    #\r
-    # We define a Popen class similar to the one from subprocess, but\r
-    # whose constructor takes a process object as its argument.\r
-    #\r
-\r
-    class Popen(object):\r
-\r
-        def __init__(self, process_obj):\r
-            sys.stdout.flush()\r
-            sys.stderr.flush()\r
-            self.returncode = None\r
-\r
-            self.pid = os.fork()\r
-            if self.pid == 0:\r
-                if 'random' in sys.modules:\r
-                    import random\r
-                    random.seed()\r
-                code = process_obj._bootstrap()\r
-                sys.stdout.flush()\r
-                sys.stderr.flush()\r
-                os._exit(code)\r
-\r
-        def poll(self, flag=os.WNOHANG):\r
-            if self.returncode is None:\r
-                pid, sts = os.waitpid(self.pid, flag)\r
-                if pid == self.pid:\r
-                    if os.WIFSIGNALED(sts):\r
-                        self.returncode = -os.WTERMSIG(sts)\r
-                    else:\r
-                        assert os.WIFEXITED(sts)\r
-                        self.returncode = os.WEXITSTATUS(sts)\r
-            return self.returncode\r
-\r
-        def wait(self, timeout=None):\r
-            if timeout is None:\r
-                return self.poll(0)\r
-            deadline = time.time() + timeout\r
-            delay = 0.0005\r
-            while 1:\r
-                res = self.poll()\r
-                if res is not None:\r
-                    break\r
-                remaining = deadline - time.time()\r
-                if remaining <= 0:\r
-                    break\r
-                delay = min(delay * 2, remaining, 0.05)\r
-                time.sleep(delay)\r
-            return res\r
-\r
-        def terminate(self):\r
-            if self.returncode is None:\r
-                try:\r
-                    os.kill(self.pid, signal.SIGTERM)\r
-                except OSError, e:\r
-                    if self.wait(timeout=0.1) is None:\r
-                        raise\r
-\r
-        @staticmethod\r
-        def thread_is_spawning():\r
-            return False\r
-\r
-#\r
-# Windows\r
-#\r
-\r
-else:\r
-    import thread\r
-    import msvcrt\r
-    import _subprocess\r
-    import copy_reg\r
-    import time\r
-\r
-    from ._multiprocessing import win32, Connection, PipeConnection\r
-    from .util import Finalize\r
-\r
-    try:\r
-        from cPickle import dump, load, HIGHEST_PROTOCOL\r
-    except ImportError:\r
-        from pickle import dump, load, HIGHEST_PROTOCOL\r
-\r
-    #\r
-    #\r
-    #\r
-\r
-    TERMINATE = 0x10000\r
-    WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))\r
-\r
-    exit = win32.ExitProcess\r
-    close = win32.CloseHandle\r
-\r
-    #\r
-    # _python_exe is the assumed path to the python executable.\r
-    # People embedding Python want to modify it.\r
-    #\r
-\r
-    if sys.executable.lower().endswith('pythonservice.exe'):\r
-        _python_exe = os.path.join(sys.exec_prefix, 'python.exe')\r
-    else:\r
-        _python_exe = sys.executable\r
-\r
-    def set_executable(exe):\r
-        global _python_exe\r
-        _python_exe = exe\r
-\r
-    #\r
-    #\r
-    #\r
-\r
-    def duplicate(handle, target_process=None, inheritable=False):\r
-        if target_process is None:\r
-            target_process = _subprocess.GetCurrentProcess()\r
-        return _subprocess.DuplicateHandle(\r
-            _subprocess.GetCurrentProcess(), handle, target_process,\r
-            0, inheritable, _subprocess.DUPLICATE_SAME_ACCESS\r
-            ).Detach()\r
-\r
-    #\r
-    # We define a Popen class similar to the one from subprocess, but\r
-    # whose constructor takes a process object as its argument.\r
-    #\r
-\r
-    class Popen(object):\r
-        '''\r
-        Start a subprocess to run the code of a process object\r
-        '''\r
-        _tls = thread._local()\r
-\r
-        def __init__(self, process_obj):\r
-            # create pipe for communication with child\r
-            rfd, wfd = os.pipe()\r
-\r
-            # get handle for read end of the pipe and make it inheritable\r
-            rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True)\r
-            os.close(rfd)\r
-\r
-            # start process\r
-            cmd = get_command_line() + [rhandle]\r
-            cmd = ' '.join('"%s"' % x for x in cmd)\r
-            hp, ht, pid, tid = _subprocess.CreateProcess(\r
-                _python_exe, cmd, None, None, 1, 0, None, None, None\r
-                )\r
-            ht.Close()\r
-            close(rhandle)\r
-\r
-            # set attributes of self\r
-            self.pid = pid\r
-            self.returncode = None\r
-            self._handle = hp\r
-\r
-            # send information to child\r
-            prep_data = get_preparation_data(process_obj._name)\r
-            to_child = os.fdopen(wfd, 'wb')\r
-            Popen._tls.process_handle = int(hp)\r
-            try:\r
-                dump(prep_data, to_child, HIGHEST_PROTOCOL)\r
-                dump(process_obj, to_child, HIGHEST_PROTOCOL)\r
-            finally:\r
-                del Popen._tls.process_handle\r
-                to_child.close()\r
-\r
-        @staticmethod\r
-        def thread_is_spawning():\r
-            return getattr(Popen._tls, 'process_handle', None) is not None\r
-\r
-        @staticmethod\r
-        def duplicate_for_child(handle):\r
-            return duplicate(handle, Popen._tls.process_handle)\r
-\r
-        def wait(self, timeout=None):\r
-            if self.returncode is None:\r
-                if timeout is None:\r
-                    msecs = _subprocess.INFINITE\r
-                else:\r
-                    msecs = max(0, int(timeout * 1000 + 0.5))\r
-\r
-                res = _subprocess.WaitForSingleObject(int(self._handle), msecs)\r
-                if res == _subprocess.WAIT_OBJECT_0:\r
-                    code = _subprocess.GetExitCodeProcess(self._handle)\r
-                    if code == TERMINATE:\r
-                        code = -signal.SIGTERM\r
-                    self.returncode = code\r
-\r
-            return self.returncode\r
-\r
-        def poll(self):\r
-            return self.wait(timeout=0)\r
-\r
-        def terminate(self):\r
-            if self.returncode is None:\r
-                try:\r
-                    _subprocess.TerminateProcess(int(self._handle), TERMINATE)\r
-                except WindowsError:\r
-                    if self.wait(timeout=0.1) is None:\r
-                        raise\r
-\r
-    #\r
-    #\r
-    #\r
-\r
-    def is_forking(argv):\r
-        '''\r
-        Return whether commandline indicates we are forking\r
-        '''\r
-        if len(argv) >= 2 and argv[1] == '--multiprocessing-fork':\r
-            assert len(argv) == 3\r
-            return True\r
-        else:\r
-            return False\r
-\r
-\r
-    def freeze_support():\r
-        '''\r
-        Run code for process object if this in not the main process\r
-        '''\r
-        if is_forking(sys.argv):\r
-            main()\r
-            sys.exit()\r
-\r
-\r
-    def get_command_line():\r
-        '''\r
-        Returns prefix of command line used for spawning a child process\r
-        '''\r
-        if process.current_process()._identity==() and is_forking(sys.argv):\r
-            raise RuntimeError('''\r
-            Attempt to start a new process before the current process\r
-            has finished its bootstrapping phase.\r
-\r
-            This probably means that you are on Windows and you have\r
-            forgotten to use the proper idiom in the main module:\r
-\r
-                if __name__ == '__main__':\r
-                    freeze_support()\r
-                    ...\r
-\r
-            The "freeze_support()" line can be omitted if the program\r
-            is not going to be frozen to produce a Windows executable.''')\r
-\r
-        if getattr(sys, 'frozen', False):\r
-            return [sys.executable, '--multiprocessing-fork']\r
-        else:\r
-            prog = 'from multiprocessing.forking import main; main()'\r
-            return [_python_exe, '-c', prog, '--multiprocessing-fork']\r
-\r
-\r
-    def main():\r
-        '''\r
-        Run code specifed by data received over pipe\r
-        '''\r
-        assert is_forking(sys.argv)\r
-\r
-        handle = int(sys.argv[-1])\r
-        fd = msvcrt.open_osfhandle(handle, os.O_RDONLY)\r
-        from_parent = os.fdopen(fd, 'rb')\r
-\r
-        process.current_process()._inheriting = True\r
-        preparation_data = load(from_parent)\r
-        prepare(preparation_data)\r
-        self = load(from_parent)\r
-        process.current_process()._inheriting = False\r
-\r
-        from_parent.close()\r
-\r
-        exitcode = self._bootstrap()\r
-        exit(exitcode)\r
-\r
-\r
-    def get_preparation_data(name):\r
-        '''\r
-        Return info about parent needed by child to unpickle process object\r
-        '''\r
-        from .util import _logger, _log_to_stderr\r
-\r
-        d = dict(\r
-            name=name,\r
-            sys_path=sys.path,\r
-            sys_argv=sys.argv,\r
-            log_to_stderr=_log_to_stderr,\r
-            orig_dir=process.ORIGINAL_DIR,\r
-            authkey=process.current_process().get_authkey(),\r
-            )\r
-\r
-        if _logger is not None:\r
-            d['log_level'] = _logger.getEffectiveLevel()\r
-\r
-        if not WINEXE:\r
-            main_path = getattr(sys.modules['__main__'], '__file__', None)\r
-            if not main_path and sys.argv[0] not in ('', '-c'):\r
-                main_path = sys.argv[0]\r
-            if main_path is not None:\r
-                if not os.path.isabs(main_path) and \\r
-                                          process.ORIGINAL_DIR is not None:\r
-                    main_path = os.path.join(process.ORIGINAL_DIR, main_path)\r
-                d['main_path'] = os.path.normpath(main_path)\r
-\r
-        return d\r
-\r
-    #\r
-    # Make (Pipe)Connection picklable\r
-    #\r
-\r
-    def reduce_connection(conn):\r
-        if not Popen.thread_is_spawning():\r
-            raise RuntimeError(\r
-                'By default %s objects can only be shared between processes\n'\r
-                'using inheritance' % type(conn).__name__\r
-                )\r
-        return type(conn), (Popen.duplicate_for_child(conn.fileno()),\r
-                            conn.readable, conn.writable)\r
-\r
-    copy_reg.pickle(Connection, reduce_connection)\r
-    copy_reg.pickle(PipeConnection, reduce_connection)\r
-\r
-\r
-#\r
-# Prepare current process\r
-#\r
-\r
-old_main_modules = []\r
-\r
-def prepare(data):\r
-    '''\r
-    Try to get current process ready to unpickle process object\r
-    '''\r
-    old_main_modules.append(sys.modules['__main__'])\r
-\r
-    if 'name' in data:\r
-        process.current_process().set_name(data['name'])\r
-\r
-    if 'authkey' in data:\r
-        process.current_process()._authkey = data['authkey']\r
-\r
-    if 'log_to_stderr' in data and data['log_to_stderr']:\r
-        util.log_to_stderr()\r
-\r
-    if 'log_level' in data:\r
-        util.get_logger().setLevel(data['log_level'])\r
-\r
-    if 'sys_path' in data:\r
-        sys.path = data['sys_path']\r
-\r
-    if 'sys_argv' in data:\r
-        sys.argv = data['sys_argv']\r
-\r
-    if 'dir' in data:\r
-        os.chdir(data['dir'])\r
-\r
-    if 'orig_dir' in data:\r
-        process.ORIGINAL_DIR = data['orig_dir']\r
-\r
-    if 'main_path' in data:\r
-        main_path = data['main_path']\r
-        main_name = os.path.splitext(os.path.basename(main_path))[0]\r
-        if main_name == '__init__':\r
-            main_name = os.path.basename(os.path.dirname(main_path))\r
-\r
-        if main_name != 'ipython':\r
-            import imp\r
-\r
-            if main_path is None:\r
-                dirs = None\r
-            elif os.path.basename(main_path).startswith('__init__.py'):\r
-                dirs = [os.path.dirname(os.path.dirname(main_path))]\r
-            else:\r
-                dirs = [os.path.dirname(main_path)]\r
-\r
-            assert main_name not in sys.modules, main_name\r
-            file, path_name, etc = imp.find_module(main_name, dirs)\r
-            try:\r
-                # We would like to do "imp.load_module('__main__', ...)"\r
-                # here.  However, that would cause 'if __name__ ==\r
-                # "__main__"' clauses to be executed.\r
-                main_module = imp.load_module(\r
-                    '__parents_main__', file, path_name, etc\r
-                    )\r
-            finally:\r
-                if file:\r
-                    file.close()\r
-\r
-            sys.modules['__main__'] = main_module\r
-            main_module.__name__ = '__main__'\r
-\r
-            # Try to make the potentially picklable objects in\r
-            # sys.modules['__main__'] realize they are in the main\r
-            # module -- somewhat ugly.\r
-            for obj in main_module.__dict__.values():\r
-                try:\r
-                    if obj.__module__ == '__parents_main__':\r
-                        obj.__module__ = '__main__'\r
-                except Exception:\r
-                    pass\r
+#
+# Module for starting a process object using os.fork() or CreateProcess()
+#
+# multiprocessing/forking.py
+#
+# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+#
+
+import os
+import sys
+import signal
+
+from multiprocessing import util, process
+
+__all__ = ['Popen', 'assert_spawning', 'exit', 'duplicate', 'close']
+
+#
+# Check that the current thread is spawning a child process
+#
+
+def assert_spawning(self):
+    if not Popen.thread_is_spawning():
+        raise RuntimeError(
+            '%s objects should only be shared between processes'
+            ' through inheritance' % type(self).__name__
+            )
+
+#
+# Unix
+#
+
+if sys.platform != 'win32':
+    import time
+
+    exit = os._exit
+    duplicate = os.dup
+    close = os.close
+
+    #
+    # We define a Popen class similar to the one from subprocess, but
+    # whose constructor takes a process object as its argument.
+    #
+
+    class Popen(object):
+
+        def __init__(self, process_obj):
+            sys.stdout.flush()
+            sys.stderr.flush()
+            self.returncode = None
+
+            self.pid = os.fork()
+            if self.pid == 0:
+                if 'random' in sys.modules:
+                    import random
+                    random.seed()
+                code = process_obj._bootstrap()
+                sys.stdout.flush()
+                sys.stderr.flush()
+                os._exit(code)
+
+        def poll(self, flag=os.WNOHANG):
+            if self.returncode is None:
+                pid, sts = os.waitpid(self.pid, flag)
+                if pid == self.pid:
+                    if os.WIFSIGNALED(sts):
+                        self.returncode = -os.WTERMSIG(sts)
+                    else:
+                        assert os.WIFEXITED(sts)
+                        self.returncode = os.WEXITSTATUS(sts)
+            return self.returncode
+
+        def wait(self, timeout=None):
+            if timeout is None:
+                return self.poll(0)
+            deadline = time.time() + timeout
+            delay = 0.0005
+            while 1:
+                res = self.poll()
+                if res is not None:
+                    break
+                remaining = deadline - time.time()
+                if remaining <= 0:
+                    break
+                delay = min(delay * 2, remaining, 0.05)
+                time.sleep(delay)
+            return res
+
+        def terminate(self):
+            if self.returncode is None:
+                try:
+                    os.kill(self.pid, signal.SIGTERM)
+                except OSError, e:
+                    if self.wait(timeout=0.1) is None:
+                        raise
+
+        @staticmethod
+        def thread_is_spawning():
+            return False
+
+#
+# Windows
+#
+
+else:
+    import thread
+    import msvcrt
+    import _subprocess
+    import copy_reg
+    import time
+
+    from ._multiprocessing import win32, Connection, PipeConnection
+    from .util import Finalize
+
+    try:
+        from cPickle import dump, load, HIGHEST_PROTOCOL
+    except ImportError:
+        from pickle import dump, load, HIGHEST_PROTOCOL
+
+    #
+    #
+    #
+
+    TERMINATE = 0x10000
+    WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
+
+    exit = win32.ExitProcess
+    close = win32.CloseHandle
+
+    #
+    # _python_exe is the assumed path to the python executable.
+    # People embedding Python want to modify it.
+    #
+
+    if sys.executable.lower().endswith('pythonservice.exe'):
+        _python_exe = os.path.join(sys.exec_prefix, 'python.exe')
+    else:
+        _python_exe = sys.executable
+
+    def set_executable(exe):
+        global _python_exe
+        _python_exe = exe
+
+    #
+    #
+    #
+
+    def duplicate(handle, target_process=None, inheritable=False):
+        if target_process is None:
+            target_process = _subprocess.GetCurrentProcess()
+        return _subprocess.DuplicateHandle(
+            _subprocess.GetCurrentProcess(), handle, target_process,
+            0, inheritable, _subprocess.DUPLICATE_SAME_ACCESS
+            ).Detach()
+
+    #
+    # We define a Popen class similar to the one from subprocess, but
+    # whose constructor takes a process object as its argument.
+    #
+
+    class Popen(object):
+        '''
+        Start a subprocess to run the code of a process object
+        '''
+        _tls = thread._local()
+
+        def __init__(self, process_obj):
+            # create pipe for communication with child
+            rfd, wfd = os.pipe()
+
+            # get handle for read end of the pipe and make it inheritable
+            rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True)
+            os.close(rfd)
+
+            # start process
+            cmd = get_command_line() + [rhandle]
+            cmd = ' '.join('"%s"' % x for x in cmd)
+            hp, ht, pid, tid = _subprocess.CreateProcess(
+                _python_exe, cmd, None, None, 1, 0, None, None, None
+                )
+            ht.Close()
+            close(rhandle)
+
+            # set attributes of self
+            self.pid = pid
+            self.returncode = None
+            self._handle = hp
+
+            # send information to child
+            prep_data = get_preparation_data(process_obj._name)
+            to_child = os.fdopen(wfd, 'wb')
+            Popen._tls.process_handle = int(hp)
+            try:
+                dump(prep_data, to_child, HIGHEST_PROTOCOL)
+                dump(process_obj, to_child, HIGHEST_PROTOCOL)
+            finally:
+                del Popen._tls.process_handle
+                to_child.close()
+
+        @staticmethod
+        def thread_is_spawning():
+            return getattr(Popen._tls, 'process_handle', None) is not None
+
+        @staticmethod
+        def duplicate_for_child(handle):
+            return duplicate(handle, Popen._tls.process_handle)
+
+        def wait(self, timeout=None):
+            if self.returncode is None:
+                if timeout is None:
+                    msecs = _subprocess.INFINITE
+                else:
+                    msecs = max(0, int(timeout * 1000 + 0.5))
+
+                res = _subprocess.WaitForSingleObject(int(self._handle), msecs)
+                if res == _subprocess.WAIT_OBJECT_0:
+                    code = _subprocess.GetExitCodeProcess(self._handle)
+                    if code == TERMINATE:
+                        code = -signal.SIGTERM
+                    self.returncode = code
+
+            return self.returncode
+
+        def poll(self):
+            return self.wait(timeout=0)
+
+        def terminate(self):
+            if self.returncode is None:
+                try:
+                    _subprocess.TerminateProcess(int(self._handle), TERMINATE)
+                except WindowsError:
+                    if self.wait(timeout=0.1) is None:
+                        raise
+
+    #
+    #
+    #
+
+    def is_forking(argv):
+        '''
+        Return whether commandline indicates we are forking
+        '''
+        if len(argv) >= 2 and argv[1] == '--multiprocessing-fork':
+            assert len(argv) == 3
+            return True
+        else:
+            return False
+
+
+    def freeze_support():
+        '''
+        Run code for process object if this in not the main process
+        '''
+        if is_forking(sys.argv):
+            main()
+            sys.exit()
+
+
+    def get_command_line():
+        '''
+        Returns prefix of command line used for spawning a child process
+        '''
+        if process.current_process()._identity==() and is_forking(sys.argv):
+            raise RuntimeError('''
+            Attempt to start a new process before the current process
+            has finished its bootstrapping phase.
+
+            This probably means that you are on Windows and you have
+            forgotten to use the proper idiom in the main module:
+
+                if __name__ == '__main__':
+                    freeze_support()
+                    ...
+
+            The "freeze_support()" line can be omitted if the program
+            is not going to be frozen to produce a Windows executable.''')
+
+        if getattr(sys, 'frozen', False):
+            return [sys.executable, '--multiprocessing-fork']
+        else:
+            prog = 'from multiprocessing.forking import main; main()'
+            return [_python_exe, '-c', prog, '--multiprocessing-fork']
+
+
+    def main():
+        '''
+        Run code specifed by data received over pipe
+        '''
+        assert is_forking(sys.argv)
+
+        handle = int(sys.argv[-1])
+        fd = msvcrt.open_osfhandle(handle, os.O_RDONLY)
+        from_parent = os.fdopen(fd, 'rb')
+
+        process.current_process()._inheriting = True
+        preparation_data = load(from_parent)
+        prepare(preparation_data)
+        self = load(from_parent)
+        process.current_process()._inheriting = False
+
+        from_parent.close()
+
+        exitcode = self._bootstrap()
+        exit(exitcode)
+
+
+    def get_preparation_data(name):
+        '''
+        Return info about parent needed by child to unpickle process object
+        '''
+        from .util import _logger, _log_to_stderr
+
+        d = dict(
+            name=name,
+            sys_path=sys.path,
+            sys_argv=sys.argv,
+            log_to_stderr=_log_to_stderr,
+            orig_dir=process.ORIGINAL_DIR,
+            authkey=process.current_process().get_authkey(),
+            )
+
+        if _logger is not None:
+            d['log_level'] = _logger.getEffectiveLevel()
+
+        if not WINEXE:
+            main_path = getattr(sys.modules['__main__'], '__file__', None)
+            if not main_path and sys.argv[0] not in ('', '-c'):
+                main_path = sys.argv[0]
+            if main_path is not None:
+                if not os.path.isabs(main_path) and \
+                                          process.ORIGINAL_DIR is not None:
+                    main_path = os.path.join(process.ORIGINAL_DIR, main_path)
+                d['main_path'] = os.path.normpath(main_path)
+
+        return d
+
+    #
+    # Make (Pipe)Connection picklable
+    #
+
+    def reduce_connection(conn):
+        if not Popen.thread_is_spawning():
+            raise RuntimeError(
+                'By default %s objects can only be shared between processes\n'
+                'using inheritance' % type(conn).__name__
+                )
+        return type(conn), (Popen.duplicate_for_child(conn.fileno()),
+                            conn.readable, conn.writable)
+
+    copy_reg.pickle(Connection, reduce_connection)
+    copy_reg.pickle(PipeConnection, reduce_connection)
+
+
+#
+# Prepare current process
+#
+
+old_main_modules = []
+
+def prepare(data):
+    '''
+    Try to get current process ready to unpickle process object
+    '''
+    old_main_modules.append(sys.modules['__main__'])
+
+    if 'name' in data:
+        process.current_process().set_name(data['name'])
+
+    if 'authkey' in data:
+        process.current_process()._authkey = data['authkey']
+
+    if 'log_to_stderr' in data and data['log_to_stderr']:
+        util.log_to_stderr()
+
+    if 'log_level' in data:
+        util.get_logger().setLevel(data['log_level'])
+
+    if 'sys_path' in data:
+        sys.path = data['sys_path']
+
+    if 'sys_argv' in data:
+        sys.argv = data['sys_argv']
+
+    if 'dir' in data:
+        os.chdir(data['dir'])
+
+    if 'orig_dir' in data:
+        process.ORIGINAL_DIR = data['orig_dir']
+
+    if 'main_path' in data:
+        main_path = data['main_path']
+        main_name = os.path.splitext(os.path.basename(main_path))[0]
+        if main_name == '__init__':
+            main_name = os.path.basename(os.path.dirname(main_path))
+
+        if main_name != 'ipython':
+            import imp
+
+            if main_path is None:
+                dirs = None
+            elif os.path.basename(main_path).startswith('__init__.py'):
+                dirs = [os.path.dirname(os.path.dirname(main_path))]
+            else:
+                dirs = [os.path.dirname(main_path)]
+
+            assert main_name not in sys.modules, main_name
+            file, path_name, etc = imp.find_module(main_name, dirs)
+            try:
+                # We would like to do "imp.load_module('__main__', ...)"
+                # here.  However, that would cause 'if __name__ ==
+                # "__main__"' clauses to be executed.
+                main_module = imp.load_module(
+                    '__parents_main__', file, path_name, etc
+                    )
+            finally:
+                if file:
+                    file.close()
+
+            sys.modules['__main__'] = main_module
+            main_module.__name__ = '__main__'
+
+            # Try to make the potentially picklable objects in
+            # sys.modules['__main__'] realize they are in the main
+            # module -- somewhat ugly.
+            for obj in main_module.__dict__.values():
+                try:
+                    if obj.__module__ == '__parents_main__':
+                        obj.__module__ = '__main__'
+                except Exception:
+                    pass
index f6b34042f8e6ac9926f908d12c5bf8b7369c6afd..7e596ca70fa78e67576e38534f265c9b94112ba7 100644 (file)
-#\r
-# Module which supports allocation of memory from an mmap\r
-#\r
-# multiprocessing/heap.py\r
-#\r
-# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt\r
-#\r
-\r
-import bisect\r
-import mmap\r
-import tempfile\r
-import os\r
-import sys\r
-import threading\r
-import itertools\r
-\r
-import _multiprocessing\r
-from multiprocessing.util import Finalize, info\r
-from multiprocessing.forking import assert_spawning\r
-\r
-__all__ = ['BufferWrapper']\r
-\r
-#\r
-# Inheirtable class which wraps an mmap, and from which blocks can be allocated\r
-#\r
-\r
-if sys.platform == 'win32':\r
-\r
-    from ._multiprocessing import win32\r
-\r
-    class Arena(object):\r
-\r
-        _counter = itertools.count()\r
-\r
-        def __init__(self, size):\r
-            self.size = size\r
-            self.name = 'pym-%d-%d' % (os.getpid(), Arena._counter.next())\r
-            self.buffer = mmap.mmap(-1, self.size, tagname=self.name)\r
-            assert win32.GetLastError() == 0, 'tagname already in use'\r
-            self._state = (self.size, self.name)\r
-\r
-        def __getstate__(self):\r
-            assert_spawning(self)\r
-            return self._state\r
-\r
-        def __setstate__(self, state):\r
-            self.size, self.name = self._state = state\r
-            self.buffer = mmap.mmap(-1, self.size, tagname=self.name)\r
-            assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS\r
-\r
-else:\r
-\r
-    class Arena(object):\r
-\r
-        def __init__(self, size):\r
-            self.buffer = mmap.mmap(-1, size)\r
-            self.size = size\r
-            self.name = None\r
-\r
-#\r
-# Class allowing allocation of chunks of memory from arenas\r
-#\r
-\r
-class Heap(object):\r
-\r
-    _alignment = 8\r
-\r
-    def __init__(self, size=mmap.PAGESIZE):\r
-        self._lastpid = os.getpid()\r
-        self._lock = threading.Lock()\r
-        self._size = size\r
-        self._lengths = []\r
-        self._len_to_seq = {}\r
-        self._start_to_block = {}\r
-        self._stop_to_block = {}\r
-        self._allocated_blocks = set()\r
-        self._arenas = []\r
-\r
-    @staticmethod\r
-    def _roundup(n, alignment):\r
-        # alignment must be a power of 2\r
-        mask = alignment - 1\r
-        return (n + mask) & ~mask\r
-\r
-    def _malloc(self, size):\r
-        # returns a large enough block -- it might be much larger\r
-        i = bisect.bisect_left(self._lengths, size)\r
-        if i == len(self._lengths):\r
-            length = self._roundup(max(self._size, size), mmap.PAGESIZE)\r
-            self._size *= 2\r
-            info('allocating a new mmap of length %d', length)\r
-            arena = Arena(length)\r
-            self._arenas.append(arena)\r
-            return (arena, 0, length)\r
-        else:\r
-            length = self._lengths[i]\r
-            seq = self._len_to_seq[length]\r
-            block = seq.pop()\r
-            if not seq:\r
-                del self._len_to_seq[length], self._lengths[i]\r
-\r
-        (arena, start, stop) = block\r
-        del self._start_to_block[(arena, start)]\r
-        del self._stop_to_block[(arena, stop)]\r
-        return block\r
-\r
-    def _free(self, block):\r
-        # free location and try to merge with neighbours\r
-        (arena, start, stop) = block\r
-\r
-        try:\r
-            prev_block = self._stop_to_block[(arena, start)]\r
-        except KeyError:\r
-            pass\r
-        else:\r
-            start, _ = self._absorb(prev_block)\r
-\r
-        try:\r
-            next_block = self._start_to_block[(arena, stop)]\r
-        except KeyError:\r
-            pass\r
-        else:\r
-            _, stop = self._absorb(next_block)\r
-\r
-        block = (arena, start, stop)\r
-        length = stop - start\r
-\r
-        try:\r
-            self._len_to_seq[length].append(block)\r
-        except KeyError:\r
-            self._len_to_seq[length] = [block]\r
-            bisect.insort(self._lengths, length)\r
-\r
-        self._start_to_block[(arena, start)] = block\r
-        self._stop_to_block[(arena, stop)] = block\r
-\r
-    def _absorb(self, block):\r
-        # deregister this block so it can be merged with a neighbour\r
-        (arena, start, stop) = block\r
-        del self._start_to_block[(arena, start)]\r
-        del self._stop_to_block[(arena, stop)]\r
-\r
-        length = stop - start\r
-        seq = self._len_to_seq[length]\r
-        seq.remove(block)\r
-        if not seq:\r
-            del self._len_to_seq[length]\r
-            self._lengths.remove(length)\r
-\r
-        return start, stop\r
-\r
-    def free(self, block):\r
-        # free a block returned by malloc()\r
-        assert os.getpid() == self._lastpid\r
-        self._lock.acquire()\r
-        try:\r
-            self._allocated_blocks.remove(block)\r
-            self._free(block)\r
-        finally:\r
-            self._lock.release()\r
-\r
-    def malloc(self, size):\r
-        # return a block of right size (possibly rounded up)\r
-        assert 0 <= size < sys.maxint\r
-        if os.getpid() != self._lastpid:\r
-            self.__init__()                     # reinitialize after fork\r
-        self._lock.acquire()\r
-        try:\r
-            size = self._roundup(max(size,1), self._alignment)\r
-            (arena, start, stop) = self._malloc(size)\r
-            new_stop = start + size\r
-            if new_stop < stop:\r
-                self._free((arena, new_stop, stop))\r
-            block = (arena, start, new_stop)\r
-            self._allocated_blocks.add(block)\r
-            return block\r
-        finally:\r
-            self._lock.release()\r
-\r
-#\r
-# Class representing a chunk of an mmap -- can be inherited\r
-#\r
-\r
-class BufferWrapper(object):\r
-\r
-    _heap = Heap()\r
-\r
-    def __init__(self, size):\r
-        assert 0 <= size < sys.maxint\r
-        block = BufferWrapper._heap.malloc(size)\r
-        self._state = (block, size)\r
-        Finalize(self, BufferWrapper._heap.free, args=(block,))\r
-\r
-    def get_address(self):\r
-        (arena, start, stop), size = self._state\r
-        address, length = _multiprocessing.address_of_buffer(arena.buffer)\r
-        assert size <= length\r
-        return address + start\r
-\r
-    def get_size(self):\r
-        return self._state[1]\r
+#
+# Module which supports allocation of memory from an mmap
+#
+# multiprocessing/heap.py
+#
+# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt
+#
+
+import bisect
+import mmap
+import tempfile
+import os
+import sys
+import threading
+import itertools
+
+import _multiprocessing
+from multiprocessing.util import Finalize, info
+from multiprocessing.forking import assert_spawning
+
+__all__ = ['BufferWrapper']
+
+#
+# Inheirtable class which wraps an mmap, and from which blocks can be allocated
+#
+
+if sys.platform == 'win32':
+
+    from ._multiprocessing import win32
+
+    class Arena(object):
+
+        _counter = itertools.count()
+
+        def __init__(self, size):
+            self.size = size
+            self.name = 'pym-%d-%d' % (os.getpid(), Arena._counter.next())
+            self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
+            assert win32.GetLastError() == 0, 'tagname already in use'
+            self._state = (self.size, self.name)
+
+        def __getstate__(self):
+            assert_spawning(self)
+            return self._state
+
+        def __setstate__(self, state):
+            self.size, self.name = self._state = state
+            self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
+            assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
+
+else:
+
+    class Arena(object):
+
+        def __init__(self, size):
+            self.buffer = mmap.mmap(-1, size)
+            self.size = size
+            self.name = None
+
+#
+# Class allowing allocation of chunks of memory from arenas
+#
+
+class Heap(object):
+
+    _alignment = 8
+
+    def __init__(self, size=mmap.PAGESIZE):
+        self._lastpid = os.getpid()
+        self._lock = threading.Lock()
+        self._size = size
+        self._lengths = []
+        self._len_to_seq = {}
+        self._start_to_block = {}
+        self._stop_to_block = {}
+        self._allocated_blocks = set()
+        self._arenas = []
+
+    @staticmethod
+    def _roundup(n, alignment):
+        # alignment must be a power of 2
+        mask = alignment - 1
+        return (n + mask) & ~mask
+
+    def _malloc(self, size):
+        # returns a large enough block -- it might be much larger
+        i = bisect.bisect_left(self._lengths, size)
+        if i == len(self._lengths):
+            length = self._roundup(max(self._size, size), mmap.PAGESIZE)
+            self._size *= 2
+            info('allocating a new mmap of length %d', length)
+            arena = Arena(length)
+            self._arenas.append(arena)
+            return (arena, 0, length)
+        else:
+            length = self._lengths[i]
+            seq = self._len_to_seq[length]
+            block = seq.pop()
+            if not seq:
+                del self._len_to_seq[length], self._lengths[i]
+
+        (arena, start, stop) = block
+        del self._start_to_block[(arena, start)]
+        del self._stop_to_block[(arena, stop)]
+        return block
+
+    def _free(self, block):
+        # free location and try to merge with neighbours
+        (arena, start, stop) = block
+
+        try:
+            prev_block = self._stop_to_block[(arena, start)]
+        except KeyError:
+            pass
+        else:
+            start, _ = self._absorb(prev_block)
+
+        try:
+            next_block = self._start_to_block[(arena, stop)]
+        except KeyError:
+            pass
+        else:
+            _, stop = self._absorb(next_block)
+
+        block = (arena, start, stop)
+        length = stop - start
+
+        try:
+            self._len_to_seq[length].append(block)
+        except KeyError:
+            self._len_to_seq[length] = [block]
+            bisect.insort(self._lengths, length)
+
+        self._start_to_block[(arena, start)] = block
+        self._stop_to_block[(arena, stop)] = block
+
+    def _absorb(self, block):
+        # deregister this block so it can be merged with a neighbour
+        (arena, start, stop) = block
+        del self._start_to_block[(arena, start)]
+        del self._stop_to_block[(arena, stop)]
+
+        length = stop - start
+        seq = self._len_to_seq[length]
+        seq.remove(block)
+        if not seq:
+            del self._len_to_seq[length]
+            self._lengths.remove(length)
+
+        return start, stop
+
+    def free(self, block):
+        # free a block returned by malloc()
+        assert os.getpid() == self._lastpid
+        self._lock.acquire()
+        try:
+            self._allocated_blocks.remove(block)
+            self._free(block)
+        finally:
+            self._lock.release()
+
+    def malloc(self, size):
+        # return a block of right size (possibly rounded up)
+        assert 0 <= size < sys.maxint
+        if os.getpid() != self._lastpid:
+            self.__init__()                     # reinitialize after fork
+        self._lock.acquire()
+        try:
+            size = self._roundup(max(size,1), self._alignment)
+            (arena, start, stop) = self._malloc(size)
+            new_stop = start + size
+            if new_stop < stop:
+                self._free((arena, new_stop, stop))
+            block = (arena, start, new_stop)
+            self._allocated_blocks.add(block)
+            return block
+        finally:
+            self._lock.release()
+
+#
+# Class representing a chunk of an mmap -- can be inherited
+#
+
+class BufferWrapper(object):
+
+    _heap = Heap()
+
+    def __init__(self, size):
+        assert 0 <= size < sys.maxint
+        block = BufferWrapper._heap.malloc(size)
+        self._state = (block, size)
+        Finalize(self, BufferWrapper._heap.free, args=(block,))
+
+    def get_address(self):
+        (arena, start, stop), size = self._state
+        address, length = _multiprocessing.address_of_buffer(arena.buffer)
+        assert size <= length
+        return address + start
+
+    def get_size(self):
+        return self._state[1]
index 6c1d912c3cab31f446de2c3ceb01954772f17937..fb705cb690a66435bd1d70e6953d47d6f24ad452 100644 (file)
-#\r
-# Module providing the `SyncManager` class for dealing\r
-# with shared objects\r
-#\r
-# multiprocessing/managers.py\r
-#\r
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt\r
-#\r
-\r
-__all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ]\r
-\r
-#\r
-# Imports\r
-#\r
-\r
-import os\r
-import sys\r
-import weakref\r
-import threading\r
-import array\r
-import copy_reg\r
-import Queue\r
-\r
-from traceback import format_exc\r
-from multiprocessing import Process, current_process, active_children, Pool, util, connection\r
-from multiprocessing.process import AuthenticationString\r
-from multiprocessing.forking import exit, Popen, assert_spawning\r
-from multiprocessing.util import Finalize, info\r
-\r
-try:\r
-    from cPickle import PicklingError\r
-except ImportError:\r
-    from pickle import PicklingError\r
-\r
-#\r
-#\r
-#\r
-\r
-try:\r
-    bytes\r
-except NameError:\r
-    bytes = str                  # XXX not needed in Py2.6 and Py3.0\r
-\r
-#\r
-# Register some things for pickling\r
-#\r
-\r
-def reduce_array(a):\r
-    return array.array, (a.typecode, a.tostring())\r
-copy_reg.pickle(array.array, reduce_array)\r
-\r
-view_types = [type(getattr({}, name)()) for name in ('items','keys','values')]\r
-if view_types[0] is not list:       # XXX only needed in Py3.0\r
-    def rebuild_as_list(obj):\r
-        return list, (list(obj),)\r
-    for view_type in view_types:\r
-        copy_reg.pickle(view_type, rebuild_as_list)\r
-\r
-#\r
-# Type for identifying shared objects\r
-#\r
-\r
-class Token(object):\r
-    '''\r
-    Type to uniquely indentify a shared object\r
-    '''\r
-    __slots__ = ('typeid', 'address', 'id')\r
-\r
-    def __init__(self, typeid, address, id):\r
-        (self.typeid, self.address, self.id) = (typeid, address, id)\r
-\r
-    def __getstate__(self):\r
-        return (self.typeid, self.address, self.id)\r
-\r
-    def __setstate__(self, state):\r
-        (self.typeid, self.address, self.id) = state\r
-\r
-    def __repr__(self):\r
-        return 'Token(typeid=%r, address=%r, id=%r)' % \\r
-               (self.typeid, self.address, self.id)\r
-\r
-#\r
-# Function for communication with a manager's server process\r
-#\r
-\r
-def dispatch(c, id, methodname, args=(), kwds={}):\r
-    '''\r
-    Send a message to manager using connection `c` and return response\r
-    '''\r
-    c.send((id, methodname, args, kwds))\r
-    kind, result = c.recv()\r
-    if kind == '#RETURN':\r
-        return result\r
-    raise convert_to_error(kind, result)\r
-\r
-def convert_to_error(kind, result):\r
-    if kind == '#ERROR':\r
-        return result\r
-    elif kind == '#TRACEBACK':\r
-        assert type(result) is str\r
-        return  RemoteError(result)\r
-    elif kind == '#UNSERIALIZABLE':\r
-        assert type(result) is str\r
-        return RemoteError('Unserializable message: %s\n' % result)\r
-    else:\r
-        return ValueError('Unrecognized message type')\r
-\r
-class RemoteError(Exception):\r
-    def __str__(self):\r
-        return ('\n' + '-'*75 + '\n' + str(self.args[0]) + '-'*75)\r
-\r
-#\r
-# Functions for finding the method names of an object\r
-#\r
-\r
-def all_methods(obj):\r
-    '''\r
-    Return a list of names of methods of `obj`\r
-    '''\r
-    temp = []\r
-    for name in dir(obj):\r
-        func = getattr(obj, name)\r
-        if hasattr(func, '__call__'):\r
-            temp.append(name)\r
-    return temp\r
-\r
-def public_methods(obj):\r
-    '''\r
-    Return a list of names of methods of `obj` which do not start with '_'\r
-    '''\r
-    return [name for name in all_methods(obj) if name[0] != '_']\r
-\r
-#\r
-# Server which is run in a process controlled by a manager\r
-#\r
-\r
-class Server(object):\r
-    '''\r
-    Server class which runs in a process controlled by a manager object\r
-    '''\r
-    public = ['shutdown', 'create', 'accept_connection', 'get_methods',\r
-              'debug_info', 'number_of_objects', 'dummy', 'incref', 'decref']\r
-\r
-    def __init__(self, registry, address, authkey, serializer):\r
-        assert isinstance(authkey, bytes)\r
-        self.registry = registry\r
-        self.authkey = AuthenticationString(authkey)\r
-        Listener, Client = listener_client[serializer]\r
-\r
-        # do authentication later\r
-        self.listener = Listener(address=address, backlog=5)\r
-        self.address = self.listener.address\r
-\r
-        self.id_to_obj = {0: (None, ())}\r
-        self.id_to_refcount = {}\r
-        self.mutex = threading.RLock()\r
-        self.stop = 0\r
-\r
-    def serve_forever(self):\r
-        '''\r
-        Run the server forever\r
-        '''\r
-        current_process()._manager_server = self\r
-        try:\r
-            try:\r
-                while 1:\r
-                    try:\r
-                        c = self.listener.accept()\r
-                    except (OSError, IOError):\r
-                        continue\r
-                    t = threading.Thread(target=self.handle_request, args=(c,))\r
-                    t.set_daemon(True)\r
-                    t.start()\r
-            except (KeyboardInterrupt, SystemExit):\r
-                pass\r
-        finally:\r
-            self.stop = 999\r
-            self.listener.close()\r
-\r
-    def handle_request(self, c):\r
-        '''\r
-        Handle a new connection\r
-        '''\r
-        funcname = result = request = None\r
-        try:\r
-            connection.deliver_challenge(c, self.authkey)\r
-            connection.answer_challenge(c, self.authkey)\r
-            request = c.recv()\r
-            ignore, funcname, args, kwds = request\r
-            assert funcname in self.public, '%r unrecognized' % funcname\r
-            func = getattr(self, funcname)\r
-        except Exception:\r
-            msg = ('#TRACEBACK', format_exc())\r
-        else:\r
-            try:\r
-                result = func(c, *args, **kwds)\r
-            except Exception:\r
-                msg = ('#TRACEBACK', format_exc())\r
-            else:\r
-                msg = ('#RETURN', result)\r
-        try:\r
-            c.send(msg)\r
-        except Exception, e:\r
-            try:\r
-                c.send(('#TRACEBACK', format_exc()))\r
-            except Exception:\r
-                pass\r
-            util.info('Failure to send message: %r', msg)\r
-            util.info(' ... request was %r', request)\r
-            util.info(' ... exception was %r', e)\r
-\r
-        c.close()\r
-\r
-    def serve_client(self, conn):\r
-        '''\r
-        Handle requests from the proxies in a particular process/thread\r
-        '''\r
-        util.debug('starting server thread to service %r',\r
-                   threading.current_thread().get_name())\r
-\r
-        recv = conn.recv\r
-        send = conn.send\r
-        id_to_obj = self.id_to_obj\r
-\r
-        while not self.stop:\r
-\r
-            try:\r
-                methodname = obj = None\r
-                request = recv()\r
-                ident, methodname, args, kwds = request\r
-                obj, exposed, gettypeid = id_to_obj[ident]\r
-\r
-                if methodname not in exposed:\r
-                    raise AttributeError(\r
-                        'method %r of %r object is not in exposed=%r' %\r
-                        (methodname, type(obj), exposed)\r
-                        )\r
-\r
-                function = getattr(obj, methodname)\r
-\r
-                try:\r
-                    res = function(*args, **kwds)\r
-                except Exception, e:\r
-                    msg = ('#ERROR', e)\r
-                else:\r
-                    typeid = gettypeid and gettypeid.get(methodname, None)\r
-                    if typeid:\r
-                        rident, rexposed = self.create(conn, typeid, res)\r
-                        token = Token(typeid, self.address, rident)\r
-                        msg = ('#PROXY', (rexposed, token))\r
-                    else:\r
-                        msg = ('#RETURN', res)\r
-\r
-            except AttributeError:\r
-                if methodname is None:\r
-                    msg = ('#TRACEBACK', format_exc())\r
-                else:\r
-                    try:\r
-                        fallback_func = self.fallback_mapping[methodname]\r
-                        result = fallback_func(\r
-                            self, conn, ident, obj, *args, **kwds\r
-                            )\r
-                        msg = ('#RETURN', result)\r
-                    except Exception:\r
-                        msg = ('#TRACEBACK', format_exc())\r
-\r
-            except EOFError:\r
-                util.debug('got EOF -- exiting thread serving %r',\r
-                           threading.current_thread().get_name())\r
-                sys.exit(0)\r
-\r
-            except Exception:\r
-                msg = ('#TRACEBACK', format_exc())\r
-\r
-            try:\r
-                try:\r
-                    send(msg)\r
-                except Exception, e:\r
-                    send(('#UNSERIALIZABLE', repr(msg)))\r
-            except Exception, e:\r
-                util.info('exception in thread serving %r',\r
-                        threading.current_thread().get_name())\r
-                util.info(' ... message was %r', msg)\r
-                util.info(' ... exception was %r', e)\r
-                conn.close()\r
-                sys.exit(1)\r
-\r
-    def fallback_getvalue(self, conn, ident, obj):\r
-        return obj\r
-\r
-    def fallback_str(self, conn, ident, obj):\r
-        return str(obj)\r
-\r
-    def fallback_repr(self, conn, ident, obj):\r
-        return repr(obj)\r
-\r
-    fallback_mapping = {\r
-        '__str__':fallback_str,\r
-        '__repr__':fallback_repr,\r
-        '#GETVALUE':fallback_getvalue\r
-        }\r
-\r
-    def dummy(self, c):\r
-        pass\r
-\r
-    def debug_info(self, c):\r
-        '''\r
-        Return some info --- useful to spot problems with refcounting\r
-        '''\r
-        self.mutex.acquire()\r
-        try:\r
-            result = []\r
-            keys = self.id_to_obj.keys()\r
-            keys.sort()\r
-            for ident in keys:\r
-                if ident != 0:\r
-                    result.append('  %s:       refcount=%s\n    %s' %\r
-                                  (ident, self.id_to_refcount[ident],\r
-                                   str(self.id_to_obj[ident][0])[:75]))\r
-            return '\n'.join(result)\r
-        finally:\r
-            self.mutex.release()\r
-\r
-    def number_of_objects(self, c):\r
-        '''\r
-        Number of shared objects\r
-        '''\r
-        return len(self.id_to_obj) - 1      # don't count ident=0\r
-\r
-    def shutdown(self, c):\r
-        '''\r
-        Shutdown this process\r
-        '''\r
-        try:\r
-            try:\r
-                util.debug('manager received shutdown message')\r
-                c.send(('#RETURN', None))\r
-\r
-                if sys.stdout != sys.__stdout__:\r
-                    util.debug('resetting stdout, stderr')\r
-                    sys.stdout = sys.__stdout__\r
-                    sys.stderr = sys.__stderr__\r
-\r
-                util._run_finalizers(0)\r
-\r
-                for p in active_children():\r
-                    util.debug('terminating a child process of manager')\r
-                    p.terminate()\r
-\r
-                for p in active_children():\r
-                    util.debug('terminating a child process of manager')\r
-                    p.join()\r
-\r
-                util._run_finalizers()\r
-                util.info('manager exiting with exitcode 0')\r
-            except:\r
-                import traceback\r
-                traceback.print_exc()\r
-        finally:\r
-            exit(0)\r
-\r
-    def create(self, c, typeid, *args, **kwds):\r
-        '''\r
-        Create a new shared object and return its id\r
-        '''\r
-        self.mutex.acquire()\r
-        try:\r
-            callable, exposed, method_to_typeid, proxytype = \\r
-                      self.registry[typeid]\r
-\r
-            if callable is None:\r
-                assert len(args) == 1 and not kwds\r
-                obj = args[0]\r
-            else:\r
-                obj = callable(*args, **kwds)\r
-\r
-            if exposed is None:\r
-                exposed = public_methods(obj)\r
-            if method_to_typeid is not None:\r
-                assert type(method_to_typeid) is dict\r
-                exposed = list(exposed) + list(method_to_typeid)\r
-\r
-            ident = '%x' % id(obj)  # convert to string because xmlrpclib\r
-                                    # only has 32 bit signed integers\r
-            util.debug('%r callable returned object with id %r', typeid, ident)\r
-\r
-            self.id_to_obj[ident] = (obj, set(exposed), method_to_typeid)\r
-            if ident not in self.id_to_refcount:\r
-                self.id_to_refcount[ident] = None\r
-            return ident, tuple(exposed)\r
-        finally:\r
-            self.mutex.release()\r
-\r
-    def get_methods(self, c, token):\r
-        '''\r
-        Return the methods of the shared object indicated by token\r
-        '''\r
-        return tuple(self.id_to_obj[token.id][1])\r
-\r
-    def accept_connection(self, c, name):\r
-        '''\r
-        Spawn a new thread to serve this connection\r
-        '''\r
-        threading.current_thread().set_name(name)\r
-        c.send(('#RETURN', None))\r
-        self.serve_client(c)\r
-\r
-    def incref(self, c, ident):\r
-        self.mutex.acquire()\r
-        try:\r
-            try:\r
-                self.id_to_refcount[ident] += 1\r
-            except TypeError:\r
-                assert self.id_to_refcount[ident] is None\r
-                self.id_to_refcount[ident] = 1\r
-        finally:\r
-            self.mutex.release()\r
-\r
-    def decref(self, c, ident):\r
-        self.mutex.acquire()\r
-        try:\r
-            assert self.id_to_refcount[ident] >= 1\r
-            self.id_to_refcount[ident] -= 1\r
-            if self.id_to_refcount[ident] == 0:\r
-                del self.id_to_obj[ident], self.id_to_refcount[ident]\r
-                util.debug('disposing of obj with id %d', ident)\r
-        finally:\r
-            self.mutex.release()\r
-\r
-#\r
-# Class to represent state of a manager\r
-#\r
-\r
-class State(object):\r
-    __slots__ = ['value']\r
-    INITIAL = 0\r
-    STARTED = 1\r
-    SHUTDOWN = 2\r
-\r
-#\r
-# Mapping from serializer name to Listener and Client types\r
-#\r
-\r
-listener_client = {\r
-    'pickle' : (connection.Listener, connection.Client),\r
-    'xmlrpclib' : (connection.XmlListener, connection.XmlClient)\r
-    }\r
-\r
-#\r
-# Definition of BaseManager\r
-#\r
-\r
-class BaseManager(object):\r
-    '''\r
-    Base class for managers\r
-    '''\r
-    _registry = {}\r
-    _Server = Server\r
-\r
-    def __init__(self, address=None, authkey=None, serializer='pickle'):\r
-        if authkey is None:\r
-            authkey = current_process().get_authkey()\r
-        self._address = address     # XXX not final address if eg ('', 0)\r
-        self._authkey = AuthenticationString(authkey)\r
-        self._state = State()\r
-        self._state.value = State.INITIAL\r
-        self._serializer = serializer\r
-        self._Listener, self._Client = listener_client[serializer]\r
-\r
-    def __reduce__(self):\r
-        return type(self).from_address, \\r
-               (self._address, self._authkey, self._serializer)\r
-\r
-    def get_server(self):\r
-        '''\r
-        Return server object with serve_forever() method and address attribute\r
-        '''\r
-        assert self._state.value == State.INITIAL\r
-        return Server(self._registry, self._address,\r
-                      self._authkey, self._serializer)\r
-\r
-    def connect(self):\r
-        '''\r
-        Connect manager object to the server process\r
-        '''\r
-        Listener, Client = listener_client[self._serializer]\r
-        conn = Client(self._address, authkey=self._authkey)\r
-        dispatch(conn, None, 'dummy')\r
-        self._state.value = State.STARTED\r
-\r
-    def start(self):\r
-        '''\r
-        Spawn a server process for this manager object\r
-        '''\r
-        assert self._state.value == State.INITIAL\r
-\r
-        # pipe over which we will retrieve address of server\r
-        reader, writer = connection.Pipe(duplex=False)\r
-\r
-        # spawn process which runs a server\r
-        self._process = Process(\r
-            target=type(self)._run_server,\r
-            args=(self._registry, self._address, self._authkey,\r
-                  self._serializer, writer),\r
-            )\r
-        ident = ':'.join(str(i) for i in self._process._identity)\r
-        self._process.set_name(type(self).__name__  + '-' + ident)\r
-        self._process.start()\r
-\r
-        # get address of server\r
-        writer.close()\r
-        self._address = reader.recv()\r
-        reader.close()\r
-\r
-        # register a finalizer\r
-        self._state.value = State.STARTED\r
-        self.shutdown = util.Finalize(\r
-            self, type(self)._finalize_manager,\r
-            args=(self._process, self._address, self._authkey,\r
-                  self._state, self._Client),\r
-            exitpriority=0\r
-            )\r
-\r
-    @classmethod\r
-    def _run_server(cls, registry, address, authkey, serializer, writer):\r
-        '''\r
-        Create a server, report its address and run it\r
-        '''\r
-        # create server\r
-        server = cls._Server(registry, address, authkey, serializer)\r
-\r
-        # inform parent process of the server's address\r
-        writer.send(server.address)\r
-        writer.close()\r
-\r
-        # run the manager\r
-        util.info('manager serving at %r', server.address)\r
-        server.serve_forever()\r
-\r
-    def _create(self, typeid, *args, **kwds):\r
-        '''\r
-        Create a new shared object; return the token and exposed tuple\r
-        '''\r
-        assert self._state.value == State.STARTED, 'server not yet started'\r
-        conn = self._Client(self._address, authkey=self._authkey)\r
-        try:\r
-            id, exposed = dispatch(conn, None, 'create', (typeid,)+args, kwds)\r
-        finally:\r
-            conn.close()\r
-        return Token(typeid, self._address, id), exposed\r
-\r
-    def join(self, timeout=None):\r
-        '''\r
-        Join the manager process (if it has been spawned)\r
-        '''\r
-        self._process.join(timeout)\r
-\r
-    def _debug_info(self):\r
-        '''\r
-        Return some info about the servers shared objects and connections\r
-        '''\r
-        conn = self._Client(self._address, authkey=self._authkey)\r
-        try:\r
-            return dispatch(conn, None, 'debug_info')\r
-        finally:\r
-            conn.close()\r
-\r
-    def _number_of_objects(self):\r
-        '''\r
-        Return the number of shared objects\r
-        '''\r
-        conn = self._Client(self._address, authkey=self._authkey)\r
-        try:\r
-            return dispatch(conn, None, 'number_of_objects')\r
-        finally:\r
-            conn.close()\r
-\r
-    def __enter__(self):\r
-        return self\r
-\r
-    def __exit__(self, exc_type, exc_val, exc_tb):\r
-        self.shutdown()\r
-\r
-    @staticmethod\r
-    def _finalize_manager(process, address, authkey, state, _Client):\r
-        '''\r
-        Shutdown the manager process; will be registered as a finalizer\r
-        '''\r
-        if process.is_alive():\r
-            util.info('sending shutdown message to manager')\r
-            try:\r
-                conn = _Client(address, authkey=authkey)\r
-                try:\r
-                    dispatch(conn, None, 'shutdown')\r
-                finally:\r
-                    conn.close()\r
-            except Exception:\r
-                pass\r
-\r
-            process.join(timeout=0.2)\r
-            if process.is_alive():\r
-                util.info('manager still alive')\r
-                if hasattr(process, 'terminate'):\r
-                    util.info('trying to `terminate()` manager process')\r
-                    process.terminate()\r
-                    process.join(timeout=0.1)\r
-                    if process.is_alive():\r
-                        util.info('manager still alive after terminate')\r
-\r
-        state.value = State.SHUTDOWN\r
-        try:\r
-            del BaseProxy._address_to_local[address]\r
-        except KeyError:\r
-            pass\r
-\r
-    address = property(lambda self: self._address)\r
-\r
-    @classmethod\r
-    def register(cls, typeid, callable=None, proxytype=None, exposed=None,\r
-                 method_to_typeid=None, create_method=True):\r
-        '''\r
-        Register a typeid with the manager type\r
-        '''\r
-        if '_registry' not in cls.__dict__:\r
-            cls._registry = cls._registry.copy()\r
-\r
-        if proxytype is None:\r
-            proxytype = AutoProxy\r
-\r
-        exposed = exposed or getattr(proxytype, '_exposed_', None)\r
-\r
-        method_to_typeid = method_to_typeid or \\r
-                           getattr(proxytype, '_method_to_typeid_', None)\r
-\r
-        if method_to_typeid:\r
-            for key, value in method_to_typeid.items():\r
-                assert type(key) is str, '%r is not a string' % key\r
-                assert type(value) is str, '%r is not a string' % value\r
-\r
-        cls._registry[typeid] = (\r
-            callable, exposed, method_to_typeid, proxytype\r
-            )\r
-\r
-        if create_method:\r
-            def temp(self, *args, **kwds):\r
-                util.debug('requesting creation of a shared %r object', typeid)\r
-                token, exp = self._create(typeid, *args, **kwds)\r
-                proxy = proxytype(\r
-                    token, self._serializer, manager=self,\r
-                    authkey=self._authkey, exposed=exp\r
-                    )\r
-                return proxy\r
-            temp.__name__ = typeid\r
-            setattr(cls, typeid, temp)\r
-\r
-#\r
-# Subclass of set which get cleared after a fork\r
-#\r
-\r
-class ProcessLocalSet(set):\r
-    def __init__(self):\r
-        util.register_after_fork(self, lambda obj: obj.clear())\r
-    def __reduce__(self):\r
-        return type(self), ()\r
-\r
-#\r
-# Definition of BaseProxy\r
-#\r
-\r
-class BaseProxy(object):\r
-    '''\r
-    A base for proxies of shared objects\r
-    '''\r
-    _address_to_local = {}\r
-    _mutex = util.ForkAwareThreadLock()\r
-\r
-    def __init__(self, token, serializer, manager=None,\r
-                 authkey=None, exposed=None, incref=True):\r
-        BaseProxy._mutex.acquire()\r
-        try:\r
-            tls_idset = BaseProxy._address_to_local.get(token.address, None)\r
-            if tls_idset is None:\r
-                tls_idset = util.ForkAwareLocal(), ProcessLocalSet()\r
-                BaseProxy._address_to_local[token.address] = tls_idset\r
-        finally:\r
-            BaseProxy._mutex.release()\r
-\r
-        # self._tls is used to record the connection used by this\r
-        # thread to communicate with the manager at token.address\r
-        self._tls = tls_idset[0]\r
-\r
-        # self._idset is used to record the identities of all shared\r
-        # objects for which the current process owns references and\r
-        # which are in the manager at token.address\r
-        self._idset = tls_idset[1]\r
-\r
-        self._token = token\r
-        self._id = self._token.id\r
-        self._manager = manager\r
-        self._serializer = serializer\r
-        self._Client = listener_client[serializer][1]\r
-\r
-        if authkey is not None:\r
-            self._authkey = AuthenticationString(authkey)\r
-        elif self._manager is not None:\r
-            self._authkey = self._manager._authkey\r
-        else:\r
-            self._authkey = current_process().get_authkey()\r
-\r
-        if incref:\r
-            self._incref()\r
-\r
-        util.register_after_fork(self, BaseProxy._after_fork)\r
-\r
-    def _connect(self):\r
-        util.debug('making connection to manager')\r
-        name = current_process().get_name()\r
-        if threading.current_thread().get_name() != 'MainThread':\r
-            name += '|' + threading.current_thread().get_name()\r
-        conn = self._Client(self._token.address, authkey=self._authkey)\r
-        dispatch(conn, None, 'accept_connection', (name,))\r
-        self._tls.connection = conn\r
-\r
-    def _callmethod(self, methodname, args=(), kwds={}):\r
-        '''\r
-        Try to call a method of the referrent and return a copy of the result\r
-        '''\r
-        try:\r
-            conn = self._tls.connection\r
-        except AttributeError:\r
-            util.debug('thread %r does not own a connection',\r
-                       threading.current_thread().get_name())\r
-            self._connect()\r
-            conn = self._tls.connection\r
-\r
-        conn.send((self._id, methodname, args, kwds))\r
-        kind, result = conn.recv()\r
-\r
-        if kind == '#RETURN':\r
-            return result\r
-        elif kind == '#PROXY':\r
-            exposed, token = result\r
-            proxytype = self._manager._registry[token.typeid][-1]\r
-            return proxytype(\r
-                token, self._serializer, manager=self._manager,\r
-                authkey=self._authkey, exposed=exposed\r
-                )\r
-        raise convert_to_error(kind, result)\r
-\r
-    def _getvalue(self):\r
-        '''\r
-        Get a copy of the value of the referent\r
-        '''\r
-        return self._callmethod('#GETVALUE')\r
-\r
-    def _incref(self):\r
-        conn = self._Client(self._token.address, authkey=self._authkey)\r
-        dispatch(conn, None, 'incref', (self._id,))\r
-        util.debug('INCREF %r', self._token.id)\r
-\r
-        self._idset.add(self._id)\r
-\r
-        state = self._manager and self._manager._state\r
-\r
-        self._close = util.Finalize(\r
-            self, BaseProxy._decref,\r
-            args=(self._token, self._authkey, state,\r
-                  self._tls, self._idset, self._Client),\r
-            exitpriority=10\r
-            )\r
-\r
-    @staticmethod\r
-    def _decref(token, authkey, state, tls, idset, _Client):\r
-        idset.discard(token.id)\r
-\r
-        # check whether manager is still alive\r
-        if state is None or state.value == State.STARTED:\r
-            # tell manager this process no longer cares about referent\r
-            try:\r
-                util.debug('DECREF %r', token.id)\r
-                conn = _Client(token.address, authkey=authkey)\r
-                dispatch(conn, None, 'decref', (token.id,))\r
-            except Exception, e:\r
-                util.debug('... decref failed %s', e)\r
-\r
-        else:\r
-            util.debug('DECREF %r -- manager already shutdown', token.id)\r
-\r
-        # check whether we can close this thread's connection because\r
-        # the process owns no more references to objects for this manager\r
-        if not idset and hasattr(tls, 'connection'):\r
-            util.debug('thread %r has no more proxies so closing conn',\r
-                       threading.current_thread().get_name())\r
-            tls.connection.close()\r
-            del tls.connection\r
-\r
-    def _after_fork(self):\r
-        self._manager = None\r
-        try:\r
-            self._incref()\r
-        except Exception, e:\r
-            # the proxy may just be for a manager which has shutdown\r
-            util.info('incref failed: %s' % e)\r
-\r
-    def __reduce__(self):\r
-        kwds = {}\r
-        if Popen.thread_is_spawning():\r
-            kwds['authkey'] = self._authkey\r
-\r
-        if getattr(self, '_isauto', False):\r
-            kwds['exposed'] = self._exposed_\r
-            return (RebuildProxy,\r
-                    (AutoProxy, self._token, self._serializer, kwds))\r
-        else:\r
-            return (RebuildProxy,\r
-                    (type(self), self._token, self._serializer, kwds))\r
-\r
-    def __deepcopy__(self, memo):\r
-        return self._getvalue()\r
-\r
-    def __repr__(self):\r
-        return '<%s object, typeid %r at %s>' % \\r
-               (type(self).__name__, self._token.typeid, '0x%x' % id(self))\r
-\r
-    def __str__(self):\r
-        '''\r
-        Return representation of the referent (or a fall-back if that fails)\r
-        '''\r
-        try:\r
-            return self._callmethod('__repr__')\r
-        except Exception:\r
-            return repr(self)[:-1] + "; '__str__()' failed>"\r
-\r
-#\r
-# Function used for unpickling\r
-#\r
-\r
-def RebuildProxy(func, token, serializer, kwds):\r
-    '''\r
-    Function used for unpickling proxy objects.\r
-\r
-    If possible the shared object is returned, or otherwise a proxy for it.\r
-    '''\r
-    server = getattr(current_process(), '_manager_server', None)\r
-\r
-    if server and server.address == token.address:\r
-        return server.id_to_obj[token.id][0]\r
-    else:\r
-        incref = (\r
-            kwds.pop('incref', True) and\r
-            not getattr(current_process(), '_inheriting', False)\r
-            )\r
-        return func(token, serializer, incref=incref, **kwds)\r
-\r
-#\r
-# Functions to create proxies and proxy types\r
-#\r
-\r
-def MakeProxyType(name, exposed, _cache={}):\r
-    '''\r
-    Return an proxy type whose methods are given by `exposed`\r
-    '''\r
-    exposed = tuple(exposed)\r
-    try:\r
-        return _cache[(name, exposed)]\r
-    except KeyError:\r
-        pass\r
-\r
-    dic = {}\r
-\r
-    for meth in exposed:\r
-        exec '''def %s(self, *args, **kwds):\r
-        return self._callmethod(%r, args, kwds)''' % (meth, meth) in dic\r
-\r
-    ProxyType = type(name, (BaseProxy,), dic)\r
-    ProxyType._exposed_ = exposed\r
-    _cache[(name, exposed)] = ProxyType\r
-    return ProxyType\r
-\r
-\r
-def AutoProxy(token, serializer, manager=None, authkey=None,\r
-              exposed=None, incref=True):\r
-    '''\r
-    Return an auto-proxy for `token`\r
-    '''\r
-    _Client = listener_client[serializer][1]\r
-\r
-    if exposed is None:\r
-        conn = _Client(token.address, authkey=authkey)\r
-        try:\r
-            exposed = dispatch(conn, None, 'get_methods', (token,))\r
-        finally:\r
-            conn.close()\r
-\r
-    if authkey is None and manager is not None:\r
-        authkey = manager._authkey\r
-    if authkey is None:\r
-        authkey = current_process().get_authkey()\r
-\r
-    ProxyType = MakeProxyType('AutoProxy[%s]' % token.typeid, exposed)\r
-    proxy = ProxyType(token, serializer, manager=manager, authkey=authkey,\r
-                      incref=incref)\r
-    proxy._isauto = True\r
-    return proxy\r
-\r
-#\r
-# Types/callables which we will register with SyncManager\r
-#\r
-\r
-class Namespace(object):\r
-    def __init__(self, **kwds):\r
-        self.__dict__.update(kwds)\r
-    def __repr__(self):\r
-        items = self.__dict__.items()\r
-        temp = []\r
-        for name, value in items:\r
-            if not name.startswith('_'):\r
-                temp.append('%s=%r' % (name, value))\r
-        temp.sort()\r
-        return 'Namespace(%s)' % str.join(', ', temp)\r
-\r
-class Value(object):\r
-    def __init__(self, typecode, value, lock=True):\r
-        self._typecode = typecode\r
-        self._value = value\r
-    def get(self):\r
-        return self._value\r
-    def set(self, value):\r
-        self._value = value\r
-    def __repr__(self):\r
-        return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value)\r
-    value = property(get, set)\r
-\r
-def Array(typecode, sequence, lock=True):\r
-    return array.array(typecode, sequence)\r
-\r
-#\r
-# Proxy types used by SyncManager\r
-#\r
-\r
-class IteratorProxy(BaseProxy):\r
-    # XXX remove methods for Py3.0 and Py2.6\r
-    _exposed_ = ('__next__', 'next', 'send', 'throw', 'close')\r
-    def __iter__(self):\r
-        return self\r
-    def __next__(self, *args):\r
-        return self._callmethod('__next__', args)\r
-    def next(self, *args):\r
-        return self._callmethod('next', args)\r
-    def send(self, *args):\r
-        return self._callmethod('send', args)\r
-    def throw(self, *args):\r
-        return self._callmethod('throw', args)\r
-    def close(self, *args):\r
-        return self._callmethod('close', args)\r
-\r
-\r
-class AcquirerProxy(BaseProxy):\r
-    _exposed_ = ('acquire', 'release')\r
-    def acquire(self, blocking=True):\r
-        return self._callmethod('acquire', (blocking,))\r
-    def release(self):\r
-        return self._callmethod('release')\r
-    def __enter__(self):\r
-        return self._callmethod('acquire')\r
-    def __exit__(self, exc_type, exc_val, exc_tb):\r
-        return self._callmethod('release')\r
-\r
-\r
-class ConditionProxy(AcquirerProxy):\r
-    # XXX will Condition.notfyAll() name be available in Py3.0?\r
-    _exposed_ = ('acquire', 'release', 'wait', 'notify', 'notify_all')\r
-    def wait(self, timeout=None):\r
-        return self._callmethod('wait', (timeout,))\r
-    def notify(self):\r
-        return self._callmethod('notify')\r
-    def notify_all(self):\r
-        return self._callmethod('notify_all')\r
-\r
-class EventProxy(BaseProxy):\r
-    # XXX will Event.isSet name be available in Py3.0?\r
-    _exposed_ = ('isSet', 'set', 'clear', 'wait')\r
-    def is_set(self):\r
-        return self._callmethod('isSet')\r
-    def set(self):\r
-        return self._callmethod('set')\r
-    def clear(self):\r
-        return self._callmethod('clear')\r
-    def wait(self, timeout=None):\r
-        return self._callmethod('wait', (timeout,))\r
-\r
-class NamespaceProxy(BaseProxy):\r
-    _exposed_ = ('__getattribute__', '__setattr__', '__delattr__')\r
-    def __getattr__(self, key):\r
-        if key[0] == '_':\r
-            return object.__getattribute__(self, key)\r
-        callmethod = object.__getattribute__(self, '_callmethod')\r
-        return callmethod('__getattribute__', (key,))\r
-    def __setattr__(self, key, value):\r
-        if key[0] == '_':\r
-            return object.__setattr__(self, key, value)\r
-        callmethod = object.__getattribute__(self, '_callmethod')\r
-        return callmethod('__setattr__', (key, value))\r
-    def __delattr__(self, key):\r
-        if key[0] == '_':\r
-            return object.__delattr__(self, key)\r
-        callmethod = object.__getattribute__(self, '_callmethod')\r
-        return callmethod('__delattr__', (key,))\r
-\r
-\r
-class ValueProxy(BaseProxy):\r
-    _exposed_ = ('get', 'set')\r
-    def get(self):\r
-        return self._callmethod('get')\r
-    def set(self, value):\r
-        return self._callmethod('set', (value,))\r
-    value = property(get, set)\r
-\r
-\r
-BaseListProxy = MakeProxyType('BaseListProxy', (\r
-    '__add__', '__contains__', '__delitem__', '__delslice__',\r
-    '__getitem__', '__getslice__', '__len__', '__mul__',\r
-    '__reversed__', '__rmul__', '__setitem__', '__setslice__',\r
-    'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove',\r
-    'reverse', 'sort', '__imul__'\r
-    ))                  # XXX __getslice__ and __setslice__ unneeded in Py3.0\r
-class ListProxy(BaseListProxy):\r
-    def __iadd__(self, value):\r
-        self._callmethod('extend', (value,))\r
-        return self\r
-    def __imul__(self, value):\r
-        self._callmethod('__imul__', (value,))\r
-        return self\r
-\r
-\r
-DictProxy = MakeProxyType('DictProxy', (\r
-    '__contains__', '__delitem__', '__getitem__', '__len__',\r
-    '__setitem__', 'clear', 'copy', 'get', 'has_key', 'items',\r
-    'keys', 'pop', 'popitem', 'setdefault', 'update', 'values'\r
-    ))\r
-\r
-\r
-ArrayProxy = MakeProxyType('ArrayProxy', (\r
-    '__len__', '__getitem__', '__setitem__', '__getslice__', '__setslice__'\r
-    ))                  # XXX __getslice__ and __setslice__ unneeded in Py3.0\r
-\r
-\r
-PoolProxy = MakeProxyType('PoolProxy', (\r
-    'apply', 'apply_async', 'close', 'imap', 'imap_unordered', 'join',\r
-    'map', 'map_async', 'terminate'\r
-    ))\r
-PoolProxy._method_to_typeid_ = {\r
-    'apply_async': 'AsyncResult',\r
-    'map_async': 'AsyncResult',\r
-    'imap': 'Iterator',\r
-    'imap_unordered': 'Iterator'\r
-    }\r
-\r
-#\r
-# Definition of SyncManager\r
-#\r
-\r
-class SyncManager(BaseManager):\r
-    '''\r
-    Subclass of `BaseManager` which supports a number of shared object types.\r
-\r
-    The types registered are those intended for the synchronization\r
-    of threads, plus `dict`, `list` and `Namespace`.\r
-\r
-    The `multiprocessing.Manager()` function creates started instances of\r
-    this class.\r
-    '''\r
-\r
-SyncManager.register('Queue', Queue.Queue)\r
-SyncManager.register('JoinableQueue', Queue.Queue)\r
-SyncManager.register('Event', threading.Event, EventProxy)\r
-SyncManager.register('Lock', threading.Lock, AcquirerProxy)\r
-SyncManager.register('RLock', threading.RLock, AcquirerProxy)\r
-SyncManager.register('Semaphore', threading.Semaphore, AcquirerProxy)\r
-SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore,\r
-                     AcquirerProxy)\r
-SyncManager.register('Condition', threading.Condition, ConditionProxy)\r
-SyncManager.register('Pool', Pool, PoolProxy)\r
-SyncManager.register('list', list, ListProxy)\r
-SyncManager.register('dict', dict, DictProxy)\r
-SyncManager.register('Value', Value, ValueProxy)\r
-SyncManager.register('Array', Array, ArrayProxy)\r
-SyncManager.register('Namespace', Namespace, NamespaceProxy)\r
-\r
-# types returned by methods of PoolProxy\r
-SyncManager.register('Iterator', proxytype=IteratorProxy, create_method=False)\r
-SyncManager.register('AsyncResult', create_method=False)\r
+#
+# Module providing the `SyncManager` class for dealing
+# with shared objects
+#
+# multiprocessing/managers.py
+#
+# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+#
+
+__all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ]
+
+#
+# Imports
+#
+
+import os
+import sys
+import weakref
+import threading
+import array
+import copy_reg
+import Queue
+
+from traceback import format_exc
+from multiprocessing import Process, current_process, active_children, Pool, util, connection
+from multiprocessing.process import AuthenticationString
+from multiprocessing.forking import exit, Popen, assert_spawning
+from multiprocessing.util import Finalize, info
+
+try:
+    from cPickle import PicklingError
+except ImportError:
+    from pickle import PicklingError
+
+#
+#
+#
+
+try:
+    bytes
+except NameError:
+    bytes = str                  # XXX not needed in Py2.6 and Py3.0
+
+#
+# Register some things for pickling
+#
+
+def reduce_array(a):
+    return array.array, (a.typecode, a.tostring())
+copy_reg.pickle(array.array, reduce_array)
+
+view_types = [type(getattr({}, name)()) for name in ('items','keys','values')]
+if view_types[0] is not list:       # XXX only needed in Py3.0
+    def rebuild_as_list(obj):
+        return list, (list(obj),)
+    for view_type in view_types:
+        copy_reg.pickle(view_type, rebuild_as_list)
+
+#
+# Type for identifying shared objects
+#
+
+class Token(object):
+    '''
+    Type to uniquely indentify a shared object
+    '''
+    __slots__ = ('typeid', 'address', 'id')
+
+    def __init__(self, typeid, address, id):
+        (self.typeid, self.address, self.id) = (typeid, address, id)
+
+    def __getstate__(self):
+        return (self.typeid, self.address, self.id)
+
+    def __setstate__(self, state):
+        (self.typeid, self.address, self.id) = state
+
+    def __repr__(self):
+        return 'Token(typeid=%r, address=%r, id=%r)' % \
+               (self.typeid, self.address, self.id)
+
+#
+# Function for communication with a manager's server process
+#
+
+def dispatch(c, id, methodname, args=(), kwds={}):
+    '''
+    Send a message to manager using connection `c` and return response
+    '''
+    c.send((id, methodname, args, kwds))
+    kind, result = c.recv()
+    if kind == '#RETURN':
+        return result
+    raise convert_to_error(kind, result)
+
+def convert_to_error(kind, result):
+    if kind == '#ERROR':
+        return result
+    elif kind == '#TRACEBACK':
+        assert type(result) is str
+        return  RemoteError(result)
+    elif kind == '#UNSERIALIZABLE':
+        assert type(result) is str
+        return RemoteError('Unserializable message: %s\n' % result)
+    else:
+        return ValueError('Unrecognized message type')
+
+class RemoteError(Exception):
+    def __str__(self):
+        return ('\n' + '-'*75 + '\n' + str(self.args[0]) + '-'*75)
+
+#
+# Functions for finding the method names of an object
+#
+
+def all_methods(obj):
+    '''
+    Return a list of names of methods of `obj`
+    '''
+    temp = []
+    for name in dir(obj):
+        func = getattr(obj, name)
+        if hasattr(func, '__call__'):
+            temp.append(name)
+    return temp
+
+def public_methods(obj):
+    '''
+    Return a list of names of methods of `obj` which do not start with '_'
+    '''
+    return [name for name in all_methods(obj) if name[0] != '_']
+
+#
+# Server which is run in a process controlled by a manager
+#
+
+class Server(object):
+    '''
+    Server class which runs in a process controlled by a manager object
+    '''
+    public = ['shutdown', 'create', 'accept_connection', 'get_methods',
+              'debug_info', 'number_of_objects', 'dummy', 'incref', 'decref']
+
+    def __init__(self, registry, address, authkey, serializer):
+        assert isinstance(authkey, bytes)
+        self.registry = registry
+        self.authkey = AuthenticationString(authkey)
+        Listener, Client = listener_client[serializer]
+
+        # do authentication later
+        self.listener = Listener(address=address, backlog=5)
+        self.address = self.listener.address
+
+        self.id_to_obj = {0: (None, ())}
+        self.id_to_refcount = {}
+        self.mutex = threading.RLock()
+        self.stop = 0
+
+    def serve_forever(self):
+        '''
+        Run the server forever
+        '''
+        current_process()._manager_server = self
+        try:
+            try:
+                while 1:
+                    try:
+                        c = self.listener.accept()
+                    except (OSError, IOError):
+                        continue
+                    t = threading.Thread(target=self.handle_request, args=(c,))
+                    t.set_daemon(True)
+                    t.start()
+            except (KeyboardInterrupt, SystemExit):
+                pass
+        finally:
+            self.stop = 999
+            self.listener.close()
+
+    def handle_request(self, c):
+        '''
+        Handle a new connection
+        '''
+        funcname = result = request = None
+        try:
+            connection.deliver_challenge(c, self.authkey)
+            connection.answer_challenge(c, self.authkey)
+            request = c.recv()
+            ignore, funcname, args, kwds = request
+            assert funcname in self.public, '%r unrecognized' % funcname
+            func = getattr(self, funcname)
+        except Exception:
+            msg = ('#TRACEBACK', format_exc())
+        else:
+            try:
+                result = func(c, *args, **kwds)
+            except Exception:
+                msg = ('#TRACEBACK', format_exc())
+            else:
+                msg = ('#RETURN', result)
+        try:
+            c.send(msg)
+        except Exception, e:
+            try:
+                c.send(('#TRACEBACK', format_exc()))
+            except Exception:
+                pass
+            util.info('Failure to send message: %r', msg)
+            util.info(' ... request was %r', request)
+            util.info(' ... exception was %r', e)
+
+        c.close()
+
+    def serve_client(self, conn):
+        '''
+        Handle requests from the proxies in a particular process/thread
+        '''
+        util.debug('starting server thread to service %r',
+                   threading.current_thread().get_name())
+
+        recv = conn.recv
+        send = conn.send
+        id_to_obj = self.id_to_obj
+
+        while not self.stop:
+
+            try:
+                methodname = obj = None
+                request = recv()
+                ident, methodname, args, kwds = request
+                obj, exposed, gettypeid = id_to_obj[ident]
+
+                if methodname not in exposed:
+                    raise AttributeError(
+                        'method %r of %r object is not in exposed=%r' %
+                        (methodname, type(obj), exposed)
+                        )
+
+                function = getattr(obj, methodname)
+
+                try:
+                    res = function(*args, **kwds)
+                except Exception, e:
+                    msg = ('#ERROR', e)
+                else:
+                    typeid = gettypeid and gettypeid.get(methodname, None)
+                    if typeid:
+                        rident, rexposed = self.create(conn, typeid, res)
+                        token = Token(typeid, self.address, rident)
+                        msg = ('#PROXY', (rexposed, token))
+                    else:
+                        msg = ('#RETURN', res)
+
+            except AttributeError:
+                if methodname is None:
+                    msg = ('#TRACEBACK', format_exc())
+                else:
+                    try:
+                        fallback_func = self.fallback_mapping[methodname]
+                        result = fallback_func(
+                            self, conn, ident, obj, *args, **kwds
+                            )
+                        msg = ('#RETURN', result)
+                    except Exception:
+                        msg = ('#TRACEBACK', format_exc())
+
+            except EOFError:
+                util.debug('got EOF -- exiting thread serving %r',
+                           threading.current_thread().get_name())
+                sys.exit(0)
+
+            except Exception:
+                msg = ('#TRACEBACK', format_exc())
+
+            try:
+                try:
+                    send(msg)
+                except Exception, e:
+                    send(('#UNSERIALIZABLE', repr(msg)))
+            except Exception, e:
+                util.info('exception in thread serving %r',
+                        threading.current_thread().get_name())
+                util.info(' ... message was %r', msg)
+                util.info(' ... exception was %r', e)
+                conn.close()
+                sys.exit(1)
+
+    def fallback_getvalue(self, conn, ident, obj):
+        return obj
+
+    def fallback_str(self, conn, ident, obj):
+        return str(obj)
+
+    def fallback_repr(self, conn, ident, obj):
+        return repr(obj)
+
+    fallback_mapping = {
+        '__str__':fallback_str,
+        '__repr__':fallback_repr,
+        '#GETVALUE':fallback_getvalue
+        }
+
+    def dummy(self, c):
+        pass
+
+    def debug_info(self, c):
+        '''
+        Return some info --- useful to spot problems with refcounting
+        '''
+        self.mutex.acquire()
+        try:
+            result = []
+            keys = self.id_to_obj.keys()
+            keys.sort()
+            for ident in keys:
+                if ident != 0:
+                    result.append('  %s:       refcount=%s\n    %s' %
+                                  (ident, self.id_to_refcount[ident],
+                                   str(self.id_to_obj[ident][0])[:75]))
+            return '\n'.join(result)
+        finally:
+            self.mutex.release()
+
+    def number_of_objects(self, c):
+        '''
+        Number of shared objects
+        '''
+        return len(self.id_to_obj) - 1      # don't count ident=0
+
+    def shutdown(self, c):
+        '''
+        Shutdown this process
+        '''
+        try:
+            try:
+                util.debug('manager received shutdown message')
+                c.send(('#RETURN', None))
+
+                if sys.stdout != sys.__stdout__:
+                    util.debug('resetting stdout, stderr')
+                    sys.stdout = sys.__stdout__
+                    sys.stderr = sys.__stderr__
+
+                util._run_finalizers(0)
+
+                for p in active_children():
+                    util.debug('terminating a child process of manager')
+                    p.terminate()
+
+                for p in active_children():
+                    util.debug('terminating a child process of manager')
+                    p.join()
+
+                util._run_finalizers()
+                util.info('manager exiting with exitcode 0')
+            except:
+                import traceback
+                traceback.print_exc()
+        finally:
+            exit(0)
+
+    def create(self, c, typeid, *args, **kwds):
+        '''
+        Create a new shared object and return its id
+        '''
+        self.mutex.acquire()
+        try:
+            callable, exposed, method_to_typeid, proxytype = \
+                      self.registry[typeid]
+
+            if callable is None:
+                assert len(args) == 1 and not kwds
+                obj = args[0]
+            else:
+                obj = callable(*args, **kwds)
+
+            if exposed is None:
+                exposed = public_methods(obj)
+            if method_to_typeid is not None:
+                assert type(method_to_typeid) is dict
+                exposed = list(exposed) + list(method_to_typeid)
+
+            ident = '%x' % id(obj)  # convert to string because xmlrpclib
+                                    # only has 32 bit signed integers
+            util.debug('%r callable returned object with id %r', typeid, ident)
+
+            self.id_to_obj[ident] = (obj, set(exposed), method_to_typeid)
+            if ident not in self.id_to_refcount:
+                self.id_to_refcount[ident] = None
+            return ident, tuple(exposed)
+        finally:
+            self.mutex.release()
+
+    def get_methods(self, c, token):
+        '''
+        Return the methods of the shared object indicated by token
+        '''
+        return tuple(self.id_to_obj[token.id][1])
+
+    def accept_connection(self, c, name):
+        '''
+        Spawn a new thread to serve this connection
+        '''
+        threading.current_thread().set_name(name)
+        c.send(('#RETURN', None))
+        self.serve_client(c)
+
+    def incref(self, c, ident):
+        self.mutex.acquire()
+        try:
+            try:
+                self.id_to_refcount[ident] += 1
+            except TypeError:
+                assert self.id_to_refcount[ident] is None
+                self.id_to_refcount[ident] = 1
+        finally:
+            self.mutex.release()
+
+    def decref(self, c, ident):
+        self.mutex.acquire()
+        try:
+            assert self.id_to_refcount[ident] >= 1
+            self.id_to_refcount[ident] -= 1
+            if self.id_to_refcount[ident] == 0:
+                del self.id_to_obj[ident], self.id_to_refcount[ident]
+                util.debug('disposing of obj with id %d', ident)
+        finally:
+            self.mutex.release()
+
+#
+# Class to represent state of a manager
+#
+
+class State(object):
+    __slots__ = ['value']
+    INITIAL = 0
+    STARTED = 1
+    SHUTDOWN = 2
+
+#
+# Mapping from serializer name to Listener and Client types
+#
+
+listener_client = {
+    'pickle' : (connection.Listener, connection.Client),
+    'xmlrpclib' : (connection.XmlListener, connection.XmlClient)
+    }
+
+#
+# Definition of BaseManager
+#
+
+class BaseManager(object):
+    '''
+    Base class for managers
+    '''
+    _registry = {}
+    _Server = Server
+
+    def __init__(self, address=None, authkey=None, serializer='pickle'):
+        if authkey is None:
+            authkey = current_process().get_authkey()
+        self._address = address     # XXX not final address if eg ('', 0)
+        self._authkey = AuthenticationString(authkey)
+        self._state = State()
+        self._state.value = State.INITIAL
+        self._serializer = serializer
+        self._Listener, self._Client = listener_client[serializer]
+
+    def __reduce__(self):
+        return type(self).from_address, \
+               (self._address, self._authkey, self._serializer)
+
+    def get_server(self):
+        '''
+        Return server object with serve_forever() method and address attribute
+        '''
+        assert self._state.value == State.INITIAL
+        return Server(self._registry, self._address,
+                      self._authkey, self._serializer)
+
+    def connect(self):
+        '''
+        Connect manager object to the server process
+        '''
+        Listener, Client = listener_client[self._serializer]
+        conn = Client(self._address, authkey=self._authkey)
+        dispatch(conn, None, 'dummy')
+        self._state.value = State.STARTED
+
+    def start(self):
+        '''
+        Spawn a server process for this manager object
+        '''
+        assert self._state.value == State.INITIAL
+
+        # pipe over which we will retrieve address of server
+        reader, writer = connection.Pipe(duplex=False)
+
+        # spawn process which runs a server
+        self._process = Process(
+            target=type(self)._run_server,
+            args=(self._registry, self._address, self._authkey,
+                  self._serializer, writer),
+            )
+        ident = ':'.join(str(i) for i in self._process._identity)
+        self._process.set_name(type(self).__name__  + '-' + ident)
+        self._process.start()
+
+        # get address of server
+        writer.close()
+        self._address = reader.recv()
+        reader.close()
+
+        # register a finalizer
+        self._state.value = State.STARTED
+        self.shutdown = util.Finalize(
+            self, type(self)._finalize_manager,
+            args=(self._process, self._address, self._authkey,
+                  self._state, self._Client),
+            exitpriority=0
+            )
+
+    @classmethod
+    def _run_server(cls, registry, address, authkey, serializer, writer):
+        '''
+        Create a server, report its address and run it
+        '''
+        # create server
+        server = cls._Server(registry, address, authkey, serializer)
+
+        # inform parent process of the server's address
+        writer.send(server.address)
+        writer.close()
+
+        # run the manager
+        util.info('manager serving at %r', server.address)
+        server.serve_forever()
+
+    def _create(self, typeid, *args, **kwds):
+        '''
+        Create a new shared object; return the token and exposed tuple
+        '''
+        assert self._state.value == State.STARTED, 'server not yet started'
+        conn = self._Client(self._address, authkey=self._authkey)
+        try:
+            id, exposed = dispatch(conn, None, 'create', (typeid,)+args, kwds)
+        finally:
+            conn.close()
+        return Token(typeid, self._address, id), exposed
+
+    def join(self, timeout=None):
+        '''
+        Join the manager process (if it has been spawned)
+        '''
+        self._process.join(timeout)
+
+    def _debug_info(self):
+        '''
+        Return some info about the servers shared objects and connections
+        '''
+        conn = self._Client(self._address, authkey=self._authkey)
+        try:
+            return dispatch(conn, None, 'debug_info')
+        finally:
+            conn.close()
+
+    def _number_of_objects(self):
+        '''
+        Return the number of shared objects
+        '''
+        conn = self._Client(self._address, authkey=self._authkey)
+        try:
+            return dispatch(conn, None, 'number_of_objects')
+        finally:
+            conn.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.shutdown()
+
+    @staticmethod
+    def _finalize_manager(process, address, authkey, state, _Client):
+        '''
+        Shutdown the manager process; will be registered as a finalizer
+        '''
+        if process.is_alive():
+            util.info('sending shutdown message to manager')
+            try:
+                conn = _Client(address, authkey=authkey)
+                try:
+                    dispatch(conn, None, 'shutdown')
+                finally:
+                    conn.close()
+            except Exception:
+                pass
+
+            process.join(timeout=0.2)
+            if process.is_alive():
+                util.info('manager still alive')
+                if hasattr(process, 'terminate'):
+                    util.info('trying to `terminate()` manager process')
+                    process.terminate()
+                    process.join(timeout=0.1)
+                    if process.is_alive():
+                        util.info('manager still alive after terminate')
+
+        state.value = State.SHUTDOWN
+        try:
+            del BaseProxy._address_to_local[address]
+        except KeyError:
+            pass
+
+    address = property(lambda self: self._address)
+
+    @classmethod
+    def register(cls, typeid, callable=None, proxytype=None, exposed=None,
+                 method_to_typeid=None, create_method=True):
+        '''
+        Register a typeid with the manager type
+        '''
+        if '_registry' not in cls.__dict__:
+            cls._registry = cls._registry.copy()
+
+        if proxytype is None:
+            proxytype = AutoProxy
+
+        exposed = exposed or getattr(proxytype, '_exposed_', None)
+
+        method_to_typeid = method_to_typeid or \
+                           getattr(proxytype, '_method_to_typeid_', None)
+
+        if method_to_typeid:
+            for key, value in method_to_typeid.items():
+                assert type(key) is str, '%r is not a string' % key
+                assert type(value) is str, '%r is not a string' % value
+
+        cls._registry[typeid] = (
+            callable, exposed, method_to_typeid, proxytype
+            )
+
+        if create_method:
+            def temp(self, *args, **kwds):
+                util.debug('requesting creation of a shared %r object', typeid)
+                token, exp = self._create(typeid, *args, **kwds)
+                proxy = proxytype(
+                    token, self._serializer, manager=self,
+                    authkey=self._authkey, exposed=exp
+                    )
+                return proxy
+            temp.__name__ = typeid
+            setattr(cls, typeid, temp)
+
+#
+# Subclass of set which get cleared after a fork
+#
+
+class ProcessLocalSet(set):
+    def __init__(self):
+        util.register_after_fork(self, lambda obj: obj.clear())
+    def __reduce__(self):
+        return type(self), ()
+
+#
+# Definition of BaseProxy
+#
+
+class BaseProxy(object):
+    '''
+    A base for proxies of shared objects
+    '''
+    _address_to_local = {}
+    _mutex = util.ForkAwareThreadLock()
+
+    def __init__(self, token, serializer, manager=None,
+                 authkey=None, exposed=None, incref=True):
+        BaseProxy._mutex.acquire()
+        try:
+            tls_idset = BaseProxy._address_to_local.get(token.address, None)
+            if tls_idset is None:
+                tls_idset = util.ForkAwareLocal(), ProcessLocalSet()
+                BaseProxy._address_to_local[token.address] = tls_idset
+        finally:
+            BaseProxy._mutex.release()
+
+        # self._tls is used to record the connection used by this
+        # thread to communicate with the manager at token.address
+        self._tls = tls_idset[0]
+
+        # self._idset is used to record the identities of all shared
+        # objects for which the current process owns references and
+        # which are in the manager at token.address
+        self._idset = tls_idset[1]
+
+        self._token = token
+        self._id = self._token.id
+        self._manager = manager
+        self._serializer = serializer
+        self._Client = listener_client[serializer][1]
+
+        if authkey is not None:
+            self._authkey = AuthenticationString(authkey)
+        elif self._manager is not None:
+            self._authkey = self._manager._authkey
+        else:
+            self._authkey = current_process().get_authkey()
+
+        if incref:
+            self._incref()
+
+        util.register_after_fork(self, BaseProxy._after_fork)
+
+    def _connect(self):
+        util.debug('making connection to manager')
+        name = current_process().get_name()
+        if threading.current_thread().get_name() != 'MainThread':
+            name += '|' + threading.current_thread().get_name()
+        conn = self._Client(self._token.address, authkey=self._authkey)
+        dispatch(conn, None, 'accept_connection', (name,))
+        self._tls.connection = conn
+
+    def _callmethod(self, methodname, args=(), kwds={}):
+        '''
+        Try to call a method of the referrent and return a copy of the result
+        '''
+        try:
+            conn = self._tls.connection
+        except AttributeError:
+            util.debug('thread %r does not own a connection',
+                       threading.current_thread().get_name())
+            self._connect()
+            conn = self._tls.connection
+
+        conn.send((self._id, methodname, args, kwds))
+        kind, result = conn.recv()
+
+        if kind == '#RETURN':
+            return result
+        elif kind == '#PROXY':
+            exposed, token = result
+            proxytype = self._manager._registry[token.typeid][-1]
+            return proxytype(
+                token, self._serializer, manager=self._manager,
+                authkey=self._authkey, exposed=exposed
+                )
+        raise convert_to_error(kind, result)
+
+    def _getvalue(self):
+        '''
+        Get a copy of the value of the referent
+        '''
+        return self._callmethod('#GETVALUE')
+
+    def _incref(self):
+        conn = self._Client(self._token.address, authkey=self._authkey)
+        dispatch(conn, None, 'incref', (self._id,))
+        util.debug('INCREF %r', self._token.id)
+
+        self._idset.add(self._id)
+
+        state = self._manager and self._manager._state
+
+        self._close = util.Finalize(
+            self, BaseProxy._decref,
+            args=(self._token, self._authkey, state,
+                  self._tls, self._idset, self._Client),
+            exitpriority=10
+            )
+
+    @staticmethod
+    def _decref(token, authkey, state, tls, idset, _Client):
+        idset.discard(token.id)
+
+        # check whether manager is still alive
+        if state is None or state.value == State.STARTED:
+            # tell manager this process no longer cares about referent
+            try:
+                util.debug('DECREF %r', token.id)
+                conn = _Client(token.address, authkey=authkey)
+                dispatch(conn, None, 'decref', (token.id,))
+            except Exception, e:
+                util.debug('... decref failed %s', e)
+
+        else:
+            util.debug('DECREF %r -- manager already shutdown', token.id)
+
+        # check whether we can close this thread's connection because
+        # the process owns no more references to objects for this manager
+        if not idset and hasattr(tls, 'connection'):
+            util.debug('thread %r has no more proxies so closing conn',
+                       threading.current_thread().get_name())
+            tls.connection.close()
+            del tls.connection
+
+    def _after_fork(self):
+        self._manager = None
+        try:
+            self._incref()
+        except Exception, e:
+            # the proxy may just be for a manager which has shutdown
+            util.info('incref failed: %s' % e)
+
+    def __reduce__(self):
+        kwds = {}
+        if Popen.thread_is_spawning():
+            kwds['authkey'] = self._authkey
+
+        if getattr(self, '_isauto', False):
+            kwds['exposed'] = self._exposed_
+            return (RebuildProxy,
+                    (AutoProxy, self._token, self._serializer, kwds))
+        else:
+            return (RebuildProxy,
+                    (type(self), self._token, self._serializer, kwds))
+
+    def __deepcopy__(self, memo):
+        return self._getvalue()
+
+    def __repr__(self):
+        return '<%s object, typeid %r at %s>' % \
+               (type(self).__name__, self._token.typeid, '0x%x' % id(self))
+
+    def __str__(self):
+        '''
+        Return representation of the referent (or a fall-back if that fails)
+        '''
+        try:
+            return self._callmethod('__repr__')
+        except Exception:
+            return repr(self)[:-1] + "; '__str__()' failed>"
+
+#
+# Function used for unpickling
+#
+
+def RebuildProxy(func, token, serializer, kwds):
+    '''
+    Function used for unpickling proxy objects.
+
+    If possible the shared object is returned, or otherwise a proxy for it.
+    '''
+    server = getattr(current_process(), '_manager_server', None)
+
+    if server and server.address == token.address:
+        return server.id_to_obj[token.id][0]
+    else:
+        incref = (
+            kwds.pop('incref', True) and
+            not getattr(current_process(), '_inheriting', False)
+            )
+        return func(token, serializer, incref=incref, **kwds)
+
+#
+# Functions to create proxies and proxy types
+#
+
+def MakeProxyType(name, exposed, _cache={}):
+    '''
+    Return an proxy type whose methods are given by `exposed`
+    '''
+    exposed = tuple(exposed)
+    try:
+        return _cache[(name, exposed)]
+    except KeyError:
+        pass
+
+    dic = {}
+
+    for meth in exposed:
+        exec '''def %s(self, *args, **kwds):
+        return self._callmethod(%r, args, kwds)''' % (meth, meth) in dic
+
+    ProxyType = type(name, (BaseProxy,), dic)
+    ProxyType._exposed_ = exposed
+    _cache[(name, exposed)] = ProxyType
+    return ProxyType
+
+
+def AutoProxy(token, serializer, manager=None, authkey=None,
+              exposed=None, incref=True):
+    '''
+    Return an auto-proxy for `token`
+    '''
+    _Client = listener_client[serializer][1]
+
+    if exposed is None:
+        conn = _Client(token.address, authkey=authkey)
+        try:
+            exposed = dispatch(conn, None, 'get_methods', (token,))
+        finally:
+            conn.close()
+
+    if authkey is None and manager is not None:
+        authkey = manager._authkey
+    if authkey is None:
+        authkey = current_process().get_authkey()
+
+    ProxyType = MakeProxyType('AutoProxy[%s]' % token.typeid, exposed)
+    proxy = ProxyType(token, serializer, manager=manager, authkey=authkey,
+                      incref=incref)
+    proxy._isauto = True
+    return proxy
+
+#
+# Types/callables which we will register with SyncManager
+#
+
+class Namespace(object):
+    def __init__(self, **kwds):
+        self.__dict__.update(kwds)
+    def __repr__(self):
+        items = self.__dict__.items()
+        temp = []
+        for name, value in items:
+            if not name.startswith('_'):
+                temp.append('%s=%r' % (name, value))
+        temp.sort()
+        return 'Namespace(%s)' % str.join(', ', temp)
+
+class Value(object):
+    def __init__(self, typecode, value, lock=True):
+        self._typecode = typecode
+        self._value = value
+    def get(self):
+        return self._value
+    def set(self, value):
+        self._value = value
+    def __repr__(self):
+        return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value)
+    value = property(get, set)
+
+def Array(typecode, sequence, lock=True):
+    return array.array(typecode, sequence)
+
+#
+# Proxy types used by SyncManager
+#
+
+class IteratorProxy(BaseProxy):
+    # XXX remove methods for Py3.0 and Py2.6
+    _exposed_ = ('__next__', 'next', 'send', 'throw', 'close')
+    def __iter__(self):
+        return self
+    def __next__(self, *args):
+        return self._callmethod('__next__', args)
+    def next(self, *args):
+        return self._callmethod('next', args)
+    def send(self, *args):
+        return self._callmethod('send', args)
+    def throw(self, *args):
+        return self._callmethod('throw', args)
+    def close(self, *args):
+        return self._callmethod('close', args)
+
+
+class AcquirerProxy(BaseProxy):
+    _exposed_ = ('acquire', 'release')
+    def acquire(self, blocking=True):
+        return self._callmethod('acquire', (blocking,))
+    def release(self):
+        return self._callmethod('release')
+    def __enter__(self):
+        return self._callmethod('acquire')
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return self._callmethod('release')
+
+
+class ConditionProxy(AcquirerProxy):
+    # XXX will Condition.notfyAll() name be available in Py3.0?
+    _exposed_ = ('acquire', 'release', 'wait', 'notify', 'notify_all')
+    def wait(self, timeout=None):
+        return self._callmethod('wait', (timeout,))
+    def notify(self):
+        return self._callmethod('notify')
+    def notify_all(self):
+        return self._callmethod('notify_all')
+
+class EventProxy(BaseProxy):
+    # XXX will Event.isSet name be available in Py3.0?
+    _exposed_ = ('isSet', 'set', 'clear', 'wait')
+    def is_set(self):
+        return self._callmethod('isSet')
+    def set(self):
+        return self._callmethod('set')
+    def clear(self):
+        return self._callmethod('clear')
+    def wait(self, timeout=None):
+        return self._callmethod('wait', (timeout,))
+
+class NamespaceProxy(BaseProxy):
+    _exposed_ = ('__getattribute__', '__setattr__', '__delattr__')
+    def __getattr__(self, key):
+        if key[0] == '_':
+            return object.__getattribute__(self, key)
+        callmethod = object.__getattribute__(self, '_callmethod')
+        return callmethod('__getattribute__', (key,))
+    def __setattr__(self, key, value):
+        if key[0] == '_':
+            return object.__setattr__(self, key, value)
+        callmethod = object.__getattribute__(self, '_callmethod')
+        return callmethod('__setattr__', (key, value))
+    def __delattr__(self, key):
+        if key[0] == '_':
+            return object.__delattr__(self, key)
+        callmethod = object.__getattribute__(self, '_callmethod')
+        return callmethod('__delattr__', (key,))
+
+
+class ValueProxy(BaseProxy):
+    _exposed_ = ('get', 'set')
+    def get(self):
+        return self._callmethod('get')
+    def set(self, value):
+        return self._callmethod('set', (value,))
+    value = property(get, set)
+
+
+BaseListProxy = MakeProxyType('BaseListProxy', (
+    '__add__', '__contains__', '__delitem__', '__delslice__',
+    '__getitem__', '__getslice__', '__len__', '__mul__',
+    '__reversed__', '__rmul__', '__setitem__', '__setslice__',
+    'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove',
+    'reverse', 'sort', '__imul__'
+    ))                  # XXX __getslice__ and __setslice__ unneeded in Py3.0
+class ListProxy(BaseListProxy):
+    def __iadd__(self, value):
+        self._callmethod('extend', (value,))
+        return self
+    def __imul__(self, value):
+        self._callmethod('__imul__', (value,))
+        return self
+
+
+DictProxy = MakeProxyType('DictProxy', (
+    '__contains__', '__delitem__', '__getitem__', '__len__',
+    '__setitem__', 'clear', 'copy', 'get', 'has_key', 'items',
+    'keys', 'pop', 'popitem', 'setdefault', 'update', 'values'
+    ))
+
+
+ArrayProxy = MakeProxyType('ArrayProxy', (
+    '__len__', '__getitem__', '__setitem__', '__getslice__', '__setslice__'
+    ))                  # XXX __getslice__ and __setslice__ unneeded in Py3.0
+
+
+PoolProxy = MakeProxyType('PoolProxy', (
+    'apply', 'apply_async', 'close', 'imap', 'imap_unordered', 'join',
+    'map', 'map_async', 'terminate'
+    ))
+PoolProxy._method_to_typeid_ = {
+    'apply_async': 'AsyncResult',
+    'map_async': 'AsyncResult',
+    'imap': 'Iterator',
+    'imap_unordered': 'Iterator'
+    }
+
+#
+# Definition of SyncManager
+#
+
+class SyncManager(BaseManager):
+    '''
+    Subclass of `BaseManager` which supports a number of shared object types.
+
+    The types registered are those intended for the synchronization
+    of threads, plus `dict`, `list` and `Namespace`.
+
+    The `multiprocessing.Manager()` function creates started instances of
+    this class.
+    '''
+
+SyncManager.register('Queue', Queue.Queue)
+SyncManager.register('JoinableQueue', Queue.Queue)
+SyncManager.register('Event', threading.Event, EventProxy)
+SyncManager.register('Lock', threading.Lock, AcquirerProxy)
+SyncManager.register('RLock', threading.RLock, AcquirerProxy)
+SyncManager.register('Semaphore', threading.Semaphore, AcquirerProxy)
+SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore,
+                     AcquirerProxy)
+SyncManager.register('Condition', threading.Condition, ConditionProxy)
+SyncManager.register('Pool', Pool, PoolProxy)
+SyncManager.register('list', list, ListProxy)
+SyncManager.register('dict', dict, DictProxy)
+SyncManager.register('Value', Value, ValueProxy)
+SyncManager.register('Array', Array, ArrayProxy)
+SyncManager.register('Namespace', Namespace, NamespaceProxy)
+
+# types returned by methods of PoolProxy
+SyncManager.register('Iterator', proxytype=IteratorProxy, create_method=False)
+SyncManager.register('AsyncResult', create_method=False)
index 79f0a2929ca2c63140f2af297b4517f0b42296fc..8aaec63d00fe6f74cb1cdcf2dc730f974a52a890 100644 (file)
-#\r
-# Module providing the `Pool` class for managing a process pool\r
-#\r
-# multiprocessing/pool.py\r
-#\r
-# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt\r
-#\r
-\r
-__all__ = ['Pool']\r
-\r
-#\r
-# Imports\r
-#\r
-\r
-import threading\r
-import Queue\r
-import itertools\r
-import collections\r
-import time\r
-\r
-from multiprocessing import Process, cpu_count, TimeoutError\r
-from multiprocessing.util import Finalize, debug\r
-\r
-#\r
-# Constants representing the state of a pool\r
-#\r
-\r
-RUN = 0\r
-CLOSE = 1\r
-TERMINATE = 2\r
-\r
-#\r
-# Miscellaneous\r
-#\r
-\r
-job_counter = itertools.count()\r
-\r
-def mapstar(args):\r
-    return map(*args)\r
-\r
-#\r
-# Code run by worker processes\r
-#\r
-\r
-def worker(inqueue, outqueue, initializer=None, initargs=()):\r
-    put = outqueue.put\r
-    get = inqueue.get\r
-    if hasattr(inqueue, '_writer'):\r
-        inqueue._writer.close()\r
-        outqueue._reader.close()\r
-\r
-    if initializer is not None:\r
-        initializer(*initargs)\r
-\r
-    while 1:\r
-        try:\r
-            task = get()\r
-        except (EOFError, IOError):\r
-            debug('worker got EOFError or IOError -- exiting')\r
-            break\r
-\r
-        if task is None:\r
-            debug('worker got sentinel -- exiting')\r
-            break\r
-\r
-        job, i, func, args, kwds = task\r
-        try:\r
-            result = (True, func(*args, **kwds))\r
-        except Exception, e:\r
-            result = (False, e)\r
-        put((job, i, result))\r
-\r
-#\r
-# Class representing a process pool\r
-#\r
-\r
-class Pool(object):\r
-    '''\r
-    Class which supports an async version of the `apply()` builtin\r
-    '''\r
-    Process = Process\r
-\r
-    def __init__(self, processes=None, initializer=None, initargs=()):\r
-        self._setup_queues()\r
-        self._taskqueue = Queue.Queue()\r
-        self._cache = {}\r
-        self._state = RUN\r
-\r
-        if processes is None:\r
-            try:\r
-                processes = cpu_count()\r
-            except NotImplementedError:\r
-                processes = 1\r
-\r
-        self._pool = []\r
-        for i in range(processes):\r
-            w = self.Process(\r
-                target=worker,\r
-                args=(self._inqueue, self._outqueue, initializer, initargs)\r
-                )\r
-            self._pool.append(w)\r
-            w.set_name(w.get_name().replace('Process', 'PoolWorker'))\r
-            w.set_daemon(True)\r
-            w.start()\r
-\r
-        self._task_handler = threading.Thread(\r
-            target=Pool._handle_tasks,\r
-            args=(self._taskqueue, self._quick_put, self._outqueue, self._pool)\r
-            )\r
-        self._task_handler.set_daemon(True)\r
-        self._task_handler._state = RUN\r
-        self._task_handler.start()\r
-\r
-        self._result_handler = threading.Thread(\r
-            target=Pool._handle_results,\r
-            args=(self._outqueue, self._quick_get, self._cache)\r
-            )\r
-        self._result_handler.set_daemon(True)\r
-        self._result_handler._state = RUN\r
-        self._result_handler.start()\r
-\r
-        self._terminate = Finalize(\r
-            self, self._terminate_pool,\r
-            args=(self._taskqueue, self._inqueue, self._outqueue, self._pool,\r
-                  self._task_handler, self._result_handler, self._cache),\r
-            exitpriority=15\r
-            )\r
-\r
-    def _setup_queues(self):\r
-        from .queues import SimpleQueue\r
-        self._inqueue = SimpleQueue()\r
-        self._outqueue = SimpleQueue()\r
-        self._quick_put = self._inqueue._writer.send\r
-        self._quick_get = self._outqueue._reader.recv\r
-\r
-    def apply(self, func, args=(), kwds={}):\r
-        '''\r
-        Equivalent of `apply()` builtin\r
-        '''\r
-        assert self._state == RUN\r
-        return self.apply_async(func, args, kwds).get()\r
-\r
-    def map(self, func, iterable, chunksize=None):\r
-        '''\r
-        Equivalent of `map()` builtin\r
-        '''\r
-        assert self._state == RUN\r
-        return self.map_async(func, iterable, chunksize).get()\r
-\r
-    def imap(self, func, iterable, chunksize=1):\r
-        '''\r
-        Equivalent of `itertool.imap()` -- can be MUCH slower than `Pool.map()`\r
-        '''\r
-        assert self._state == RUN\r
-        if chunksize == 1:\r
-            result = IMapIterator(self._cache)\r
-            self._taskqueue.put((((result._job, i, func, (x,), {})\r
-                         for i, x in enumerate(iterable)), result._set_length))\r
-            return result\r
-        else:\r
-            assert chunksize > 1\r
-            task_batches = Pool._get_tasks(func, iterable, chunksize)\r
-            result = IMapIterator(self._cache)\r
-            self._taskqueue.put((((result._job, i, mapstar, (x,), {})\r
-                     for i, x in enumerate(task_batches)), result._set_length))\r
-            return (item for chunk in result for item in chunk)\r
-\r
-    def imap_unordered(self, func, iterable, chunksize=1):\r
-        '''\r
-        Like `imap()` method but ordering of results is arbitrary\r
-        '''\r
-        assert self._state == RUN\r
-        if chunksize == 1:\r
-            result = IMapUnorderedIterator(self._cache)\r
-            self._taskqueue.put((((result._job, i, func, (x,), {})\r
-                         for i, x in enumerate(iterable)), result._set_length))\r
-            return result\r
-        else:\r
-            assert chunksize > 1\r
-            task_batches = Pool._get_tasks(func, iterable, chunksize)\r
-            result = IMapUnorderedIterator(self._cache)\r
-            self._taskqueue.put((((result._job, i, mapstar, (x,), {})\r
-                     for i, x in enumerate(task_batches)), result._set_length))\r
-            return (item for chunk in result for item in chunk)\r
-\r
-    def apply_async(self, func, args=(), kwds={}, callback=None):\r
-        '''\r
-        Asynchronous equivalent of `apply()` builtin\r
-        '''\r
-        assert self._state == RUN\r
-        result = ApplyResult(self._cache, callback)\r
-        self._taskqueue.put(([(result._job, None, func, args, kwds)], None))\r
-        return result\r
-\r
-    def map_async(self, func, iterable, chunksize=None, callback=None):\r
-        '''\r
-        Asynchronous equivalent of `map()` builtin\r
-        '''\r
-        assert self._state == RUN\r
-        if not hasattr(iterable, '__len__'):\r
-            iterable = list(iterable)\r
-\r
-        if chunksize is None:\r
-            chunksize, extra = divmod(len(iterable), len(self._pool) * 4)\r
-            if extra:\r
-                chunksize += 1\r
-\r
-        task_batches = Pool._get_tasks(func, iterable, chunksize)\r
-        result = MapResult(self._cache, chunksize, len(iterable), callback)\r
-        self._taskqueue.put((((result._job, i, mapstar, (x,), {})\r
-                              for i, x in enumerate(task_batches)), None))\r
-        return result\r
-\r
-    @staticmethod\r
-    def _handle_tasks(taskqueue, put, outqueue, pool):\r
-        thread = threading.current_thread()\r
-\r
-        for taskseq, set_length in iter(taskqueue.get, None):\r
-            i = -1\r
-            for i, task in enumerate(taskseq):\r
-                if thread._state:\r
-                    debug('task handler found thread._state != RUN')\r
-                    break\r
-                try:\r
-                    put(task)\r
-                except IOError:\r
-                    debug('could not put task on queue')\r
-                    break\r
-            else:\r
-                if set_length:\r
-                    debug('doing set_length()')\r
-                    set_length(i+1)\r
-                continue\r
-            break\r
-        else:\r
-            debug('task handler got sentinel')\r
-\r
-\r
-        try:\r
-            # tell result handler to finish when cache is empty\r
-            debug('task handler sending sentinel to result handler')\r
-            outqueue.put(None)\r
-\r
-            # tell workers there is no more work\r
-            debug('task handler sending sentinel to workers')\r
-            for p in pool:\r
-                put(None)\r
-        except IOError:\r
-            debug('task handler got IOError when sending sentinels')\r
-\r
-        debug('task handler exiting')\r
-\r
-    @staticmethod\r
-    def _handle_results(outqueue, get, cache):\r
-        thread = threading.current_thread()\r
-\r
-        while 1:\r
-            try:\r
-                task = get()\r
-            except (IOError, EOFError):\r
-                debug('result handler got EOFError/IOError -- exiting')\r
-                return\r
-\r
-            if thread._state:\r
-                assert thread._state == TERMINATE\r
-                debug('result handler found thread._state=TERMINATE')\r
-                break\r
-\r
-            if task is None:\r
-                debug('result handler got sentinel')\r
-                break\r
-\r
-            job, i, obj = task\r
-            try:\r
-                cache[job]._set(i, obj)\r
-            except KeyError:\r
-                pass\r
-\r
-        while cache and thread._state != TERMINATE:\r
-            try:\r
-                task = get()\r
-            except (IOError, EOFError):\r
-                debug('result handler got EOFError/IOError -- exiting')\r
-                return\r
-\r
-            if task is None:\r
-                debug('result handler ignoring extra sentinel')\r
-                continue\r
-            job, i, obj = task\r
-            try:\r
-                cache[job]._set(i, obj)\r
-            except KeyError:\r
-                pass\r
-\r
-        if hasattr(outqueue, '_reader'):\r
-            debug('ensuring that outqueue is not full')\r
-            # If we don't make room available in outqueue then\r
-            # attempts to add the sentinel (None) to outqueue may\r
-            # block.  There is guaranteed to be no more than 2 sentinels.\r
-            try:\r
-                for i in range(10):\r
-                    if not outqueue._reader.poll():\r
-                        break\r
-                    get()\r
-            except (IOError, EOFError):\r
-                pass\r
-\r
-        debug('result handler exiting: len(cache)=%s, thread._state=%s',\r
-              len(cache), thread._state)\r
-\r
-    @staticmethod\r
-    def _get_tasks(func, it, size):\r
-        it = iter(it)\r
-        while 1:\r
-            x = tuple(itertools.islice(it, size))\r
-            if not x:\r
-                return\r
-            yield (func, x)\r
-\r
-    def __reduce__(self):\r
-        raise NotImplementedError(\r
-              'pool objects cannot be passed between processes or pickled'\r
-              )\r
-\r
-    def close(self):\r
-        debug('closing pool')\r
-        if self._state == RUN:\r
-            self._state = CLOSE\r
-            self._taskqueue.put(None)\r
-\r
-    def terminate(self):\r
-        debug('terminating pool')\r
-        self._state = TERMINATE\r
-        self._terminate()\r
-\r
-    def join(self):\r
-        debug('joining pool')\r
-        assert self._state in (CLOSE, TERMINATE)\r
-        self._task_handler.join()\r
-        self._result_handler.join()\r
-        for p in self._pool:\r
-            p.join()\r
-\r
-    @staticmethod\r
-    def _help_stuff_finish(inqueue, task_handler, size):\r
-        # task_handler may be blocked trying to put items on inqueue\r
-        debug('removing tasks from inqueue until task handler finished')\r
-        inqueue._rlock.acquire()\r
-        while task_handler.is_alive() and inqueue._reader.poll():\r
-            inqueue._reader.recv()\r
-            time.sleep(0)\r
-\r
-    @classmethod\r
-    def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool,\r
-                        task_handler, result_handler, cache):\r
-        # this is guaranteed to only be called once\r
-        debug('finalizing pool')\r
-\r
-        task_handler._state = TERMINATE\r
-        taskqueue.put(None)                 # sentinel\r
-\r
-        debug('helping task handler/workers to finish')\r
-        cls._help_stuff_finish(inqueue, task_handler, len(pool))\r
-\r
-        assert result_handler.is_alive() or len(cache) == 0\r
-\r
-        result_handler._state = TERMINATE\r
-        outqueue.put(None)                  # sentinel\r
-\r
-        if pool and hasattr(pool[0], 'terminate'):\r
-            debug('terminating workers')\r
-            for p in pool:\r
-                p.terminate()\r
-\r
-        debug('joining task handler')\r
-        task_handler.join(1e100)\r
-\r
-        debug('joining result handler')\r
-        result_handler.join(1e100)\r
-\r
-        if pool and hasattr(pool[0], 'terminate'):\r
-            debug('joining pool workers')\r
-            for p in pool:\r
-                p.join()\r
-\r
-#\r
-# Class whose instances are returned by `Pool.apply_async()`\r
-#\r
-\r
-class ApplyResult(object):\r
-\r
-    def __init__(self, cache, callback):\r
-        self._cond = threading.Condition(threading.Lock())\r
-        self._job = job_counter.next()\r
-        self._cache = cache\r
-        self._ready = False\r
-        self._callback = callback\r
-        cache[self._job] = self\r
-\r
-    def ready(self):\r
-        return self._ready\r
-\r
-    def successful(self):\r
-        assert self._ready\r
-        return self._success\r
-\r
-    def wait(self, timeout=None):\r
-        self._cond.acquire()\r
-        try:\r
-            if not self._ready:\r
-                self._cond.wait(timeout)\r
-        finally:\r
-            self._cond.release()\r
-\r
-    def get(self, timeout=None):\r
-        self.wait(timeout)\r
-        if not self._ready:\r
-            raise TimeoutError\r
-        if self._success:\r
-            return self._value\r
-        else:\r
-            raise self._value\r
-\r
-    def _set(self, i, obj):\r
-        self._success, self._value = obj\r
-        if self._callback and self._success:\r
-            self._callback(self._value)\r
-        self._cond.acquire()\r
-        try:\r
-            self._ready = True\r
-            self._cond.notify()\r
-        finally:\r
-            self._cond.release()\r
-        del self._cache[self._job]\r
-\r
-#\r
-# Class whose instances are returned by `Pool.map_async()`\r
-#\r
-\r
-class MapResult(ApplyResult):\r
-\r
-    def __init__(self, cache, chunksize, length, callback):\r
-        ApplyResult.__init__(self, cache, callback)\r
-        self._success = True\r
-        self._value = [None] * length\r
-        self._chunksize = chunksize\r
-        if chunksize <= 0:\r
-            self._number_left = 0\r
-            self._ready = True\r
-        else:\r
-            self._number_left = length//chunksize + bool(length % chunksize)\r
-\r
-    def _set(self, i, success_result):\r
-        success, result = success_result\r
-        if success:\r
-            self._value[i*self._chunksize:(i+1)*self._chunksize] = result\r
-            self._number_left -= 1\r
-            if self._number_left == 0:\r
-                if self._callback:\r
-                    self._callback(self._value)\r
-                del self._cache[self._job]\r
-                self._cond.acquire()\r
-                try:\r
-                    self._ready = True\r
-                    self._cond.notify()\r
-                finally:\r
-                    self._cond.release()\r
-\r
-        else:\r
-            self._success = False\r
-            self._value = result\r
-            del self._cache[self._job]\r
-            self._cond.acquire()\r
-            try:\r
-                self._ready = True\r
-                self._cond.notify()\r
-            finally:\r
-                self._cond.release()\r
-\r
-#\r
-# Class whose instances are returned by `Pool.imap()`\r
-#\r
-\r
-class IMapIterator(object):\r
-\r
-    def __init__(self, cache):\r
-        self._cond = threading.Condition(threading.Lock())\r
-        self._job = job_counter.next()\r
-        self._cache = cache\r
-        self._items = collections.deque()\r
-        self._index = 0\r
-        self._length = None\r
-        self._unsorted = {}\r
-        cache[self._job] = self\r
-\r
-    def __iter__(self):\r
-        return self\r
-\r
-    def next(self, timeout=None):\r
-        self._cond.acquire()\r
-        try:\r
-            try:\r
-                item = self._items.popleft()\r
-            except IndexError:\r
-                if self._index == self._length:\r
-                    raise StopIteration\r
-                self._cond.wait(timeout)\r
-                try:\r
-                    item = self._items.popleft()\r
-                except IndexError:\r
-                    if self._index == self._length:\r
-                        raise StopIteration\r
-                    raise TimeoutError\r
-        finally:\r
-            self._cond.release()\r
-\r
-        success, value = item\r
-        if success:\r
-            return value\r
-        raise value\r
-\r
-    __next__ = next                    # XXX\r
-\r
-    def _set(self, i, obj):\r
-        self._cond.acquire()\r
-        try:\r
-            if self._index == i:\r
-                self._items.append(obj)\r
-                self._index += 1\r
-                while self._index in self._unsorted:\r
-                    obj = self._unsorted.pop(self._index)\r
-                    self._items.append(obj)\r
-                    self._index += 1\r
-                self._cond.notify()\r
-            else:\r
-                self._unsorted[i] = obj\r
-\r
-            if self._index == self._length:\r
-                del self._cache[self._job]\r
-        finally:\r
-            self._cond.release()\r
-\r
-    def _set_length(self, length):\r
-        self._cond.acquire()\r
-        try:\r
-            self._length = length\r
-            if self._index == self._length:\r
-                self._cond.notify()\r
-                del self._cache[self._job]\r
-        finally:\r
-            self._cond.release()\r
-\r
-#\r
-# Class whose instances are returned by `Pool.imap_unordered()`\r
-#\r
-\r
-class IMapUnorderedIterator(IMapIterator):\r
-\r
-    def _set(self, i, obj):\r
-        self._cond.acquire()\r
-        try:\r
-            self._items.append(obj)\r
-            self._index += 1\r
-            self._cond.notify()\r
-            if self._index == self._length:\r
-                del self._cache[self._job]\r
-        finally:\r
-            self._cond.release()\r
-\r
-#\r
-#\r
-#\r
-\r
-class ThreadPool(Pool):\r
-\r
-    from .dummy import Process\r
-\r
-    def __init__(self, processes=None, initializer=None, initargs=()):\r
-        Pool.__init__(self, processes, initializer, initargs)\r
-\r
-    def _setup_queues(self):\r
-        self._inqueue = Queue.Queue()\r
-        self._outqueue = Queue.Queue()\r
-        self._quick_put = self._inqueue.put\r
-        self._quick_get = self._outqueue.get\r
-\r
-    @staticmethod\r
-    def _help_stuff_finish(inqueue, task_handler, size):\r
-        # put sentinels at head of inqueue to make workers finish\r
-        inqueue.not_empty.acquire()\r
-        try:\r
-            inqueue.queue.clear()\r
-            inqueue.queue.extend([None] * size)\r
-            inqueue.not_empty.notify_all()\r
-        finally:\r
-            inqueue.not_empty.release()\r
+#
+# Module providing the `Pool` class for managing a process pool
+#
+# multiprocessing/pool.py
+#
+# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt
+#
+
+__all__ = ['Pool']
+
+#
+# Imports
+#
+
+import threading
+import Queue
+import itertools
+import collections
+import time
+
+from multiprocessing import Process, cpu_count, TimeoutError
+from multiprocessing.util import Finalize, debug
+
+#
+# Constants representing the state of a pool
+#
+
+RUN = 0
+CLOSE = 1
+TERMINATE = 2
+
+#
+# Miscellaneous
+#
+
+job_counter = itertools.count()
+
+def mapstar(args):
+    return map(*args)
+
+#
+# Code run by worker processes
+#
+
+def worker(inqueue, outqueue, initializer=None, initargs=()):
+    put = outqueue.put
+    get = inqueue.get
+    if hasattr(inqueue, '_writer'):
+        inqueue._writer.close()
+        outqueue._reader.close()
+
+    if initializer is not None:
+        initializer(*initargs)
+
+    while 1:
+        try:
+            task = get()
+        except (EOFError, IOError):
+            debug('worker got EOFError or IOError -- exiting')
+            break
+
+        if task is None:
+            debug('worker got sentinel -- exiting')
+            break
+
+        job, i, func, args, kwds = task
+        try:
+            result = (True, func(*args, **kwds))
+        except Exception, e:
+            result = (False, e)
+        put((job, i, result))
+
+#
+# Class representing a process pool
+#
+
+class Pool(object):
+    '''
+    Class which supports an async version of the `apply()` builtin
+    '''
+    Process = Process
+
+    def __init__(self, processes=None, initializer=None, initargs=()):
+        self._setup_queues()
+        self._taskqueue = Queue.Queue()
+        self._cache = {}
+        self._state = RUN
+
+        if processes is None:
+            try:
+                processes = cpu_count()
+            except NotImplementedError:
+                processes = 1
+
+        self._pool = []
+        for i in range(processes):
+            w = self.Process(
+                target=worker,
+                args=(self._inqueue, self._outqueue, initializer, initargs)
+                )
+            self._pool.append(w)
+            w.set_name(w.get_name().replace('Process', 'PoolWorker'))
+            w.set_daemon(True)
+            w.start()
+
+        self._task_handler = threading.Thread(
+            target=Pool._handle_tasks,
+            args=(self._taskqueue, self._quick_put, self._outqueue, self._pool)
+            )
+        self._task_handler.set_daemon(True)
+        self._task_handler._state = RUN
+        self._task_handler.start()
+
+        self._result_handler = threading.Thread(
+            target=Pool._handle_results,
+            args=(self._outqueue, self._quick_get, self._cache)
+            )
+        self._result_handler.set_daemon(True)
+        self._result_handler._state = RUN
+        self._result_handler.start()
+
+        self._terminate = Finalize(
+            self, self._terminate_pool,
+            args=(self._taskqueue, self._inqueue, self._outqueue, self._pool,
+                  self._task_handler, self._result_handler, self._cache),
+            exitpriority=15
+            )
+
+    def _setup_queues(self):
+        from .queues import SimpleQueue
+        self._inqueue = SimpleQueue()
+        self._outqueue = SimpleQueue()
+        self._quick_put = self._inqueue._writer.send
+        self._quick_get = self._outqueue._reader.recv
+
+    def apply(self, func, args=(), kwds={}):
+        '''
+        Equivalent of `apply()` builtin
+        '''
+        assert self._state == RUN
+        return self.apply_async(func, args, kwds).get()
+
+    def map(self, func, iterable, chunksize=None):
+        '''
+        Equivalent of `map()` builtin
+        '''
+        assert self._state == RUN
+        return self.map_async(func, iterable, chunksize).get()
+
+    def imap(self, func, iterable, chunksize=1):
+        '''
+        Equivalent of `itertool.imap()` -- can be MUCH slower than `Pool.map()`
+        '''
+        assert self._state == RUN
+        if chunksize == 1:
+            result = IMapIterator(self._cache)
+            self._taskqueue.put((((result._job, i, func, (x,), {})
+                         for i, x in enumerate(iterable)), result._set_length))
+            return result
+        else:
+            assert chunksize > 1
+            task_batches = Pool._get_tasks(func, iterable, chunksize)
+            result = IMapIterator(self._cache)
+            self._taskqueue.put((((result._job, i, mapstar, (x,), {})
+                     for i, x in enumerate(task_batches)), result._set_length))
+            return (item for chunk in result for item in chunk)
+
+    def imap_unordered(self, func, iterable, chunksize=1):
+        '''
+        Like `imap()` method but ordering of results is arbitrary
+        '''
+        assert self._state == RUN
+        if chunksize == 1:
+            result = IMapUnorderedIterator(self._cache)
+            self._taskqueue.put((((result._job, i, func, (x,), {})
+                         for i, x in enumerate(iterable)), result._set_length))
+            return result
+        else:
+            assert chunksize > 1
+            task_batches = Pool._get_tasks(func, iterable, chunksize)
+            result = IMapUnorderedIterator(self._cache)
+            self._taskqueue.put((((result._job, i, mapstar, (x,), {})
+                     for i, x in enumerate(task_batches)), result._set_length))
+            return (item for chunk in result for item in chunk)
+
+    def apply_async(self, func, args=(), kwds={}, callback=None):
+        '''
+        Asynchronous equivalent of `apply()` builtin
+        '''
+        assert self._state == RUN
+        result = ApplyResult(self._cache, callback)
+        self._taskqueue.put(([(result._job, None, func, args, kwds)], None))
+        return result
+
+    def map_async(self, func, iterable, chunksize=None, callback=None):
+        '''
+        Asynchronous equivalent of `map()` builtin
+        '''
+        assert self._state == RUN
+        if not hasattr(iterable, '__len__'):
+            iterable = list(iterable)
+
+        if chunksize is None:
+            chunksize, extra = divmod(len(iterable), len(self._pool) * 4)
+            if extra:
+                chunksize += 1
+
+        task_batches = Pool._get_tasks(func, iterable, chunksize)
+        result = MapResult(self._cache, chunksize, len(iterable), callback)
+        self._taskqueue.put((((result._job, i, mapstar, (x,), {})
+                              for i, x in enumerate(task_batches)), None))
+        return result
+
+    @staticmethod
+    def _handle_tasks(taskqueue, put, outqueue, pool):
+        thread = threading.current_thread()
+
+        for taskseq, set_length in iter(taskqueue.get, None):
+            i = -1
+            for i, task in enumerate(taskseq):
+                if thread._state:
+                    debug('task handler found thread._state != RUN')
+                    break
+                try:
+                    put(task)
+                except IOError:
+                    debug('could not put task on queue')
+                    break
+            else:
+                if set_length:
+                    debug('doing set_length()')
+                    set_length(i+1)
+                continue
+            break
+        else:
+            debug('task handler got sentinel')
+
+
+        try:
+            # tell result handler to finish when cache is empty
+            debug('task handler sending sentinel to result handler')
+            outqueue.put(None)
+
+            # tell workers there is no more work
+            debug('task handler sending sentinel to workers')
+            for p in pool:
+                put(None)
+        except IOError:
+            debug('task handler got IOError when sending sentinels')
+
+        debug('task handler exiting')
+
+    @staticmethod
+    def _handle_results(outqueue, get, cache):
+        thread = threading.current_thread()
+
+        while 1:
+            try:
+                task = get()
+            except (IOError, EOFError):
+                debug('result handler got EOFError/IOError -- exiting')
+                return
+
+            if thread._state:
+                assert thread._state == TERMINATE
+                debug('result handler found thread._state=TERMINATE')
+                break
+
+            if task is None:
+                debug('result handler got sentinel')
+                break
+
+            job, i, obj = task
+            try:
+                cache[job]._set(i, obj)
+            except KeyError:
+                pass
+
+        while cache and thread._state != TERMINATE:
+            try:
+                task = get()
+            except (IOError, EOFError):
+                debug('result handler got EOFError/IOError -- exiting')
+                return
+
+            if task is None:
+                debug('result handler ignoring extra sentinel')
+                continue
+            job, i, obj = task
+            try:
+                cache[job]._set(i, obj)
+            except KeyError:
+                pass
+
+        if hasattr(outqueue, '_reader'):
+            debug('ensuring that outqueue is not full')
+            # If we don't make room available in outqueue then
+            # attempts to add the sentinel (None) to outqueue may
+            # block.  There is guaranteed to be no more than 2 sentinels.
+            try:
+                for i in range(10):
+                    if not outqueue._reader.poll():
+                        break
+                    get()
+            except (IOError, EOFError):
+                pass
+
+        debug('result handler exiting: len(cache)=%s, thread._state=%s',
+              len(cache), thread._state)
+
+    @staticmethod
+    def _get_tasks(func, it, size):
+        it = iter(it)
+        while 1:
+            x = tuple(itertools.islice(it, size))
+            if not x:
+                return
+            yield (func, x)
+
+    def __reduce__(self):
+        raise NotImplementedError(
+              'pool objects cannot be passed between processes or pickled'
+              )
+
+    def close(self):
+        debug('closing pool')
+        if self._state == RUN:
+            self._state = CLOSE
+            self._taskqueue.put(None)
+
+    def terminate(self):
+        debug('terminating pool')
+        self._state = TERMINATE
+        self._terminate()
+
+    def join(self):
+        debug('joining pool')
+        assert self._state in (CLOSE, TERMINATE)
+        self._task_handler.join()
+        self._result_handler.join()
+        for p in self._pool:
+            p.join()
+
+    @staticmethod
+    def _help_stuff_finish(inqueue, task_handler, size):
+        # task_handler may be blocked trying to put items on inqueue
+        debug('removing tasks from inqueue until task handler finished')
+        inqueue._rlock.acquire()
+        while task_handler.is_alive() and inqueue._reader.poll():
+            inqueue._reader.recv()
+            time.sleep(0)
+
+    @classmethod
+    def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool,
+                        task_handler, result_handler, cache):
+        # this is guaranteed to only be called once
+        debug('finalizing pool')
+
+        task_handler._state = TERMINATE
+        taskqueue.put(None)                 # sentinel
+
+        debug('helping task handler/workers to finish')
+        cls._help_stuff_finish(inqueue, task_handler, len(pool))
+
+        assert result_handler.is_alive() or len(cache) == 0
+
+        result_handler._state = TERMINATE
+        outqueue.put(None)                  # sentinel
+
+        if pool and hasattr(pool[0], 'terminate'):
+            debug('terminating workers')
+            for p in pool:
+                p.terminate()
+
+        debug('joining task handler')
+        task_handler.join(1e100)
+
+        debug('joining result handler')
+        result_handler.join(1e100)
+
+        if pool and hasattr(pool[0], 'terminate'):
+            debug('joining pool workers')
+            for p in pool:
+                p.join()
+
+#
+# Class whose instances are returned by `Pool.apply_async()`
+#
+
+class ApplyResult(object):
+
+    def __init__(self, cache, callback):
+        self._cond = threading.Condition(threading.Lock())
+        self._job = job_counter.next()
+        self._cache = cache
+        self._ready = False
+        self._callback = callback
+        cache[self._job] = self
+
+    def ready(self):
+        return self._ready
+
+    def successful(self):
+        assert self._ready
+        return self._success
+
+    def wait(self, timeout=None):
+        self._cond.acquire()
+        try:
+            if not self._ready:
+                self._cond.wait(timeout)
+        finally:
+            self._cond.release()
+
+    def get(self, timeout=None):
+        self.wait(timeout)
+        if not self._ready:
+            raise TimeoutError
+        if self._success:
+            return self._value
+        else:
+            raise self._value
+
+    def _set(self, i, obj):
+        self._success, self._value = obj
+        if self._callback and self._success:
+            self._callback(self._value)
+        self._cond.acquire()
+        try:
+            self._ready = True
+            self._cond.notify()
+        finally:
+            self._cond.release()
+        del self._cache[self._job]
+
+#
+# Class whose instances are returned by `Pool.map_async()`
+#
+
+class MapResult(ApplyResult):
+
+    def __init__(self, cache, chunksize, length, callback):
+        ApplyResult.__init__(self, cache, callback)
+        self._success = True
+        self._value = [None] * length
+        self._chunksize = chunksize
+        if chunksize <= 0:
+            self._number_left = 0
+            self._ready = True
+        else:
+            self._number_left = length//chunksize + bool(length % chunksize)
+
+    def _set(self, i, success_result):
+        success, result = success_result
+        if success:
+            self._value[i*self._chunksize:(i+1)*self._chunksize] = result
+            self._number_left -= 1
+            if self._number_left == 0:
+                if self._callback:
+                    self._callback(self._value)
+                del self._cache[self._job]
+                self._cond.acquire()
+                try:
+                    self._ready = True
+                    self._cond.notify()
+                finally:
+                    self._cond.release()
+
+        else:
+            self._success = False
+            self._value = result
+            del self._cache[self._job]
+            self._cond.acquire()
+            try:
+                self._ready = True
+                self._cond.notify()
+            finally:
+                self._cond.release()
+
+#
+# Class whose instances are returned by `Pool.imap()`
+#
+
+class IMapIterator(object):
+
+    def __init__(self, cache):
+        self._cond = threading.Condition(threading.Lock())
+        self._job = job_counter.next()
+        self._cache = cache
+        self._items = collections.deque()
+        self._index = 0
+        self._length = None
+        self._unsorted = {}
+        cache[self._job] = self
+
+    def __iter__(self):
+        return self
+
+    def next(self, timeout=None):
+        self._cond.acquire()
+        try:
+            try:
+                item = self._items.popleft()
+            except IndexError:
+                if self._index == self._length:
+                    raise StopIteration
+                self._cond.wait(timeout)
+                try:
+                    item = self._items.popleft()
+                except IndexError:
+                    if self._index == self._length:
+                        raise StopIteration
+                    raise TimeoutError
+        finally:
+            self._cond.release()
+
+        success, value = item
+        if success:
+            return value
+        raise value
+
+    __next__ = next                    # XXX
+
+    def _set(self, i, obj):
+        self._cond.acquire()
+        try:
+            if self._index == i:
+                self._items.append(obj)
+                self._index += 1
+                while self._index in self._unsorted:
+                    obj = self._unsorted.pop(self._index)
+                    self._items.append(obj)
+                    self._index += 1
+                self._cond.notify()
+            else:
+                self._unsorted[i] = obj
+
+            if self._index == self._length:
+                del self._cache[self._job]
+        finally:
+            self._cond.release()
+
+    def _set_length(self, length):
+        self._cond.acquire()
+        try:
+            self._length = length
+            if self._index == self._length:
+                self._cond.notify()
+                del self._cache[self._job]
+        finally:
+            self._cond.release()
+
+#
+# Class whose instances are returned by `Pool.imap_unordered()`
+#
+
+class IMapUnorderedIterator(IMapIterator):
+
+    def _set(self, i, obj):
+        self._cond.acquire()
+        try:
+            self._items.append(obj)
+            self._index += 1
+            self._cond.notify()
+            if self._index == self._length:
+                del self._cache[self._job]
+        finally:
+            self._cond.release()
+
+#
+#
+#
+
+class ThreadPool(Pool):
+
+    from .dummy import Process
+
+    def __init__(self, processes=None, initializer=None, initargs=()):
+        Pool.__init__(self, processes, initializer, initargs)
+
+    def _setup_queues(self):
+        self._inqueue = Queue.Queue()
+        self._outqueue = Queue.Queue()
+        self._quick_put = self._inqueue.put
+        self._quick_get = self._outqueue.get
+
+    @staticmethod
+    def _help_stuff_finish(inqueue, task_handler, size):
+        # put sentinels at head of inqueue to make workers finish
+        inqueue.not_empty.acquire()
+        try:
+            inqueue.queue.clear()
+            inqueue.queue.extend([None] * size)
+            inqueue.not_empty.notify_all()
+        finally:
+            inqueue.not_empty.release()
index 43d8297672f0eca17c9a146dfee1449ab4839a96..e1189cff78214eef4999c653d0cc6bcd4554ec21 100644 (file)
-#\r
-# Module providing the `Process` class which emulates `threading.Thread`\r
-#\r
-# multiprocessing/process.py\r
-#\r
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt\r
-#\r
-\r
-__all__ = ['Process', 'current_process', 'active_children']\r
-\r
-#\r
-# Imports\r
-#\r
-\r
-import os\r
-import sys\r
-import signal\r
-import itertools\r
-\r
-#\r
-#\r
-#\r
-\r
-try:\r
-    ORIGINAL_DIR = os.path.abspath(os.getcwd())\r
-except OSError:\r
-    ORIGINAL_DIR = None\r
-\r
-try:\r
-    bytes\r
-except NameError:\r
-    bytes = str                  # XXX not needed in Py2.6 and Py3.0\r
-\r
-#\r
-# Public functions\r
-#\r
-\r
-def current_process():\r
-    '''\r
-    Return process object representing the current process\r
-    '''\r
-    return _current_process\r
-\r
-def active_children():\r
-    '''\r
-    Return list of process objects corresponding to live child processes\r
-    '''\r
-    _cleanup()\r
-    return list(_current_process._children)\r
-\r
-#\r
-#\r
-#\r
-\r
-def _cleanup():\r
-    # check for processes which have finished\r
-    for p in list(_current_process._children):\r
-        if p._popen.poll() is not None:\r
-            _current_process._children.discard(p)\r
-\r
-#\r
-# The `Process` class\r
-#\r
-\r
-class Process(object):\r
-    '''\r
-    Process objects represent activity that is run in a separate process\r
-\r
-    The class is analagous to `threading.Thread`\r
-    '''\r
-    _Popen = None\r
-\r
-    def __init__(self, group=None, target=None, name=None, args=(), kwargs={}):\r
-        assert group is None, 'group argument must be None for now'\r
-        count = _current_process._counter.next()\r
-        self._identity = _current_process._identity + (count,)\r
-        self._authkey = _current_process._authkey\r
-        self._daemonic = _current_process._daemonic\r
-        self._tempdir = _current_process._tempdir\r
-        self._parent_pid = os.getpid()\r
-        self._popen = None\r
-        self._target = target\r
-        self._args = tuple(args)\r
-        self._kwargs = dict(kwargs)\r
-        self._name = name or type(self).__name__ + '-' + \\r
-                     ':'.join(str(i) for i in self._identity)\r
-\r
-    def run(self):\r
-        '''\r
-        Method to be run in sub-process; can be overridden in sub-class\r
-        '''\r
-        if self._target:\r
-            self._target(*self._args, **self._kwargs)\r
-\r
-    def start(self):\r
-        '''\r
-        Start child process\r
-        '''\r
-        assert self._popen is None, 'cannot start a process twice'\r
-        assert self._parent_pid == os.getpid(), \\r
-               'can only start a process object created by current process'\r
-        assert not _current_process._daemonic, \\r
-               'daemonic processes are not allowed to have children'\r
-        _cleanup()\r
-        if self._Popen is not None:\r
-            Popen = self._Popen\r
-        else:\r
-            from .forking import Popen\r
-        self._popen = Popen(self)\r
-        _current_process._children.add(self)\r
-\r
-    def terminate(self):\r
-        '''\r
-        Terminate process; sends SIGTERM signal or uses TerminateProcess()\r
-        '''\r
-        self._popen.terminate()\r
-\r
-    def join(self, timeout=None):\r
-        '''\r
-        Wait until child process terminates\r
-        '''\r
-        assert self._parent_pid == os.getpid(), 'can only join a child process'\r
-        assert self._popen is not None, 'can only join a started process'\r
-        res = self._popen.wait(timeout)\r
-        if res is not None:\r
-            _current_process._children.discard(self)\r
-\r
-    def is_alive(self):\r
-        '''\r
-        Return whether process is alive\r
-        '''\r
-        if self is _current_process:\r
-            return True\r
-        assert self._parent_pid == os.getpid(), 'can only test a child process'\r
-        if self._popen is None:\r
-            return False\r
-        self._popen.poll()\r
-        return self._popen.returncode is None\r
-\r
-    def get_name(self):\r
-        '''\r
-        Return name of process\r
-        '''\r
-        return self._name\r
-\r
-    def set_name(self, name):\r
-        '''\r
-        Set name of process\r
-        '''\r
-        assert isinstance(name, str), 'name must be a string'\r
-        self._name = name\r
-\r
-    def is_daemon(self):\r
-        '''\r
-        Return whether process is a daemon\r
-        '''\r
-        return self._daemonic\r
-\r
-    def set_daemon(self, daemonic):\r
-        '''\r
-        Set whether process is a daemon\r
-        '''\r
-        assert self._popen is None, 'process has already started'\r
-        self._daemonic = daemonic\r
-\r
-    def get_authkey(self):\r
-        '''\r
-        Return authorization key of process\r
-        '''\r
-        return self._authkey\r
-\r
-    def set_authkey(self, authkey):\r
-        '''\r
-        Set authorization key of process\r
-        '''\r
-        self._authkey = AuthenticationString(authkey)\r
-\r
-    def get_exitcode(self):\r
-        '''\r
-        Return exit code of process or `None` if it has yet to stop\r
-        '''\r
-        if self._popen is None:\r
-            return self._popen\r
-        return self._popen.poll()\r
-\r
-    def get_ident(self):\r
-        '''\r
-        Return indentifier (PID) of process or `None` if it has yet to start\r
-        '''\r
-        if self is _current_process:\r
-            return os.getpid()\r
-        else:\r
-            return self._popen and self._popen.pid\r
-\r
-    pid = property(get_ident)\r
-\r
-    def __repr__(self):\r
-        if self is _current_process:\r
-            status = 'started'\r
-        elif self._parent_pid != os.getpid():\r
-            status = 'unknown'\r
-        elif self._popen is None:\r
-            status = 'initial'\r
-        else:\r
-            if self._popen.poll() is not None:\r
-                status = self.get_exitcode()\r
-            else:\r
-                status = 'started'\r
-\r
-        if type(status) is int:\r
-            if status == 0:\r
-                status = 'stopped'\r
-            else:\r
-                status = 'stopped[%s]' % _exitcode_to_name.get(status, status)\r
-\r
-        return '<%s(%s, %s%s)>' % (type(self).__name__, self._name,\r
-                                   status, self._daemonic and ' daemon' or '')\r
-\r
-    ##\r
-\r
-    def _bootstrap(self):\r
-        from . import util\r
-        global _current_process\r
-\r
-        try:\r
-            self._children = set()\r
-            self._counter = itertools.count(1)\r
-            try:\r
-                os.close(sys.stdin.fileno())\r
-            except (OSError, ValueError):\r
-                pass\r
-            _current_process = self\r
-            util._finalizer_registry.clear()\r
-            util._run_after_forkers()\r
-            util.info('child process calling self.run()')\r
-            try:\r
-                self.run()\r
-                exitcode = 0\r
-            finally:\r
-                util._exit_function()\r
-        except SystemExit, e:\r
-            if not e.args:\r
-                exitcode = 1\r
-            elif type(e.args[0]) is int:\r
-                exitcode = e.args[0]\r
-            else:\r
-                sys.stderr.write(e.args[0] + '\n')\r
-                sys.stderr.flush()\r
-                exitcode = 1\r
-        except:\r
-            exitcode = 1\r
-            import traceback\r
-            sys.stderr.write('Process %s:\n' % self.get_name())\r
-            sys.stderr.flush()\r
-            traceback.print_exc()\r
-\r
-        util.info('process exiting with exitcode %d' % exitcode)\r
-        return exitcode\r
-\r
-#\r
-# We subclass bytes to avoid accidental transmission of auth keys over network\r
-#\r
-\r
-class AuthenticationString(bytes):\r
-    def __reduce__(self):\r
-        from .forking import Popen\r
-        if not Popen.thread_is_spawning():\r
-            raise TypeError(\r
-                'Pickling an AuthenticationString object is '\r
-                'disallowed for security reasons'\r
-                )\r
-        return AuthenticationString, (bytes(self),)\r
-\r
-#\r
-# Create object representing the main process\r
-#\r
-\r
-class _MainProcess(Process):\r
-\r
-    def __init__(self):\r
-        self._identity = ()\r
-        self._daemonic = False\r
-        self._name = 'MainProcess'\r
-        self._parent_pid = None\r
-        self._popen = None\r
-        self._counter = itertools.count(1)\r
-        self._children = set()\r
-        self._authkey = AuthenticationString(os.urandom(32))\r
-        self._tempdir = None\r
-\r
-_current_process = _MainProcess()\r
-del _MainProcess\r
-\r
-#\r
-# Give names to some return codes\r
-#\r
-\r
-_exitcode_to_name = {}\r
-\r
-for name, signum in signal.__dict__.items():\r
-    if name[:3]=='SIG' and '_' not in name:\r
-        _exitcode_to_name[-signum] = name\r
+#
+# Module providing the `Process` class which emulates `threading.Thread`
+#
+# multiprocessing/process.py
+#
+# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+#
+
+__all__ = ['Process', 'current_process', 'active_children']
+
+#
+# Imports
+#
+
+import os
+import sys
+import signal
+import itertools
+
+#
+#
+#
+
+try:
+    ORIGINAL_DIR = os.path.abspath(os.getcwd())
+except OSError:
+    ORIGINAL_DIR = None
+
+try:
+    bytes
+except NameError:
+    bytes = str                  # XXX not needed in Py2.6 and Py3.0
+
+#
+# Public functions
+#
+
+def current_process():
+    '''
+    Return process object representing the current process
+    '''
+    return _current_process
+
+def active_children():
+    '''
+    Return list of process objects corresponding to live child processes
+    '''
+    _cleanup()
+    return list(_current_process._children)
+
+#
+#
+#
+
+def _cleanup():
+    # check for processes which have finished
+    for p in list(_current_process._children):
+        if p._popen.poll() is not None:
+            _current_process._children.discard(p)
+
+#
+# The `Process` class
+#
+
+class Process(object):
+    '''
+    Process objects represent activity that is run in a separate process
+
+    The class is analagous to `threading.Thread`
+    '''
+    _Popen = None
+
+    def __init__(self, group=None, target=None, name=None, args=(), kwargs={}):
+        assert group is None, 'group argument must be None for now'
+        count = _current_process._counter.next()
+        self._identity = _current_process._identity + (count,)
+        self._authkey = _current_process._authkey
+        self._daemonic = _current_process._daemonic
+        self._tempdir = _current_process._tempdir
+        self._parent_pid = os.getpid()
+        self._popen = None
+        self._target = target
+        self._args = tuple(args)
+        self._kwargs = dict(kwargs)
+        self._name = name or type(self).__name__ + '-' + \
+                     ':'.join(str(i) for i in self._identity)
+
+    def run(self):
+        '''
+        Method to be run in sub-process; can be overridden in sub-class
+        '''
+        if self._target:
+            self._target(*self._args, **self._kwargs)
+
+    def start(self):
+        '''
+        Start child process
+        '''
+        assert self._popen is None, 'cannot start a process twice'
+        assert self._parent_pid == os.getpid(), \
+               'can only start a process object created by current process'
+        assert not _current_process._daemonic, \
+               'daemonic processes are not allowed to have children'
+        _cleanup()
+        if self._Popen is not None:
+            Popen = self._Popen
+        else:
+            from .forking import Popen
+        self._popen = Popen(self)
+        _current_process._children.add(self)
+
+    def terminate(self):
+        '''
+        Terminate process; sends SIGTERM signal or uses TerminateProcess()
+        '''
+        self._popen.terminate()
+
+    def join(self, timeout=None):
+        '''
+        Wait until child process terminates
+        '''
+        assert self._parent_pid == os.getpid(), 'can only join a child process'
+        assert self._popen is not None, 'can only join a started process'
+        res = self._popen.wait(timeout)
+        if res is not None:
+            _current_process._children.discard(self)
+
+    def is_alive(self):
+        '''
+        Return whether process is alive
+        '''
+        if self is _current_process:
+            return True
+        assert self._parent_pid == os.getpid(), 'can only test a child process'
+        if self._popen is None:
+            return False
+        self._popen.poll()
+        return self._popen.returncode is None
+
+    def get_name(self):
+        '''
+        Return name of process
+        '''
+        return self._name
+
+    def set_name(self, name):
+        '''
+        Set name of process
+        '''
+        assert isinstance(name, str), 'name must be a string'
+        self._name = name
+
+    def is_daemon(self):
+        '''
+        Return whether process is a daemon
+        '''
+        return self._daemonic
+
+    def set_daemon(self, daemonic):
+        '''
+        Set whether process is a daemon
+        '''
+        assert self._popen is None, 'process has already started'
+        self._daemonic = daemonic
+
+    def get_authkey(self):
+        '''
+        Return authorization key of process
+        '''
+        return self._authkey
+
+    def set_authkey(self, authkey):
+        '''
+        Set authorization key of process
+        '''
+        self._authkey = AuthenticationString(authkey)
+
+    def get_exitcode(self):
+        '''
+        Return exit code of process or `None` if it has yet to stop
+        '''
+        if self._popen is None:
+            return self._popen
+        return self._popen.poll()
+
+    def get_ident(self):
+        '''
+        Return indentifier (PID) of process or `None` if it has yet to start
+        '''
+        if self is _current_process:
+            return os.getpid()
+        else:
+            return self._popen and self._popen.pid
+
+    pid = property(get_ident)
+
+    def __repr__(self):
+        if self is _current_process:
+            status = 'started'
+        elif self._parent_pid != os.getpid():
+            status = 'unknown'
+        elif self._popen is None:
+            status = 'initial'
+        else:
+            if self._popen.poll() is not None:
+                status = self.get_exitcode()
+            else:
+                status = 'started'
+
+        if type(status) is int:
+            if status == 0:
+                status = 'stopped'
+            else:
+                status = 'stopped[%s]' % _exitcode_to_name.get(status, status)
+
+        return '<%s(%s, %s%s)>' % (type(self).__name__, self._name,
+                                   status, self._daemonic and ' daemon' or '')
+
+    ##
+
+    def _bootstrap(self):
+        from . import util
+        global _current_process
+
+        try:
+            self._children = set()
+            self._counter = itertools.count(1)
+            try:
+                os.close(sys.stdin.fileno())
+            except (OSError, ValueError):
+                pass
+            _current_process = self
+            util._finalizer_registry.clear()
+            util._run_after_forkers()
+            util.info('child process calling self.run()')
+            try:
+                self.run()
+                exitcode = 0
+            finally:
+                util._exit_function()
+        except SystemExit, e:
+            if not e.args:
+                exitcode = 1
+            elif type(e.args[0]) is int:
+                exitcode = e.args[0]
+            else:
+                sys.stderr.write(e.args[0] + '\n')
+                sys.stderr.flush()
+                exitcode = 1
+        except:
+            exitcode = 1
+            import traceback
+            sys.stderr.write('Process %s:\n' % self.get_name())
+            sys.stderr.flush()
+            traceback.print_exc()
+
+        util.info('process exiting with exitcode %d' % exitcode)
+        return exitcode
+
+#
+# We subclass bytes to avoid accidental transmission of auth keys over network
+#
+
+class AuthenticationString(bytes):
+    def __reduce__(self):
+        from .forking import Popen
+        if not Popen.thread_is_spawning():
+            raise TypeError(
+                'Pickling an AuthenticationString object is '
+                'disallowed for security reasons'
+                )
+        return AuthenticationString, (bytes(self),)
+
+#
+# Create object representing the main process
+#
+
+class _MainProcess(Process):
+
+    def __init__(self):
+        self._identity = ()
+        self._daemonic = False
+        self._name = 'MainProcess'
+        self._parent_pid = None
+        self._popen = None
+        self._counter = itertools.count(1)
+        self._children = set()
+        self._authkey = AuthenticationString(os.urandom(32))
+        self._tempdir = None
+
+_current_process = _MainProcess()
+del _MainProcess
+
+#
+# Give names to some return codes
+#
+
+_exitcode_to_name = {}
+
+for name, signum in signal.__dict__.items():
+    if name[:3]=='SIG' and '_' not in name:
+        _exitcode_to_name[-signum] = name
index ea8909055932f34f6a63723c6e47447923dba4f7..7235a41c76731afe3ab6e86ceadb503e925fdcb9 100644 (file)
-#\r
-# Module implementing queues\r
-#\r
-# multiprocessing/queues.py\r
-#\r
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt\r
-#\r
-\r
-__all__ = ['Queue', 'SimpleQueue']\r
-\r
-import sys\r
-import os\r
-import threading\r
-import collections\r
-import time\r
-import atexit\r
-import weakref\r
-\r
-from Queue import Empty, Full\r
-import _multiprocessing\r
-from multiprocessing import Pipe\r
-from multiprocessing.synchronize import Lock, BoundedSemaphore, Semaphore, Condition\r
-from multiprocessing.util import debug, info, Finalize, register_after_fork\r
-from multiprocessing.forking import assert_spawning\r
-\r
-#\r
-# Queue type using a pipe, buffer and thread\r
-#\r
-\r
-class Queue(object):\r
-\r
-    def __init__(self, maxsize=0):\r
-        if maxsize <= 0:\r
-            maxsize = _multiprocessing.SemLock.SEM_VALUE_MAX\r
-        self._maxsize = maxsize\r
-        self._reader, self._writer = Pipe(duplex=False)\r
-        self._rlock = Lock()\r
-        self._opid = os.getpid()\r
-        if sys.platform == 'win32':\r
-            self._wlock = None\r
-        else:\r
-            self._wlock = Lock()\r
-        self._sem = BoundedSemaphore(maxsize)\r
-\r
-        self._after_fork()\r
-\r
-        if sys.platform != 'win32':\r
-            register_after_fork(self, Queue._after_fork)\r
-\r
-    def __getstate__(self):\r
-        assert_spawning(self)\r
-        return (self._maxsize, self._reader, self._writer,\r
-                self._rlock, self._wlock, self._sem, self._opid)\r
-\r
-    def __setstate__(self, state):\r
-        (self._maxsize, self._reader, self._writer,\r
-         self._rlock, self._wlock, self._sem, self._opid) = state\r
-        self._after_fork()\r
-\r
-    def _after_fork(self):\r
-        debug('Queue._after_fork()')\r
-        self._notempty = threading.Condition(threading.Lock())\r
-        self._buffer = collections.deque()\r
-        self._thread = None\r
-        self._jointhread = None\r
-        self._joincancelled = False\r
-        self._closed = False\r
-        self._close = None\r
-        self._send = self._writer.send\r
-        self._recv = self._reader.recv\r
-        self._poll = self._reader.poll\r
-\r
-    def put(self, obj, block=True, timeout=None):\r
-        assert not self._closed\r
-        if not self._sem.acquire(block, timeout):\r
-            raise Full\r
-\r
-        self._notempty.acquire()\r
-        try:\r
-            if self._thread is None:\r
-                self._start_thread()\r
-            self._buffer.append(obj)\r
-            self._notempty.notify()\r
-        finally:\r
-            self._notempty.release()\r
-\r
-    def get(self, block=True, timeout=None):\r
-        if block and timeout is None:\r
-            self._rlock.acquire()\r
-            try:\r
-                res = self._recv()\r
-                self._sem.release()\r
-                return res\r
-            finally:\r
-                self._rlock.release()\r
-\r
-        else:\r
-            if block:\r
-                deadline = time.time() + timeout\r
-            if not self._rlock.acquire(block, timeout):\r
-                raise Empty\r
-            try:\r
-                if not self._poll(block and (deadline-time.time()) or 0.0):\r
-                    raise Empty\r
-                res = self._recv()\r
-                self._sem.release()\r
-                return res\r
-            finally:\r
-                self._rlock.release()\r
-\r
-    def qsize(self):\r
-        # Raises NotImplementError on Mac OSX because of broken sem_getvalue()\r
-        return self._maxsize - self._sem._semlock._get_value()\r
-\r
-    def empty(self):\r
-        return not self._poll()\r
-\r
-    def full(self):\r
-        return self._sem._semlock._is_zero()\r
-\r
-    def get_nowait(self):\r
-        return self.get(False)\r
-\r
-    def put_nowait(self, obj):\r
-        return self.put(obj, False)\r
-\r
-    def close(self):\r
-        self._closed = True\r
-        self._reader.close()\r
-        if self._close:\r
-            self._close()\r
-\r
-    def join_thread(self):\r
-        debug('Queue.join_thread()')\r
-        assert self._closed\r
-        if self._jointhread:\r
-            self._jointhread()\r
-\r
-    def cancel_join_thread(self):\r
-        debug('Queue.cancel_join_thread()')\r
-        self._joincancelled = True\r
-        try:\r
-            self._jointhread.cancel()\r
-        except AttributeError:\r
-            pass\r
-\r
-    def _start_thread(self):\r
-        debug('Queue._start_thread()')\r
-\r
-        # Start thread which transfers data from buffer to pipe\r
-        self._buffer.clear()\r
-        self._thread = threading.Thread(\r
-            target=Queue._feed,\r
-            args=(self._buffer, self._notempty, self._send,\r
-                  self._wlock, self._writer.close),\r
-            name='QueueFeederThread'\r
-            )\r
-        self._thread.set_daemon(True)\r
-\r
-        debug('doing self._thread.start()')\r
-        self._thread.start()\r
-        debug('... done self._thread.start()')\r
-\r
-        # On process exit we will wait for data to be flushed to pipe.\r
-        #\r
-        # However, if this process created the queue then all\r
-        # processes which use the queue will be descendants of this\r
-        # process.  Therefore waiting for the queue to be flushed\r
-        # is pointless once all the child processes have been joined.\r
-        created_by_this_process = (self._opid == os.getpid())\r
-        if not self._joincancelled and not created_by_this_process:\r
-            self._jointhread = Finalize(\r
-                self._thread, Queue._finalize_join,\r
-                [weakref.ref(self._thread)],\r
-                exitpriority=-5\r
-                )\r
-\r
-        # Send sentinel to the thread queue object when garbage collected\r
-        self._close = Finalize(\r
-            self, Queue._finalize_close,\r
-            [self._buffer, self._notempty],\r
-            exitpriority=10\r
-            )\r
-\r
-    @staticmethod\r
-    def _finalize_join(twr):\r
-        debug('joining queue thread')\r
-        thread = twr()\r
-        if thread is not None:\r
-            thread.join()\r
-            debug('... queue thread joined')\r
-        else:\r
-            debug('... queue thread already dead')\r
-\r
-    @staticmethod\r
-    def _finalize_close(buffer, notempty):\r
-        debug('telling queue thread to quit')\r
-        notempty.acquire()\r
-        try:\r
-            buffer.append(_sentinel)\r
-            notempty.notify()\r
-        finally:\r
-            notempty.release()\r
-\r
-    @staticmethod\r
-    def _feed(buffer, notempty, send, writelock, close):\r
-        debug('starting thread to feed data to pipe')\r
-        from .util import is_exiting\r
-\r
-        nacquire = notempty.acquire\r
-        nrelease = notempty.release\r
-        nwait = notempty.wait\r
-        bpopleft = buffer.popleft\r
-        sentinel = _sentinel\r
-        if sys.platform != 'win32':\r
-            wacquire = writelock.acquire\r
-            wrelease = writelock.release\r
-        else:\r
-            wacquire = None\r
-\r
-        try:\r
-            while 1:\r
-                nacquire()\r
-                try:\r
-                    if not buffer:\r
-                        nwait()\r
-                finally:\r
-                    nrelease()\r
-                try:\r
-                    while 1:\r
-                        obj = bpopleft()\r
-                        if obj is sentinel:\r
-                            debug('feeder thread got sentinel -- exiting')\r
-                            close()\r
-                            return\r
-\r
-                        if wacquire is None:\r
-                            send(obj)\r
-                        else:\r
-                            wacquire()\r
-                            try:\r
-                                send(obj)\r
-                            finally:\r
-                                wrelease()\r
-                except IndexError:\r
-                    pass\r
-        except Exception, e:\r
-            # Since this runs in a daemon thread the resources it uses\r
-            # may be become unusable while the process is cleaning up.\r
-            # We ignore errors which happen after the process has\r
-            # started to cleanup.\r
-            try:\r
-                if is_exiting():\r
-                    info('error in queue thread: %s', e)\r
-                else:\r
-                    import traceback\r
-                    traceback.print_exc()\r
-            except Exception:\r
-                pass\r
-\r
-_sentinel = object()\r
-\r
-#\r
-# A queue type which also supports join() and task_done() methods\r
-#\r
-# Note that if you do not call task_done() for each finished task then\r
-# eventually the counter's semaphore may overflow causing Bad Things\r
-# to happen.\r
-#\r
-\r
-class JoinableQueue(Queue):\r
-\r
-    def __init__(self, maxsize=0):\r
-        Queue.__init__(self, maxsize)\r
-        self._unfinished_tasks = Semaphore(0)\r
-        self._cond = Condition()\r
-\r
-    def __getstate__(self):\r
-        return Queue.__getstate__(self) + (self._cond, self._unfinished_tasks)\r
-\r
-    def __setstate__(self, state):\r
-        Queue.__setstate__(self, state[:-2])\r
-        self._cond, self._unfinished_tasks = state[-2:]\r
-\r
-    def put(self, item, block=True, timeout=None):\r
-        Queue.put(self, item, block, timeout)\r
-        self._unfinished_tasks.release()\r
-\r
-    def task_done(self):\r
-        self._cond.acquire()\r
-        try:\r
-            if not self._unfinished_tasks.acquire(False):\r
-                raise ValueError('task_done() called too many times')\r
-            if self._unfinished_tasks._semlock._is_zero():\r
-                self._cond.notify_all()\r
-        finally:\r
-            self._cond.release()\r
-\r
-    def join(self):\r
-        self._cond.acquire()\r
-        try:\r
-            if not self._unfinished_tasks._semlock._is_zero():\r
-                self._cond.wait()\r
-        finally:\r
-            self._cond.release()\r
-\r
-#\r
-# Simplified Queue type -- really just a locked pipe\r
-#\r
-\r
-class SimpleQueue(object):\r
-\r
-    def __init__(self):\r
-        self._reader, self._writer = Pipe(duplex=False)\r
-        self._rlock = Lock()\r
-        if sys.platform == 'win32':\r
-            self._wlock = None\r
-        else:\r
-            self._wlock = Lock()\r
-        self._make_methods()\r
-\r
-    def empty(self):\r
-        return not self._reader.poll()\r
-\r
-    def __getstate__(self):\r
-        assert_spawning(self)\r
-        return (self._reader, self._writer, self._rlock, self._wlock)\r
-\r
-    def __setstate__(self, state):\r
-        (self._reader, self._writer, self._rlock, self._wlock) = state\r
-        self._make_methods()\r
-\r
-    def _make_methods(self):\r
-        recv = self._reader.recv\r
-        racquire, rrelease = self._rlock.acquire, self._rlock.release\r
-        def get():\r
-            racquire()\r
-            try:\r
-                return recv()\r
-            finally:\r
-                rrelease()\r
-        self.get = get\r
-\r
-        if self._wlock is None:\r
-            # writes to a message oriented win32 pipe are atomic\r
-            self.put = self._writer.send\r
-        else:\r
-            send = self._writer.send\r
-            wacquire, wrelease = self._wlock.acquire, self._wlock.release\r
-            def put(obj):\r
-                wacquire()\r
-                try:\r
-                    return send(obj)\r
-                finally:\r
-                    wrelease()\r
-            self.put = put\r
+#
+# Module implementing queues
+#
+# multiprocessing/queues.py
+#
+# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+#
+
+__all__ = ['Queue', 'SimpleQueue']
+
+import sys
+import os
+import threading
+import collections
+import time
+import atexit
+import weakref
+
+from Queue import Empty, Full
+import _multiprocessing
+from multiprocessing import Pipe
+from multiprocessing.synchronize import Lock, BoundedSemaphore, Semaphore, Condition
+from multiprocessing.util import debug, info, Finalize, register_after_fork
+from multiprocessing.forking import assert_spawning
+
+#
+# Queue type using a pipe, buffer and thread
+#
+
+class Queue(object):
+
+    def __init__(self, maxsize=0):
+        if maxsize <= 0:
+            maxsize = _multiprocessing.SemLock.SEM_VALUE_MAX
+        self._maxsize = maxsize
+        self._reader, self._writer = Pipe(duplex=False)
+        self._rlock = Lock()
+        self._opid = os.getpid()
+        if sys.platform == 'win32':
+            self._wlock = None
+        else:
+            self._wlock = Lock()
+        self._sem = BoundedSemaphore(maxsize)
+
+        self._after_fork()
+
+        if sys.platform != 'win32':
+            register_after_fork(self, Queue._after_fork)
+
+    def __getstate__(self):
+        assert_spawning(self)
+        return (self._maxsize, self._reader, self._writer,
+                self._rlock, self._wlock, self._sem, self._opid)
+
+    def __setstate__(self, state):
+        (self._maxsize, self._reader, self._writer,
+         self._rlock, self._wlock, self._sem, self._opid) = state
+        self._after_fork()
+
+    def _after_fork(self):
+        debug('Queue._after_fork()')
+        self._notempty = threading.Condition(threading.Lock())
+        self._buffer = collections.deque()
+        self._thread = None
+        self._jointhread = None
+        self._joincancelled = False
+        self._closed = False
+        self._close = None
+        self._send = self._writer.send
+        self._recv = self._reader.recv
+        self._poll = self._reader.poll
+
+    def put(self, obj, block=True, timeout=None):
+        assert not self._closed
+        if not self._sem.acquire(block, timeout):
+            raise Full
+
+        self._notempty.acquire()
+        try:
+            if self._thread is None:
+                self._start_thread()
+            self._buffer.append(obj)
+            self._notempty.notify()
+        finally:
+            self._notempty.release()
+
+    def get(self, block=True, timeout=None):
+        if block and timeout is None:
+            self._rlock.acquire()
+            try:
+                res = self._recv()
+                self._sem.release()
+                return res
+            finally:
+                self._rlock.release()
+
+        else:
+            if block:
+                deadline = time.time() + timeout
+            if not self._rlock.acquire(block, timeout):
+                raise Empty
+            try:
+                if not self._poll(block and (deadline-time.time()) or 0.0):
+                    raise Empty
+                res = self._recv()
+                self._sem.release()
+                return res
+            finally:
+                self._rlock.release()
+
+    def qsize(self):
+        # Raises NotImplementError on Mac OSX because of broken sem_getvalue()
+        return self._maxsize - self._sem._semlock._get_value()
+
+    def empty(self):
+        return not self._poll()
+
+    def full(self):
+        return self._sem._semlock._is_zero()
+
+    def get_nowait(self):
+        return self.get(False)
+
+    def put_nowait(self, obj):
+        return self.put(obj, False)
+
+    def close(self):
+        self._closed = True
+        self._reader.close()
+        if self._close:
+            self._close()
+
+    def join_thread(self):
+        debug('Queue.join_thread()')
+        assert self._closed
+        if self._jointhread:
+            self._jointhread()
+
+    def cancel_join_thread(self):
+        debug('Queue.cancel_join_thread()')
+        self._joincancelled = True
+        try:
+            self._jointhread.cancel()
+        except AttributeError:
+            pass
+
+    def _start_thread(self):
+        debug('Queue._start_thread()')
+
+        # Start thread which transfers data from buffer to pipe
+        self._buffer.clear()
+        self._thread = threading.Thread(
+            target=Queue._feed,
+            args=(self._buffer, self._notempty, self._send,
+                  self._wlock, self._writer.close),
+            name='QueueFeederThread'
+            )
+        self._thread.set_daemon(True)
+
+        debug('doing self._thread.start()')
+        self._thread.start()
+        debug('... done self._thread.start()')
+
+        # On process exit we will wait for data to be flushed to pipe.
+        #
+        # However, if this process created the queue then all
+        # processes which use the queue will be descendants of this
+        # process.  Therefore waiting for the queue to be flushed
+        # is pointless once all the child processes have been joined.
+        created_by_this_process = (self._opid == os.getpid())
+        if not self._joincancelled and not created_by_this_process:
+            self._jointhread = Finalize(
+                self._thread, Queue._finalize_join,
+                [weakref.ref(self._thread)],
+                exitpriority=-5
+                )
+
+        # Send sentinel to the thread queue object when garbage collected
+        self._close = Finalize(
+            self, Queue._finalize_close,
+            [self._buffer, self._notempty],
+            exitpriority=10
+            )
+
+    @staticmethod
+    def _finalize_join(twr):
+        debug('joining queue thread')
+        thread = twr()
+        if thread is not None:
+            thread.join()
+            debug('... queue thread joined')
+        else:
+            debug('... queue thread already dead')
+
+    @staticmethod
+    def _finalize_close(buffer, notempty):
+        debug('telling queue thread to quit')
+        notempty.acquire()
+        try:
+            buffer.append(_sentinel)
+            notempty.notify()
+        finally:
+            notempty.release()
+
+    @staticmethod
+    def _feed(buffer, notempty, send, writelock, close):
+        debug('starting thread to feed data to pipe')
+        from .util import is_exiting
+
+        nacquire = notempty.acquire
+        nrelease = notempty.release
+        nwait = notempty.wait
+        bpopleft = buffer.popleft
+        sentinel = _sentinel
+        if sys.platform != 'win32':
+            wacquire = writelock.acquire
+            wrelease = writelock.release
+        else:
+            wacquire = None
+
+        try:
+            while 1:
+                nacquire()
+                try:
+                    if not buffer:
+                        nwait()
+                finally:
+                    nrelease()
+                try:
+                    while 1:
+                        obj = bpopleft()
+                        if obj is sentinel:
+                            debug('feeder thread got sentinel -- exiting')
+                            close()
+                            return
+
+                        if wacquire is None:
+                            send(obj)
+                        else:
+                            wacquire()
+                            try:
+                                send(obj)
+                            finally:
+                                wrelease()
+                except IndexError:
+                    pass
+        except Exception, e:
+            # Since this runs in a daemon thread the resources it uses
+            # may be become unusable while the process is cleaning up.
+            # We ignore errors which happen after the process has
+            # started to cleanup.
+            try:
+                if is_exiting():
+                    info('error in queue thread: %s', e)
+                else:
+                    import traceback
+                    traceback.print_exc()
+            except Exception:
+                pass
+
+_sentinel = object()
+
+#
+# A queue type which also supports join() and task_done() methods
+#
+# Note that if you do not call task_done() for each finished task then
+# eventually the counter's semaphore may overflow causing Bad Things
+# to happen.
+#
+
+class JoinableQueue(Queue):
+
+    def __init__(self, maxsize=0):
+        Queue.__init__(self, maxsize)
+        self._unfinished_tasks = Semaphore(0)
+        self._cond = Condition()
+
+    def __getstate__(self):
+        return Queue.__getstate__(self) + (self._cond, self._unfinished_tasks)
+
+    def __setstate__(self, state):
+        Queue.__setstate__(self, state[:-2])
+        self._cond, self._unfinished_tasks = state[-2:]
+
+    def put(self, item, block=True, timeout=None):
+        Queue.put(self, item, block, timeout)
+        self._unfinished_tasks.release()
+
+    def task_done(self):
+        self._cond.acquire()
+        try:
+            if not self._unfinished_tasks.acquire(False):
+                raise ValueError('task_done() called too many times')
+            if self._unfinished_tasks._semlock._is_zero():
+                self._cond.notify_all()
+        finally:
+            self._cond.release()
+
+    def join(self):
+        self._cond.acquire()
+        try:
+            if not self._unfinished_tasks._semlock._is_zero():
+                self._cond.wait()
+        finally:
+            self._cond.release()
+
+#
+# Simplified Queue type -- really just a locked pipe
+#
+
+class SimpleQueue(object):
+
+    def __init__(self):
+        self._reader, self._writer = Pipe(duplex=False)
+        self._rlock = Lock()
+        if sys.platform == 'win32':
+            self._wlock = None
+        else:
+            self._wlock = Lock()
+        self._make_methods()
+
+    def empty(self):
+        return not self._reader.poll()
+
+    def __getstate__(self):
+        assert_spawning(self)
+        return (self._reader, self._writer, self._rlock, self._wlock)
+
+    def __setstate__(self, state):
+        (self._reader, self._writer, self._rlock, self._wlock) = state
+        self._make_methods()
+
+    def _make_methods(self):
+        recv = self._reader.recv
+        racquire, rrelease = self._rlock.acquire, self._rlock.release
+        def get():
+            racquire()
+            try:
+                return recv()
+            finally:
+                rrelease()
+        self.get = get
+
+        if self._wlock is None:
+            # writes to a message oriented win32 pipe are atomic
+            self.put = self._writer.send
+        else:
+            send = self._writer.send
+            wacquire, wrelease = self._wlock.acquire, self._wlock.release
+            def put(obj):
+                wacquire()
+                try:
+                    return send(obj)
+                finally:
+                    wrelease()
+            self.put = put
index 17778ef803fbd80e1e3d463fc9421098d2c0d15d..1f514b20a96b578bf5c7cdca20441b228a1acd9a 100644 (file)
-#\r
-# Module to allow connection and socket objects to be transferred\r
-# between processes\r
-#\r
-# multiprocessing/reduction.py\r
-#\r
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt\r
-#\r
-\r
-__all__ = []\r
-\r
-import os\r
-import sys\r
-import socket\r
-import threading\r
-import copy_reg\r
-\r
-import _multiprocessing\r
-from multiprocessing import current_process\r
-from multiprocessing.forking import Popen, duplicate, close\r
-from multiprocessing.util import register_after_fork, debug, sub_debug\r
-from multiprocessing.connection import Client, Listener\r
-\r
-\r
-#\r
-#\r
-#\r
-\r
-if not(sys.platform == 'win32' or hasattr(_multiprocessing, 'recvfd')):\r
-    raise ImportError('pickling of connections not supported')\r
-\r
-#\r
-# Platform specific definitions\r
-#\r
-\r
-if sys.platform == 'win32':\r
-    import _subprocess\r
-    from ._multiprocessing import win32\r
-\r
-    def send_handle(conn, handle, destination_pid):\r
-        process_handle = win32.OpenProcess(\r
-            win32.PROCESS_ALL_ACCESS, False, destination_pid\r
-            )\r
-        try:\r
-            new_handle = duplicate(handle, process_handle)\r
-            conn.send(new_handle)\r
-        finally:\r
-            close(process_handle)\r
-\r
-    def recv_handle(conn):\r
-        return conn.recv()\r
-\r
-else:\r
-    def send_handle(conn, handle, destination_pid):\r
-        _multiprocessing.sendfd(conn.fileno(), handle)\r
-\r
-    def recv_handle(conn):\r
-        return _multiprocessing.recvfd(conn.fileno())\r
-\r
-#\r
-# Support for a per-process server thread which caches pickled handles\r
-#\r
-\r
-_cache = set()\r
-\r
-def _reset(obj):\r
-    global _lock, _listener, _cache\r
-    for h in _cache:\r
-        close(h)\r
-    _cache.clear()\r
-    _lock = threading.Lock()\r
-    _listener = None\r
-\r
-_reset(None)\r
-register_after_fork(_reset, _reset)\r
-\r
-def _get_listener():\r
-    global _listener\r
-\r
-    if _listener is None:\r
-        _lock.acquire()\r
-        try:\r
-            if _listener is None:\r
-                debug('starting listener and thread for sending handles')\r
-                _listener = Listener(authkey=current_process().get_authkey())\r
-                t = threading.Thread(target=_serve)\r
-                t.set_daemon(True)\r
-                t.start()\r
-        finally:\r
-            _lock.release()\r
-\r
-    return _listener\r
-\r
-def _serve():\r
-    from .util import is_exiting, sub_warning\r
-\r
-    while 1:\r
-        try:\r
-            conn = _listener.accept()\r
-            handle_wanted, destination_pid = conn.recv()\r
-            _cache.remove(handle_wanted)\r
-            send_handle(conn, handle_wanted, destination_pid)\r
-            close(handle_wanted)\r
-            conn.close()\r
-        except:\r
-            if not is_exiting():\r
-                import traceback\r
-                sub_warning(\r
-                    'thread for sharing handles raised exception :\n' +\r
-                    '-'*79 + '\n' + traceback.format_exc() + '-'*79\r
-                    )\r
-\r
-#\r
-# Functions to be used for pickling/unpickling objects with handles\r
-#\r
-\r
-def reduce_handle(handle):\r
-    if Popen.thread_is_spawning():\r
-        return (None, Popen.duplicate_for_child(handle), True)\r
-    dup_handle = duplicate(handle)\r
-    _cache.add(dup_handle)\r
-    sub_debug('reducing handle %d', handle)\r
-    return (_get_listener().address, dup_handle, False)\r
-\r
-def rebuild_handle(pickled_data):\r
-    address, handle, inherited = pickled_data\r
-    if inherited:\r
-        return handle\r
-    sub_debug('rebuilding handle %d', handle)\r
-    conn = Client(address, authkey=current_process().get_authkey())\r
-    conn.send((handle, os.getpid()))\r
-    new_handle = recv_handle(conn)\r
-    conn.close()\r
-    return new_handle\r
-\r
-#\r
-# Register `_multiprocessing.Connection` with `copy_reg`\r
-#\r
-\r
-def reduce_connection(conn):\r
-    rh = reduce_handle(conn.fileno())\r
-    return rebuild_connection, (rh, conn.readable, conn.writable)\r
-\r
-def rebuild_connection(reduced_handle, readable, writable):\r
-    handle = rebuild_handle(reduced_handle)\r
-    return _multiprocessing.Connection(\r
-        handle, readable=readable, writable=writable\r
-        )\r
-\r
-copy_reg.pickle(_multiprocessing.Connection, reduce_connection)\r
-\r
-#\r
-# Register `socket.socket` with `copy_reg`\r
-#\r
-\r
-def fromfd(fd, family, type_, proto=0):\r
-    s = socket.fromfd(fd, family, type_, proto)\r
-    if s.__class__ is not socket.socket:\r
-        s = socket.socket(_sock=s)\r
-    return s\r
-\r
-def reduce_socket(s):\r
-    reduced_handle = reduce_handle(s.fileno())\r
-    return rebuild_socket, (reduced_handle, s.family, s.type, s.proto)\r
-\r
-def rebuild_socket(reduced_handle, family, type_, proto):\r
-    fd = rebuild_handle(reduced_handle)\r
-    _sock = fromfd(fd, family, type_, proto)\r
-    close(fd)\r
-    return _sock\r
-\r
-copy_reg.pickle(socket.socket, reduce_socket)\r
-\r
-#\r
-# Register `_multiprocessing.PipeConnection` with `copy_reg`\r
-#\r
-\r
-if sys.platform == 'win32':\r
-\r
-    def reduce_pipe_connection(conn):\r
-        rh = reduce_handle(conn.fileno())\r
-        return rebuild_pipe_connection, (rh, conn.readable, conn.writable)\r
-\r
-    def rebuild_pipe_connection(reduced_handle, readable, writable):\r
-        handle = rebuild_handle(reduced_handle)\r
-        return _multiprocessing.PipeConnection(\r
-            handle, readable=readable, writable=writable\r
-            )\r
-\r
-    copy_reg.pickle(_multiprocessing.PipeConnection, reduce_pipe_connection)\r
+#
+# Module to allow connection and socket objects to be transferred
+# between processes
+#
+# multiprocessing/reduction.py
+#
+# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+#
+
+__all__ = []
+
+import os
+import sys
+import socket
+import threading
+import copy_reg
+
+import _multiprocessing
+from multiprocessing import current_process
+from multiprocessing.forking import Popen, duplicate, close
+from multiprocessing.util import register_after_fork, debug, sub_debug
+from multiprocessing.connection import Client, Listener
+
+
+#
+#
+#
+
+if not(sys.platform == 'win32' or hasattr(_multiprocessing, 'recvfd')):
+    raise ImportError('pickling of connections not supported')
+
+#
+# Platform specific definitions
+#
+
+if sys.platform == 'win32':
+    import _subprocess
+    from ._multiprocessing import win32
+
+    def send_handle(conn, handle, destination_pid):
+        process_handle = win32.OpenProcess(
+            win32.PROCESS_ALL_ACCESS, False, destination_pid
+            )
+        try:
+            new_handle = duplicate(handle, process_handle)
+            conn.send(new_handle)
+        finally:
+            close(process_handle)
+
+    def recv_handle(conn):
+        return conn.recv()
+
+else:
+    def send_handle(conn, handle, destination_pid):
+        _multiprocessing.sendfd(conn.fileno(), handle)
+
+    def recv_handle(conn):
+        return _multiprocessing.recvfd(conn.fileno())
+
+#
+# Support for a per-process server thread which caches pickled handles
+#
+
+_cache = set()
+
+def _reset(obj):
+    global _lock, _listener, _cache
+    for h in _cache:
+        close(h)
+    _cache.clear()
+    _lock = threading.Lock()
+    _listener = None
+
+_reset(None)
+register_after_fork(_reset, _reset)
+
+def _get_listener():
+    global _listener
+
+    if _listener is None:
+        _lock.acquire()
+        try:
+            if _listener is None:
+                debug('starting listener and thread for sending handles')
+                _listener = Listener(authkey=current_process().get_authkey())
+                t = threading.Thread(target=_serve)
+                t.set_daemon(True)
+                t.start()
+        finally:
+            _lock.release()
+
+    return _listener
+
+def _serve():
+    from .util import is_exiting, sub_warning
+
+    while 1:
+        try:
+            conn = _listener.accept()
+            handle_wanted, destination_pid = conn.recv()
+            _cache.remove(handle_wanted)
+            send_handle(conn, handle_wanted, destination_pid)
+            close(handle_wanted)
+            conn.close()
+        except:
+            if not is_exiting():
+                import traceback
+                sub_warning(
+                    'thread for sharing handles raised exception :\n' +
+                    '-'*79 + '\n' + traceback.format_exc() + '-'*79
+                    )
+
+#
+# Functions to be used for pickling/unpickling objects with handles
+#
+
+def reduce_handle(handle):
+    if Popen.thread_is_spawning():
+        return (None, Popen.duplicate_for_child(handle), True)
+    dup_handle = duplicate(handle)
+    _cache.add(dup_handle)
+    sub_debug('reducing handle %d', handle)
+    return (_get_listener().address, dup_handle, False)
+
+def rebuild_handle(pickled_data):
+    address, handle, inherited = pickled_data
+    if inherited:
+        return handle
+    sub_debug('rebuilding handle %d', handle)
+    conn = Client(address, authkey=current_process().get_authkey())
+    conn.send((handle, os.getpid()))
+    new_handle = recv_handle(conn)
+    conn.close()
+    return new_handle
+
+#
+# Register `_multiprocessing.Connection` with `copy_reg`
+#
+
+def reduce_connection(conn):
+    rh = reduce_handle(conn.fileno())
+    return rebuild_connection, (rh, conn.readable, conn.writable)
+
+def rebuild_connection(reduced_handle, readable, writable):
+    handle = rebuild_handle(reduced_handle)
+    return _multiprocessing.Connection(
+        handle, readable=readable, writable=writable
+        )
+
+copy_reg.pickle(_multiprocessing.Connection, reduce_connection)
+
+#
+# Register `socket.socket` with `copy_reg`
+#
+
+def fromfd(fd, family, type_, proto=0):
+    s = socket.fromfd(fd, family, type_, proto)
+    if s.__class__ is not socket.socket:
+        s = socket.socket(_sock=s)
+    return s
+
+def reduce_socket(s):
+    reduced_handle = reduce_handle(s.fileno())
+    return rebuild_socket, (reduced_handle, s.family, s.type, s.proto)
+
+def rebuild_socket(reduced_handle, family, type_, proto):
+    fd = rebuild_handle(reduced_handle)
+    _sock = fromfd(fd, family, type_, proto)
+    close(fd)
+    return _sock
+
+copy_reg.pickle(socket.socket, reduce_socket)
+
+#
+# Register `_multiprocessing.PipeConnection` with `copy_reg`
+#
+
+if sys.platform == 'win32':
+
+    def reduce_pipe_connection(conn):
+        rh = reduce_handle(conn.fileno())
+        return rebuild_pipe_connection, (rh, conn.readable, conn.writable)
+
+    def rebuild_pipe_connection(reduced_handle, readable, writable):
+        handle = rebuild_handle(reduced_handle)
+        return _multiprocessing.PipeConnection(
+            handle, readable=readable, writable=writable
+            )
+
+    copy_reg.pickle(_multiprocessing.PipeConnection, reduce_pipe_connection)
index 808d312306b6963df1e7231a3ce709a66d550926..c35c9e0e0d3eed8f2b4a85402c221913c2962d3b 100644 (file)
-#\r
-# Module which supports allocation of ctypes objects from shared memory\r
-#\r
-# multiprocessing/sharedctypes.py\r
-#\r
-# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt\r
-#\r
-\r
-import sys\r
-import ctypes\r
-import weakref\r
-import copy_reg\r
-\r
-from multiprocessing import heap, RLock\r
-from multiprocessing.forking import assert_spawning\r
-\r
-__all__ = ['RawValue', 'RawArray', 'Value', 'Array', 'copy', 'synchronized']\r
-\r
-#\r
-#\r
-#\r
-\r
-typecode_to_type = {\r
-    'c': ctypes.c_char,  'u': ctypes.c_wchar,\r
-    'b': ctypes.c_byte,  'B': ctypes.c_ubyte,\r
-    'h': ctypes.c_short, 'H': ctypes.c_ushort,\r
-    'i': ctypes.c_int,   'I': ctypes.c_uint,\r
-    'l': ctypes.c_long,  'L': ctypes.c_ulong,\r
-    'f': ctypes.c_float, 'd': ctypes.c_double\r
-    }\r
-\r
-#\r
-#\r
-#\r
-\r
-def _new_value(type_):\r
-    size = ctypes.sizeof(type_)\r
-    wrapper = heap.BufferWrapper(size)\r
-    return rebuild_ctype(type_, wrapper, None)\r
-\r
-def RawValue(typecode_or_type, *args):\r
-    '''\r
-    Returns a ctypes object allocated from shared memory\r
-    '''\r
-    type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)\r
-    obj = _new_value(type_)\r
-    ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj))\r
-    obj.__init__(*args)\r
-    return obj\r
-\r
-def RawArray(typecode_or_type, size_or_initializer):\r
-    '''\r
-    Returns a ctypes array allocated from shared memory\r
-    '''\r
-    type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)\r
-    if isinstance(size_or_initializer, int):\r
-        type_ = type_ * size_or_initializer\r
-        return _new_value(type_)\r
-    else:\r
-        type_ = type_ * len(size_or_initializer)\r
-        result = _new_value(type_)\r
-        result.__init__(*size_or_initializer)\r
-        return result\r
-\r
-def Value(typecode_or_type, *args, **kwds):\r
-    '''\r
-    Return a synchronization wrapper for a Value\r
-    '''\r
-    lock = kwds.pop('lock', None)\r
-    if kwds:\r
-        raise ValueError('unrecognized keyword argument(s): %s' % kwds.keys())\r
-    obj = RawValue(typecode_or_type, *args)\r
-    if lock is None:\r
-        lock = RLock()\r
-    assert hasattr(lock, 'acquire')\r
-    return synchronized(obj, lock)\r
-\r
-def Array(typecode_or_type, size_or_initializer, **kwds):\r
-    '''\r
-    Return a synchronization wrapper for a RawArray\r
-    '''\r
-    lock = kwds.pop('lock', None)\r
-    if kwds:\r
-        raise ValueError('unrecognized keyword argument(s): %s' % kwds.keys())\r
-    obj = RawArray(typecode_or_type, size_or_initializer)\r
-    if lock is None:\r
-        lock = RLock()\r
-    assert hasattr(lock, 'acquire')\r
-    return synchronized(obj, lock)\r
-\r
-def copy(obj):\r
-    new_obj = _new_value(type(obj))\r
-    ctypes.pointer(new_obj)[0] = obj\r
-    return new_obj\r
-\r
-def synchronized(obj, lock=None):\r
-    assert not isinstance(obj, SynchronizedBase), 'object already synchronized'\r
-\r
-    if isinstance(obj, ctypes._SimpleCData):\r
-        return Synchronized(obj, lock)\r
-    elif isinstance(obj, ctypes.Array):\r
-        if obj._type_ is ctypes.c_char:\r
-            return SynchronizedString(obj, lock)\r
-        return SynchronizedArray(obj, lock)\r
-    else:\r
-        cls = type(obj)\r
-        try:\r
-            scls = class_cache[cls]\r
-        except KeyError:\r
-            names = [field[0] for field in cls._fields_]\r
-            d = dict((name, make_property(name)) for name in names)\r
-            classname = 'Synchronized' + cls.__name__\r
-            scls = class_cache[cls] = type(classname, (SynchronizedBase,), d)\r
-        return scls(obj, lock)\r
-\r
-#\r
-# Functions for pickling/unpickling\r
-#\r
-\r
-def reduce_ctype(obj):\r
-    assert_spawning(obj)\r
-    if isinstance(obj, ctypes.Array):\r
-        return rebuild_ctype, (obj._type_, obj._wrapper, obj._length_)\r
-    else:\r
-        return rebuild_ctype, (type(obj), obj._wrapper, None)\r
-\r
-def rebuild_ctype(type_, wrapper, length):\r
-    if length is not None:\r
-        type_ = type_ * length\r
-    if sys.platform == 'win32' and type_ not in copy_reg.dispatch_table:\r
-        copy_reg.pickle(type_, reduce_ctype)\r
-    obj = type_.from_address(wrapper.get_address())\r
-    obj._wrapper = wrapper\r
-    return obj\r
-\r
-#\r
-# Function to create properties\r
-#\r
-\r
-def make_property(name):\r
-    try:\r
-        return prop_cache[name]\r
-    except KeyError:\r
-        d = {}\r
-        exec template % ((name,)*7) in d\r
-        prop_cache[name] = d[name]\r
-        return d[name]\r
-\r
-template = '''\r
-def get%s(self):\r
-    self.acquire()\r
-    try:\r
-        return self._obj.%s\r
-    finally:\r
-        self.release()\r
-def set%s(self, value):\r
-    self.acquire()\r
-    try:\r
-        self._obj.%s = value\r
-    finally:\r
-        self.release()\r
-%s = property(get%s, set%s)\r
-'''\r
-\r
-prop_cache = {}\r
-class_cache = weakref.WeakKeyDictionary()\r
-\r
-#\r
-# Synchronized wrappers\r
-#\r
-\r
-class SynchronizedBase(object):\r
-\r
-    def __init__(self, obj, lock=None):\r
-        self._obj = obj\r
-        self._lock = lock or RLock()\r
-        self.acquire = self._lock.acquire\r
-        self.release = self._lock.release\r
-\r
-    def __reduce__(self):\r
-        assert_spawning(self)\r
-        return synchronized, (self._obj, self._lock)\r
-\r
-    def get_obj(self):\r
-        return self._obj\r
-\r
-    def get_lock(self):\r
-        return self._lock\r
-\r
-    def __repr__(self):\r
-        return '<%s wrapper for %s>' % (type(self).__name__, self._obj)\r
-\r
-\r
-class Synchronized(SynchronizedBase):\r
-    value = make_property('value')\r
-\r
-\r
-class SynchronizedArray(SynchronizedBase):\r
-\r
-    def __len__(self):\r
-        return len(self._obj)\r
-\r
-    def __getitem__(self, i):\r
-        self.acquire()\r
-        try:\r
-            return self._obj[i]\r
-        finally:\r
-            self.release()\r
-\r
-    def __setitem__(self, i, value):\r
-        self.acquire()\r
-        try:\r
-            self._obj[i] = value\r
-        finally:\r
-            self.release()\r
-\r
-    def __getslice__(self, start, stop):\r
-        self.acquire()\r
-        try:\r
-            return self._obj[start:stop]\r
-        finally:\r
-            self.release()\r
-\r
-    def __setslice__(self, start, stop, values):\r
-        self.acquire()\r
-        try:\r
-            self._obj[start:stop] = values\r
-        finally:\r
-            self.release()\r
-\r
-\r
-class SynchronizedString(SynchronizedArray):\r
-    value = make_property('value')\r
-    raw = make_property('raw')\r
+#
+# Module which supports allocation of ctypes objects from shared memory
+#
+# multiprocessing/sharedctypes.py
+#
+# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt
+#
+
+import sys
+import ctypes
+import weakref
+import copy_reg
+
+from multiprocessing import heap, RLock
+from multiprocessing.forking import assert_spawning
+
+__all__ = ['RawValue', 'RawArray', 'Value', 'Array', 'copy', 'synchronized']
+
+#
+#
+#
+
+typecode_to_type = {
+    'c': ctypes.c_char,  'u': ctypes.c_wchar,
+    'b': ctypes.c_byte,  'B': ctypes.c_ubyte,
+    'h': ctypes.c_short, 'H': ctypes.c_ushort,
+    'i': ctypes.c_int,   'I': ctypes.c_uint,
+    'l': ctypes.c_long,  'L': ctypes.c_ulong,
+    'f': ctypes.c_float, 'd': ctypes.c_double
+    }
+
+#
+#
+#
+
+def _new_value(type_):
+    size = ctypes.sizeof(type_)
+    wrapper = heap.BufferWrapper(size)
+    return rebuild_ctype(type_, wrapper, None)
+
+def RawValue(typecode_or_type, *args):
+    '''
+    Returns a ctypes object allocated from shared memory
+    '''
+    type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)
+    obj = _new_value(type_)
+    ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj))
+    obj.__init__(*args)
+    return obj
+
+def RawArray(typecode_or_type, size_or_initializer):
+    '''
+    Returns a ctypes array allocated from shared memory
+    '''
+    type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)
+    if isinstance(size_or_initializer, int):
+        type_ = type_ * size_or_initializer
+        return _new_value(type_)
+    else:
+        type_ = type_ * len(size_or_initializer)
+        result = _new_value(type_)
+        result.__init__(*size_or_initializer)
+        return result
+
+def Value(typecode_or_type, *args, **kwds):
+    '''
+    Return a synchronization wrapper for a Value
+    '''
+    lock = kwds.pop('lock', None)
+    if kwds:
+        raise ValueError('unrecognized keyword argument(s): %s' % kwds.keys())
+    obj = RawValue(typecode_or_type, *args)
+    if lock is None:
+        lock = RLock()
+    assert hasattr(lock, 'acquire')
+    return synchronized(obj, lock)
+
+def Array(typecode_or_type, size_or_initializer, **kwds):
+    '''
+    Return a synchronization wrapper for a RawArray
+    '''
+    lock = kwds.pop('lock', None)
+    if kwds:
+        raise ValueError('unrecognized keyword argument(s): %s' % kwds.keys())
+    obj = RawArray(typecode_or_type, size_or_initializer)
+    if lock is None:
+        lock = RLock()
+    assert hasattr(lock, 'acquire')
+    return synchronized(obj, lock)
+
+def copy(obj):
+    new_obj = _new_value(type(obj))
+    ctypes.pointer(new_obj)[0] = obj
+    return new_obj
+
+def synchronized(obj, lock=None):
+    assert not isinstance(obj, SynchronizedBase), 'object already synchronized'
+
+    if isinstance(obj, ctypes._SimpleCData):
+        return Synchronized(obj, lock)
+    elif isinstance(obj, ctypes.Array):
+        if obj._type_ is ctypes.c_char:
+            return SynchronizedString(obj, lock)
+        return SynchronizedArray(obj, lock)
+    else:
+        cls = type(obj)
+        try:
+            scls = class_cache[cls]
+        except KeyError:
+            names = [field[0] for field in cls._fields_]
+            d = dict((name, make_property(name)) for name in names)
+            classname = 'Synchronized' + cls.__name__
+            scls = class_cache[cls] = type(classname, (SynchronizedBase,), d)
+        return scls(obj, lock)
+
+#
+# Functions for pickling/unpickling
+#
+
+def reduce_ctype(obj):
+    assert_spawning(obj)
+    if isinstance(obj, ctypes.Array):
+        return rebuild_ctype, (obj._type_, obj._wrapper, obj._length_)
+    else:
+        return rebuild_ctype, (type(obj), obj._wrapper, None)
+
+def rebuild_ctype(type_, wrapper, length):
+    if length is not None:
+        type_ = type_ * length
+    if sys.platform == 'win32' and type_ not in copy_reg.dispatch_table:
+        copy_reg.pickle(type_, reduce_ctype)
+    obj = type_.from_address(wrapper.get_address())
+    obj._wrapper = wrapper
+    return obj
+
+#
+# Function to create properties
+#
+
+def make_property(name):
+    try:
+        return prop_cache[name]
+    except KeyError:
+        d = {}
+        exec template % ((name,)*7) in d
+        prop_cache[name] = d[name]
+        return d[name]
+
+template = '''
+def get%s(self):
+    self.acquire()
+    try:
+        return self._obj.%s
+    finally:
+        self.release()
+def set%s(self, value):
+    self.acquire()
+    try:
+        self._obj.%s = value
+    finally:
+        self.release()
+%s = property(get%s, set%s)
+'''
+
+prop_cache = {}
+class_cache = weakref.WeakKeyDictionary()
+
+#
+# Synchronized wrappers
+#
+
+class SynchronizedBase(object):
+
+    def __init__(self, obj, lock=None):
+        self._obj = obj
+        self._lock = lock or RLock()
+        self.acquire = self._lock.acquire
+        self.release = self._lock.release
+
+    def __reduce__(self):
+        assert_spawning(self)
+        return synchronized, (self._obj, self._lock)
+
+    def get_obj(self):
+        return self._obj
+
+    def get_lock(self):
+        return self._lock
+
+    def __repr__(self):
+        return '<%s wrapper for %s>' % (type(self).__name__, self._obj)
+
+
+class Synchronized(SynchronizedBase):
+    value = make_property('value')
+
+
+class SynchronizedArray(SynchronizedBase):
+
+    def __len__(self):
+        return len(self._obj)
+
+    def __getitem__(self, i):
+        self.acquire()
+        try:
+            return self._obj[i]
+        finally:
+            self.release()
+
+    def __setitem__(self, i, value):
+        self.acquire()
+        try:
+            self._obj[i] = value
+        finally:
+            self.release()
+
+    def __getslice__(self, start, stop):
+        self.acquire()
+        try:
+            return self._obj[start:stop]
+        finally:
+            self.release()
+
+    def __setslice__(self, start, stop, values):
+        self.acquire()
+        try:
+            self._obj[start:stop] = values
+        finally:
+            self.release()
+
+
+class SynchronizedString(SynchronizedArray):
+    value = make_property('value')
+    raw = make_property('raw')
index 6a7189a7b87ebc76f0f8d38ad8bf5eac55dd5e69..1ebd7b62e64b0b2b5d3980477180b84f87e2798b 100644 (file)
-#\r
-# Module implementing synchronization primitives\r
-#\r
-# multiprocessing/synchronize.py\r
-#\r
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt\r
-#\r
-\r
-__all__ = [\r
-    'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', 'Event'\r
-    ]\r
-\r
-import threading\r
-import os\r
-import sys\r
-\r
-from time import time as _time, sleep as _sleep\r
-\r
-import _multiprocessing\r
-from multiprocessing.process import current_process\r
-from multiprocessing.util import Finalize, register_after_fork, debug\r
-from multiprocessing.forking import assert_spawning, Popen\r
-\r
-#\r
-# Constants\r
-#\r
-\r
-RECURSIVE_MUTEX, SEMAPHORE = range(2)\r
-SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX\r
-\r
-#\r
-# Base class for semaphores and mutexes; wraps `_multiprocessing.SemLock`\r
-#\r
-\r
-class SemLock(object):\r
-\r
-    def __init__(self, kind, value, maxvalue):\r
-        sl = self._semlock = _multiprocessing.SemLock(kind, value, maxvalue)\r
-        debug('created semlock with handle %s' % sl.handle)\r
-        self._make_methods()\r
-\r
-        if sys.platform != 'win32':\r
-            def _after_fork(obj):\r
-                obj._semlock._after_fork()\r
-            register_after_fork(self, _after_fork)\r
-\r
-    def _make_methods(self):\r
-        self.acquire = self._semlock.acquire\r
-        self.release = self._semlock.release\r
-        self.__enter__ = self._semlock.__enter__\r
-        self.__exit__ = self._semlock.__exit__\r
-\r
-    def __getstate__(self):\r
-        assert_spawning(self)\r
-        sl = self._semlock\r
-        return (Popen.duplicate_for_child(sl.handle), sl.kind, sl.maxvalue)\r
-\r
-    def __setstate__(self, state):\r
-        self._semlock = _multiprocessing.SemLock._rebuild(*state)\r
-        debug('recreated blocker with handle %r' % state[0])\r
-        self._make_methods()\r
-\r
-#\r
-# Semaphore\r
-#\r
-\r
-class Semaphore(SemLock):\r
-\r
-    def __init__(self, value=1):\r
-        SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX)\r
-\r
-    def get_value(self):\r
-        return self._semlock._get_value()\r
-\r
-    def __repr__(self):\r
-        try:\r
-            value = self._semlock._get_value()\r
-        except Exception:\r
-            value = 'unknown'\r
-        return '<Semaphore(value=%s)>' % value\r
-\r
-#\r
-# Bounded semaphore\r
-#\r
-\r
-class BoundedSemaphore(Semaphore):\r
-\r
-    def __init__(self, value=1):\r
-        SemLock.__init__(self, SEMAPHORE, value, value)\r
-\r
-    def __repr__(self):\r
-        try:\r
-            value = self._semlock._get_value()\r
-        except Exception:\r
-            value = 'unknown'\r
-        return '<BoundedSemaphore(value=%s, maxvalue=%s)>' % \\r
-               (value, self._semlock.maxvalue)\r
-\r
-#\r
-# Non-recursive lock\r
-#\r
-\r
-class Lock(SemLock):\r
-\r
-    def __init__(self):\r
-        SemLock.__init__(self, SEMAPHORE, 1, 1)\r
-\r
-    def __repr__(self):\r
-        try:\r
-            if self._semlock._is_mine():\r
-                name = current_process().get_name()\r
-                if threading.current_thread().get_name() != 'MainThread':\r
-                    name += '|' + threading.current_thread().get_name()\r
-            elif self._semlock._get_value() == 1:\r
-                name = 'None'\r
-            elif self._semlock._count() > 0:\r
-                name = 'SomeOtherThread'\r
-            else:\r
-                name = 'SomeOtherProcess'\r
-        except Exception:\r
-            name = 'unknown'\r
-        return '<Lock(owner=%s)>' % name\r
-\r
-#\r
-# Recursive lock\r
-#\r
-\r
-class RLock(SemLock):\r
-\r
-    def __init__(self):\r
-        SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1)\r
-\r
-    def __repr__(self):\r
-        try:\r
-            if self._semlock._is_mine():\r
-                name = current_process().get_name()\r
-                if threading.current_thread().get_name() != 'MainThread':\r
-                    name += '|' + threading.current_thread().get_name()\r
-                count = self._semlock._count()\r
-            elif self._semlock._get_value() == 1:\r
-                name, count = 'None', 0\r
-            elif self._semlock._count() > 0:\r
-                name, count = 'SomeOtherThread', 'nonzero'\r
-            else:\r
-                name, count = 'SomeOtherProcess', 'nonzero'\r
-        except Exception:\r
-            name, count = 'unknown', 'unknown'\r
-        return '<RLock(%s, %s)>' % (name, count)\r
-\r
-#\r
-# Condition variable\r
-#\r
-\r
-class Condition(object):\r
-\r
-    def __init__(self, lock=None):\r
-        self._lock = lock or RLock()\r
-        self._sleeping_count = Semaphore(0)\r
-        self._woken_count = Semaphore(0)\r
-        self._wait_semaphore = Semaphore(0)\r
-        self._make_methods()\r
-\r
-    def __getstate__(self):\r
-        assert_spawning(self)\r
-        return (self._lock, self._sleeping_count,\r
-                self._woken_count, self._wait_semaphore)\r
-\r
-    def __setstate__(self, state):\r
-        (self._lock, self._sleeping_count,\r
-         self._woken_count, self._wait_semaphore) = state\r
-        self._make_methods()\r
-\r
-    def _make_methods(self):\r
-        self.acquire = self._lock.acquire\r
-        self.release = self._lock.release\r
-        self.__enter__ = self._lock.__enter__\r
-        self.__exit__ = self._lock.__exit__\r
-\r
-    def __repr__(self):\r
-        try:\r
-            num_waiters = (self._sleeping_count._semlock._get_value() -\r
-                           self._woken_count._semlock._get_value())\r
-        except Exception:\r
-            num_waiters = 'unkown'\r
-        return '<Condition(%s, %s)>' % (self._lock, num_waiters)\r
-\r
-    def wait(self, timeout=None):\r
-        assert self._lock._semlock._is_mine(), \\r
-               'must acquire() condition before using wait()'\r
-\r
-        # indicate that this thread is going to sleep\r
-        self._sleeping_count.release()\r
-\r
-        # release lock\r
-        count = self._lock._semlock._count()\r
-        for i in xrange(count):\r
-            self._lock.release()\r
-\r
-        try:\r
-            # wait for notification or timeout\r
-            self._wait_semaphore.acquire(True, timeout)\r
-        finally:\r
-            # indicate that this thread has woken\r
-            self._woken_count.release()\r
-\r
-            # reacquire lock\r
-            for i in xrange(count):\r
-                self._lock.acquire()\r
-\r
-    def notify(self):\r
-        assert self._lock._semlock._is_mine(), 'lock is not owned'\r
-        assert not self._wait_semaphore.acquire(False)\r
-\r
-        # to take account of timeouts since last notify() we subtract\r
-        # woken_count from sleeping_count and rezero woken_count\r
-        while self._woken_count.acquire(False):\r
-            res = self._sleeping_count.acquire(False)\r
-            assert res\r
-\r
-        if self._sleeping_count.acquire(False): # try grabbing a sleeper\r
-            self._wait_semaphore.release()      # wake up one sleeper\r
-            self._woken_count.acquire()         # wait for the sleeper to wake\r
-\r
-            # rezero _wait_semaphore in case a timeout just happened\r
-            self._wait_semaphore.acquire(False)\r
-\r
-    def notify_all(self):\r
-        assert self._lock._semlock._is_mine(), 'lock is not owned'\r
-        assert not self._wait_semaphore.acquire(False)\r
-\r
-        # to take account of timeouts since last notify*() we subtract\r
-        # woken_count from sleeping_count and rezero woken_count\r
-        while self._woken_count.acquire(False):\r
-            res = self._sleeping_count.acquire(False)\r
-            assert res\r
-\r
-        sleepers = 0\r
-        while self._sleeping_count.acquire(False):\r
-            self._wait_semaphore.release()        # wake up one sleeper\r
-            sleepers += 1\r
-\r
-        if sleepers:\r
-            for i in xrange(sleepers):\r
-                self._woken_count.acquire()       # wait for a sleeper to wake\r
-\r
-            # rezero wait_semaphore in case some timeouts just happened\r
-            while self._wait_semaphore.acquire(False):\r
-                pass\r
-\r
-#\r
-# Event\r
-#\r
-\r
-class Event(object):\r
-\r
-    def __init__(self):\r
-        self._cond = Condition(Lock())\r
-        self._flag = Semaphore(0)\r
-\r
-    def is_set(self):\r
-        self._cond.acquire()\r
-        try:\r
-            if self._flag.acquire(False):\r
-                self._flag.release()\r
-                return True\r
-            return False\r
-        finally:\r
-            self._cond.release()\r
-\r
-    def set(self):\r
-        self._cond.acquire()\r
-        try:\r
-            self._flag.acquire(False)\r
-            self._flag.release()\r
-            self._cond.notify_all()\r
-        finally:\r
-            self._cond.release()\r
-\r
-    def clear(self):\r
-        self._cond.acquire()\r
-        try:\r
-            self._flag.acquire(False)\r
-        finally:\r
-            self._cond.release()\r
-\r
-    def wait(self, timeout=None):\r
-        self._cond.acquire()\r
-        try:\r
-            if self._flag.acquire(False):\r
-                self._flag.release()\r
-            else:\r
-                self._cond.wait(timeout)\r
-        finally:\r
-            self._cond.release()\r
+#
+# Module implementing synchronization primitives
+#
+# multiprocessing/synchronize.py
+#
+# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+#
+
+__all__ = [
+    'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', 'Event'
+    ]
+
+import threading
+import os
+import sys
+
+from time import time as _time, sleep as _sleep
+
+import _multiprocessing
+from multiprocessing.process import current_process
+from multiprocessing.util import Finalize, register_after_fork, debug
+from multiprocessing.forking import assert_spawning, Popen
+
+#
+# Constants
+#
+
+RECURSIVE_MUTEX, SEMAPHORE = range(2)
+SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX
+
+#
+# Base class for semaphores and mutexes; wraps `_multiprocessing.SemLock`
+#
+
+class SemLock(object):
+
+    def __init__(self, kind, value, maxvalue):
+        sl = self._semlock = _multiprocessing.SemLock(kind, value, maxvalue)
+        debug('created semlock with handle %s' % sl.handle)
+        self._make_methods()
+
+        if sys.platform != 'win32':
+            def _after_fork(obj):
+                obj._semlock._after_fork()
+            register_after_fork(self, _after_fork)
+
+    def _make_methods(self):
+        self.acquire = self._semlock.acquire
+        self.release = self._semlock.release
+        self.__enter__ = self._semlock.__enter__
+        self.__exit__ = self._semlock.__exit__
+
+    def __getstate__(self):
+        assert_spawning(self)
+        sl = self._semlock
+        return (Popen.duplicate_for_child(sl.handle), sl.kind, sl.maxvalue)
+
+    def __setstate__(self, state):
+        self._semlock = _multiprocessing.SemLock._rebuild(*state)
+        debug('recreated blocker with handle %r' % state[0])
+        self._make_methods()
+
+#
+# Semaphore
+#
+
+class Semaphore(SemLock):
+
+    def __init__(self, value=1):
+        SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX)
+
+    def get_value(self):
+        return self._semlock._get_value()
+
+    def __repr__(self):
+        try:
+            value = self._semlock._get_value()
+        except Exception:
+            value = 'unknown'
+        return '<Semaphore(value=%s)>' % value
+
+#
+# Bounded semaphore
+#
+
+class BoundedSemaphore(Semaphore):
+
+    def __init__(self, value=1):
+        SemLock.__init__(self, SEMAPHORE, value, value)
+
+    def __repr__(self):
+        try:
+            value = self._semlock._get_value()
+        except Exception:
+            value = 'unknown'
+        return '<BoundedSemaphore(value=%s, maxvalue=%s)>' % \
+               (value, self._semlock.maxvalue)
+
+#
+# Non-recursive lock
+#
+
+class Lock(SemLock):
+
+    def __init__(self):
+        SemLock.__init__(self, SEMAPHORE, 1, 1)
+
+    def __repr__(self):
+        try:
+            if self._semlock._is_mine():
+                name = current_process().get_name()
+                if threading.current_thread().get_name() != 'MainThread':
+                    name += '|' + threading.current_thread().get_name()
+            elif self._semlock._get_value() == 1:
+                name = 'None'
+            elif self._semlock._count() > 0:
+                name = 'SomeOtherThread'
+            else:
+                name = 'SomeOtherProcess'
+        except Exception:
+            name = 'unknown'
+        return '<Lock(owner=%s)>' % name
+
+#
+# Recursive lock
+#
+
+class RLock(SemLock):
+
+    def __init__(self):
+        SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1)
+
+    def __repr__(self):
+        try:
+            if self._semlock._is_mine():
+                name = current_process().get_name()
+                if threading.current_thread().get_name() != 'MainThread':
+                    name += '|' + threading.current_thread().get_name()
+                count = self._semlock._count()
+            elif self._semlock._get_value() == 1:
+                name, count = 'None', 0
+            elif self._semlock._count() > 0:
+                name, count = 'SomeOtherThread', 'nonzero'
+            else:
+                name, count = 'SomeOtherProcess', 'nonzero'
+        except Exception:
+            name, count = 'unknown', 'unknown'
+        return '<RLock(%s, %s)>' % (name, count)
+
+#
+# Condition variable
+#
+
+class Condition(object):
+
+    def __init__(self, lock=None):
+        self._lock = lock or RLock()
+        self._sleeping_count = Semaphore(0)
+        self._woken_count = Semaphore(0)
+        self._wait_semaphore = Semaphore(0)
+        self._make_methods()
+
+    def __getstate__(self):
+        assert_spawning(self)
+        return (self._lock, self._sleeping_count,
+                self._woken_count, self._wait_semaphore)
+
+    def __setstate__(self, state):
+        (self._lock, self._sleeping_count,
+         self._woken_count, self._wait_semaphore) = state
+        self._make_methods()
+
+    def _make_methods(self):
+        self.acquire = self._lock.acquire
+        self.release = self._lock.release
+        self.__enter__ = self._lock.__enter__
+        self.__exit__ = self._lock.__exit__
+
+    def __repr__(self):
+        try:
+            num_waiters = (self._sleeping_count._semlock._get_value() -
+                           self._woken_count._semlock._get_value())
+        except Exception:
+            num_waiters = 'unkown'
+        return '<Condition(%s, %s)>' % (self._lock, num_waiters)
+
+    def wait(self, timeout=None):
+        assert self._lock._semlock._is_mine(), \
+               'must acquire() condition before using wait()'
+
+        # indicate that this thread is going to sleep
+        self._sleeping_count.release()
+
+        # release lock
+        count = self._lock._semlock._count()
+        for i in xrange(count):
+            self._lock.release()
+
+        try:
+            # wait for notification or timeout
+            self._wait_semaphore.acquire(True, timeout)
+        finally:
+            # indicate that this thread has woken
+            self._woken_count.release()
+
+            # reacquire lock
+            for i in xrange(count):
+                self._lock.acquire()
+
+    def notify(self):
+        assert self._lock._semlock._is_mine(), 'lock is not owned'
+        assert not self._wait_semaphore.acquire(False)
+
+        # to take account of timeouts since last notify() we subtract
+        # woken_count from sleeping_count and rezero woken_count
+        while self._woken_count.acquire(False):
+            res = self._sleeping_count.acquire(False)
+            assert res
+
+        if self._sleeping_count.acquire(False): # try grabbing a sleeper
+            self._wait_semaphore.release()      # wake up one sleeper
+            self._woken_count.acquire()         # wait for the sleeper to wake
+
+            # rezero _wait_semaphore in case a timeout just happened
+            self._wait_semaphore.acquire(False)
+
+    def notify_all(self):
+        assert self._lock._semlock._is_mine(), 'lock is not owned'
+        assert not self._wait_semaphore.acquire(False)
+
+        # to take account of timeouts since last notify*() we subtract
+        # woken_count from sleeping_count and rezero woken_count
+        while self._woken_count.acquire(False):
+            res = self._sleeping_count.acquire(False)
+            assert res
+
+        sleepers = 0
+        while self._sleeping_count.acquire(False):
+            self._wait_semaphore.release()        # wake up one sleeper
+            sleepers += 1
+
+        if sleepers:
+            for i in xrange(sleepers):
+                self._woken_count.acquire()       # wait for a sleeper to wake
+
+            # rezero wait_semaphore in case some timeouts just happened
+            while self._wait_semaphore.acquire(False):
+                pass
+
+#
+# Event
+#
+
+class Event(object):
+
+    def __init__(self):
+        self._cond = Condition(Lock())
+        self._flag = Semaphore(0)
+
+    def is_set(self):
+        self._cond.acquire()
+        try:
+            if self._flag.acquire(False):
+                self._flag.release()
+                return True
+            return False
+        finally:
+            self._cond.release()
+
+    def set(self):
+        self._cond.acquire()
+        try:
+            self._flag.acquire(False)
+            self._flag.release()
+            self._cond.notify_all()
+        finally:
+            self._cond.release()
+
+    def clear(self):
+        self._cond.acquire()
+        try:
+            self._flag.acquire(False)
+        finally:
+            self._cond.release()
+
+    def wait(self, timeout=None):
+        self._cond.acquire()
+        try:
+            if self._flag.acquire(False):
+                self._flag.release()
+            else:
+                self._cond.wait(timeout)
+        finally:
+            self._cond.release()
index 25ff8bd2965cc3c2f8c493162fde09b5d4ba72e8..c20a5625e08bbc54b56dbc29a19d072f123b7682 100644 (file)
-#\r
-# Module providing various facilities to other parts of the package\r
-#\r
-# multiprocessing/util.py\r
-#\r
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt\r
-#\r
-\r
-import itertools\r
-import weakref\r
-import copy_reg\r
-import atexit\r
-import threading        # we want threading to install it's\r
-                        # cleanup function before multiprocessing does\r
-\r
-from multiprocessing.process import current_process, active_children\r
-\r
-__all__ = [\r
-    'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger',\r
-    'log_to_stderr', 'get_temp_dir', 'register_after_fork',\r
-    'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal'\r
-    ]\r
-\r
-#\r
-# Logging\r
-#\r
-\r
-NOTSET = 0\r
-SUBDEBUG = 5\r
-DEBUG = 10\r
-INFO = 20\r
-SUBWARNING = 25\r
-\r
-LOGGER_NAME = 'multiprocessing'\r
-DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s'\r
-\r
-_logger = None\r
-_log_to_stderr = False\r
-\r
-def sub_debug(msg, *args):\r
-    if _logger:\r
-        _logger.log(SUBDEBUG, msg, *args)\r
-\r
-def debug(msg, *args):\r
-    if _logger:\r
-        _logger.log(DEBUG, msg, *args)\r
-\r
-def info(msg, *args):\r
-    if _logger:\r
-        _logger.log(INFO, msg, *args)\r
-\r
-def sub_warning(msg, *args):\r
-    if _logger:\r
-        _logger.log(SUBWARNING, msg, *args)\r
-\r
-def get_logger():\r
-    '''\r
-    Returns logger used by multiprocessing\r
-    '''\r
-    global _logger\r
-\r
-    if not _logger:\r
-        import logging, atexit\r
-\r
-        # XXX multiprocessing should cleanup before logging\r
-        if hasattr(atexit, 'unregister'):\r
-            atexit.unregister(_exit_function)\r
-            atexit.register(_exit_function)\r
-        else:\r
-            atexit._exithandlers.remove((_exit_function, (), {}))\r
-            atexit._exithandlers.append((_exit_function, (), {}))\r
-\r
-        _check_logger_class()\r
-        _logger = logging.getLogger(LOGGER_NAME)\r
-\r
-    return _logger\r
-\r
-def _check_logger_class():\r
-    '''\r
-    Make sure process name is recorded when loggers are used\r
-    '''\r
-    # XXX This function is unnecessary once logging is patched\r
-    import logging\r
-    if hasattr(logging, 'multiprocessing'):\r
-        return\r
-\r
-    logging._acquireLock()\r
-    try:\r
-        OldLoggerClass = logging.getLoggerClass()\r
-        if not getattr(OldLoggerClass, '_process_aware', False):\r
-            class ProcessAwareLogger(OldLoggerClass):\r
-                _process_aware = True\r
-                def makeRecord(self, *args, **kwds):\r
-                    record = OldLoggerClass.makeRecord(self, *args, **kwds)\r
-                    record.processName = current_process()._name\r
-                    return record\r
-            logging.setLoggerClass(ProcessAwareLogger)\r
-    finally:\r
-        logging._releaseLock()\r
-\r
-def log_to_stderr(level=None):\r
-    '''\r
-    Turn on logging and add a handler which prints to stderr\r
-    '''\r
-    global _log_to_stderr\r
-    import logging\r
-    logger = get_logger()\r
-    formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)\r
-    handler = logging.StreamHandler()\r
-    handler.setFormatter(formatter)\r
-    logger.addHandler(handler)\r
-    if level is not None:\r
-        logger.setLevel(level)\r
-    _log_to_stderr = True\r
-\r
-#\r
-# Function returning a temp directory which will be removed on exit\r
-#\r
-\r
-def get_temp_dir():\r
-    # get name of a temp directory which will be automatically cleaned up\r
-    if current_process()._tempdir is None:\r
-        import shutil, tempfile\r
-        tempdir = tempfile.mkdtemp(prefix='pymp-')\r
-        info('created temp directory %s', tempdir)\r
-        Finalize(None, shutil.rmtree, args=[tempdir], exitpriority=-100)\r
-        current_process()._tempdir = tempdir\r
-    return current_process()._tempdir\r
-\r
-#\r
-# Support for reinitialization of objects when bootstrapping a child process\r
-#\r
-\r
-_afterfork_registry = weakref.WeakValueDictionary()\r
-_afterfork_counter = itertools.count()\r
-\r
-def _run_after_forkers():\r
-    items = list(_afterfork_registry.items())\r
-    items.sort()\r
-    for (index, ident, func), obj in items:\r
-        try:\r
-            func(obj)\r
-        except Exception, e:\r
-            info('after forker raised exception %s', e)\r
-\r
-def register_after_fork(obj, func):\r
-    _afterfork_registry[(_afterfork_counter.next(), id(obj), func)] = obj\r
-\r
-#\r
-# Finalization using weakrefs\r
-#\r
-\r
-_finalizer_registry = {}\r
-_finalizer_counter = itertools.count()\r
-\r
-\r
-class Finalize(object):\r
-    '''\r
-    Class which supports object finalization using weakrefs\r
-    '''\r
-    def __init__(self, obj, callback, args=(), kwargs=None, exitpriority=None):\r
-        assert exitpriority is None or type(exitpriority) is int\r
-\r
-        if obj is not None:\r
-            self._weakref = weakref.ref(obj, self)\r
-        else:\r
-            assert exitpriority is not None\r
-\r
-        self._callback = callback\r
-        self._args = args\r
-        self._kwargs = kwargs or {}\r
-        self._key = (exitpriority, _finalizer_counter.next())\r
-\r
-        _finalizer_registry[self._key] = self\r
-\r
-    def __call__(self, wr=None):\r
-        '''\r
-        Run the callback unless it has already been called or cancelled\r
-        '''\r
-        try:\r
-            del _finalizer_registry[self._key]\r
-        except KeyError:\r
-            sub_debug('finalizer no longer registered')\r
-        else:\r
-            sub_debug('finalizer calling %s with args %s and kwargs %s',\r
-                     self._callback, self._args, self._kwargs)\r
-            res = self._callback(*self._args, **self._kwargs)\r
-            self._weakref = self._callback = self._args = \\r
-                            self._kwargs = self._key = None\r
-            return res\r
-\r
-    def cancel(self):\r
-        '''\r
-        Cancel finalization of the object\r
-        '''\r
-        try:\r
-            del _finalizer_registry[self._key]\r
-        except KeyError:\r
-            pass\r
-        else:\r
-            self._weakref = self._callback = self._args = \\r
-                            self._kwargs = self._key = None\r
-\r
-    def still_active(self):\r
-        '''\r
-        Return whether this finalizer is still waiting to invoke callback\r
-        '''\r
-        return self._key in _finalizer_registry\r
-\r
-    def __repr__(self):\r
-        try:\r
-            obj = self._weakref()\r
-        except (AttributeError, TypeError):\r
-            obj = None\r
-\r
-        if obj is None:\r
-            return '<Finalize object, dead>'\r
-\r
-        x = '<Finalize object, callback=%s' % \\r
-            getattr(self._callback, '__name__', self._callback)\r
-        if self._args:\r
-            x += ', args=' + str(self._args)\r
-        if self._kwargs:\r
-            x += ', kwargs=' + str(self._kwargs)\r
-        if self._key[0] is not None:\r
-            x += ', exitprority=' + str(self._key[0])\r
-        return x + '>'\r
-\r
-\r
-def _run_finalizers(minpriority=None):\r
-    '''\r
-    Run all finalizers whose exit priority is not None and at least minpriority\r
-\r
-    Finalizers with highest priority are called first; finalizers with\r
-    the same priority will be called in reverse order of creation.\r
-    '''\r
-    if minpriority is None:\r
-        f = lambda p : p[0][0] is not None\r
-    else:\r
-        f = lambda p : p[0][0] is not None and p[0][0] >= minpriority\r
-\r
-    items = [x for x in _finalizer_registry.items() if f(x)]\r
-    items.sort(reverse=True)\r
-\r
-    for key, finalizer in items:\r
-        sub_debug('calling %s', finalizer)\r
-        try:\r
-            finalizer()\r
-        except Exception:\r
-            import traceback\r
-            traceback.print_exc()\r
-\r
-    if minpriority is None:\r
-        _finalizer_registry.clear()\r
-\r
-#\r
-# Clean up on exit\r
-#\r
-\r
-def is_exiting():\r
-    '''\r
-    Returns true if the process is shutting down\r
-    '''\r
-    return _exiting or _exiting is None\r
-\r
-_exiting = False\r
-\r
-def _exit_function():\r
-    global _exiting\r
-\r
-    info('process shutting down')\r
-    debug('running all "atexit" finalizers with priority >= 0')\r
-    _run_finalizers(0)\r
-\r
-    for p in active_children():\r
-        if p._daemonic:\r
-            info('calling terminate() for daemon %s', p.get_name())\r
-            p._popen.terminate()\r
-\r
-    for p in active_children():\r
-        info('calling join() for process %s', p.get_name())\r
-        p.join()\r
-\r
-    debug('running the remaining "atexit" finalizers')\r
-    _run_finalizers()\r
-\r
-atexit.register(_exit_function)\r
-\r
-#\r
-# Some fork aware types\r
-#\r
-\r
-class ForkAwareThreadLock(object):\r
-    def __init__(self):\r
-        self._lock = threading.Lock()\r
-        self.acquire = self._lock.acquire\r
-        self.release = self._lock.release\r
-        register_after_fork(self, ForkAwareThreadLock.__init__)\r
-\r
-class ForkAwareLocal(threading.local):\r
-    def __init__(self):\r
-        register_after_fork(self, lambda obj : obj.__dict__.clear())\r
-    def __reduce__(self):\r
-        return type(self), ()\r
-\r
-#\r
-# Try making some callable types picklable\r
-#\r
-\r
-def _reduce_method(m):\r
-    if m.im_self is None:\r
-        return getattr, (m.im_class, m.im_func.func_name)\r
-    else:\r
-        return getattr, (m.im_self, m.im_func.func_name)\r
-copy_reg.pickle(type(Finalize.__init__), _reduce_method)\r
-\r
-def _reduce_method_descriptor(m):\r
-    return getattr, (m.__objclass__, m.__name__)\r
-copy_reg.pickle(type(list.append), _reduce_method_descriptor)\r
-copy_reg.pickle(type(int.__add__), _reduce_method_descriptor)\r
-\r
-def _reduce_builtin_function_or_method(m):\r
-    return getattr, (m.__self__, m.__name__)\r
-copy_reg.pickle(type(list().append), _reduce_builtin_function_or_method)\r
-copy_reg.pickle(type(int().__add__), _reduce_builtin_function_or_method)\r
-\r
-try:\r
-    from functools import partial\r
-except ImportError:\r
-    pass\r
-else:\r
-    def _reduce_partial(p):\r
-        return _rebuild_partial, (p.func, p.args, p.keywords or {})\r
-    def _rebuild_partial(func, args, keywords):\r
-        return partial(func, *args, **keywords)\r
-    copy_reg.pickle(partial, _reduce_partial)\r
+#
+# Module providing various facilities to other parts of the package
+#
+# multiprocessing/util.py
+#
+# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+#
+
+import itertools
+import weakref
+import copy_reg
+import atexit
+import threading        # we want threading to install it's
+                        # cleanup function before multiprocessing does
+
+from multiprocessing.process import current_process, active_children
+
+__all__ = [
+    'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger',
+    'log_to_stderr', 'get_temp_dir', 'register_after_fork',
+    'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal'
+    ]
+
+#
+# Logging
+#
+
+NOTSET = 0
+SUBDEBUG = 5
+DEBUG = 10
+INFO = 20
+SUBWARNING = 25
+
+LOGGER_NAME = 'multiprocessing'
+DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s'
+
+_logger = None
+_log_to_stderr = False
+
+def sub_debug(msg, *args):
+    if _logger:
+        _logger.log(SUBDEBUG, msg, *args)
+
+def debug(msg, *args):
+    if _logger:
+        _logger.log(DEBUG, msg, *args)
+
+def info(msg, *args):
+    if _logger:
+        _logger.log(INFO, msg, *args)
+
+def sub_warning(msg, *args):
+    if _logger:
+        _logger.log(SUBWARNING, msg, *args)
+
+def get_logger():
+    '''
+    Returns logger used by multiprocessing
+    '''
+    global _logger
+
+    if not _logger:
+        import logging, atexit
+
+        # XXX multiprocessing should cleanup before logging
+        if hasattr(atexit, 'unregister'):
+            atexit.unregister(_exit_function)
+            atexit.register(_exit_function)
+        else:
+            atexit._exithandlers.remove((_exit_function, (), {}))
+            atexit._exithandlers.append((_exit_function, (), {}))
+
+        _check_logger_class()
+        _logger = logging.getLogger(LOGGER_NAME)
+
+    return _logger
+
+def _check_logger_class():
+    '''
+    Make sure process name is recorded when loggers are used
+    '''
+    # XXX This function is unnecessary once logging is patched
+    import logging
+    if hasattr(logging, 'multiprocessing'):
+        return
+
+    logging._acquireLock()
+    try:
+        OldLoggerClass = logging.getLoggerClass()
+        if not getattr(OldLoggerClass, '_process_aware', False):
+            class ProcessAwareLogger(OldLoggerClass):
+                _process_aware = True
+                def makeRecord(self, *args, **kwds):
+                    record = OldLoggerClass.makeRecord(self, *args, **kwds)
+                    record.processName = current_process()._name
+                    return record
+            logging.setLoggerClass(ProcessAwareLogger)
+    finally:
+        logging._releaseLock()
+
+def log_to_stderr(level=None):
+    '''
+    Turn on logging and add a handler which prints to stderr
+    '''
+    global _log_to_stderr
+    import logging
+    logger = get_logger()
+    formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    if level is not None:
+        logger.setLevel(level)
+    _log_to_stderr = True
+
+#
+# Function returning a temp directory which will be removed on exit
+#
+
+def get_temp_dir():
+    # get name of a temp directory which will be automatically cleaned up
+    if current_process()._tempdir is None:
+        import shutil, tempfile
+        tempdir = tempfile.mkdtemp(prefix='pymp-')
+        info('created temp directory %s', tempdir)
+        Finalize(None, shutil.rmtree, args=[tempdir], exitpriority=-100)
+        current_process()._tempdir = tempdir
+    return current_process()._tempdir
+
+#
+# Support for reinitialization of objects when bootstrapping a child process
+#
+
+_afterfork_registry = weakref.WeakValueDictionary()
+_afterfork_counter = itertools.count()
+
+def _run_after_forkers():
+    items = list(_afterfork_registry.items())
+    items.sort()
+    for (index, ident, func), obj in items:
+        try:
+            func(obj)
+        except Exception, e:
+            info('after forker raised exception %s', e)
+
+def register_after_fork(obj, func):
+    _afterfork_registry[(_afterfork_counter.next(), id(obj), func)] = obj
+
+#
+# Finalization using weakrefs
+#
+
+_finalizer_registry = {}
+_finalizer_counter = itertools.count()
+
+
+class Finalize(object):
+    '''
+    Class which supports object finalization using weakrefs
+    '''
+    def __init__(self, obj, callback, args=(), kwargs=None, exitpriority=None):
+        assert exitpriority is None or type(exitpriority) is int
+
+        if obj is not None:
+            self._weakref = weakref.ref(obj, self)
+        else:
+            assert exitpriority is not None
+
+        self._callback = callback
+        self._args = args
+        self._kwargs = kwargs or {}
+        self._key = (exitpriority, _finalizer_counter.next())
+
+        _finalizer_registry[self._key] = self
+
+    def __call__(self, wr=None):
+        '''
+        Run the callback unless it has already been called or cancelled
+        '''
+        try:
+            del _finalizer_registry[self._key]
+        except KeyError:
+            sub_debug('finalizer no longer registered')
+        else:
+            sub_debug('finalizer calling %s with args %s and kwargs %s',
+                     self._callback, self._args, self._kwargs)
+            res = self._callback(*self._args, **self._kwargs)
+            self._weakref = self._callback = self._args = \
+                            self._kwargs = self._key = None
+            return res
+
+    def cancel(self):
+        '''
+        Cancel finalization of the object
+        '''
+        try:
+            del _finalizer_registry[self._key]
+        except KeyError:
+            pass
+        else:
+            self._weakref = self._callback = self._args = \
+                            self._kwargs = self._key = None
+
+    def still_active(self):
+        '''
+        Return whether this finalizer is still waiting to invoke callback
+        '''
+        return self._key in _finalizer_registry
+
+    def __repr__(self):
+        try:
+            obj = self._weakref()
+        except (AttributeError, TypeError):
+            obj = None
+
+        if obj is None:
+            return '<Finalize object, dead>'
+
+        x = '<Finalize object, callback=%s' % \
+            getattr(self._callback, '__name__', self._callback)
+        if self._args:
+            x += ', args=' + str(self._args)
+        if self._kwargs:
+            x += ', kwargs=' + str(self._kwargs)
+        if self._key[0] is not None:
+            x += ', exitprority=' + str(self._key[0])
+        return x + '>'
+
+
+def _run_finalizers(minpriority=None):
+    '''
+    Run all finalizers whose exit priority is not None and at least minpriority
+
+    Finalizers with highest priority are called first; finalizers with
+    the same priority will be called in reverse order of creation.
+    '''
+    if minpriority is None:
+        f = lambda p : p[0][0] is not None
+    else:
+        f = lambda p : p[0][0] is not None and p[0][0] >= minpriority
+
+    items = [x for x in _finalizer_registry.items() if f(x)]
+    items.sort(reverse=True)
+
+    for key, finalizer in items:
+        sub_debug('calling %s', finalizer)
+        try:
+            finalizer()
+        except Exception:
+            import traceback
+            traceback.print_exc()
+
+    if minpriority is None:
+        _finalizer_registry.clear()
+
+#
+# Clean up on exit
+#
+
+def is_exiting():
+    '''
+    Returns true if the process is shutting down
+    '''
+    return _exiting or _exiting is None
+
+_exiting = False
+
+def _exit_function():
+    global _exiting
+
+    info('process shutting down')
+    debug('running all "atexit" finalizers with priority >= 0')
+    _run_finalizers(0)
+
+    for p in active_children():
+        if p._daemonic:
+            info('calling terminate() for daemon %s', p.get_name())
+            p._popen.terminate()
+
+    for p in active_children():
+        info('calling join() for process %s', p.get_name())
+        p.join()
+
+    debug('running the remaining "atexit" finalizers')
+    _run_finalizers()
+
+atexit.register(_exit_function)
+
+#
+# Some fork aware types
+#
+
+class ForkAwareThreadLock(object):
+    def __init__(self):
+        self._lock = threading.Lock()
+        self.acquire = self._lock.acquire
+        self.release = self._lock.release
+        register_after_fork(self, ForkAwareThreadLock.__init__)
+
+class ForkAwareLocal(threading.local):
+    def __init__(self):
+        register_after_fork(self, lambda obj : obj.__dict__.clear())
+    def __reduce__(self):
+        return type(self), ()
+
+#
+# Try making some callable types picklable
+#
+
+def _reduce_method(m):
+    if m.im_self is None:
+        return getattr, (m.im_class, m.im_func.func_name)
+    else:
+        return getattr, (m.im_self, m.im_func.func_name)
+copy_reg.pickle(type(Finalize.__init__), _reduce_method)
+
+def _reduce_method_descriptor(m):
+    return getattr, (m.__objclass__, m.__name__)
+copy_reg.pickle(type(list.append), _reduce_method_descriptor)
+copy_reg.pickle(type(int.__add__), _reduce_method_descriptor)
+
+def _reduce_builtin_function_or_method(m):
+    return getattr, (m.__self__, m.__name__)
+copy_reg.pickle(type(list().append), _reduce_builtin_function_or_method)
+copy_reg.pickle(type(int().__add__), _reduce_builtin_function_or_method)
+
+try:
+    from functools import partial
+except ImportError:
+    pass
+else:
+    def _reduce_partial(p):
+        return _rebuild_partial, (p.func, p.args, p.keywords or {})
+    def _rebuild_partial(func, args, keywords):
+        return partial(func, *args, **keywords)
+    copy_reg.pickle(partial, _reduce_partial)