]> granicus.if.org Git - python/commitdiff
Fix for issue 6851: urllib.urlopen crashes in a thread on OSX 10.6
authorRonald Oussoren <ronaldoussoren@mac.com>
Sun, 20 Sep 2009 10:31:22 +0000 (10:31 +0000)
committerRonald Oussoren <ronaldoussoren@mac.com>
Sun, 20 Sep 2009 10:31:22 +0000 (10:31 +0000)
Lib/urllib.py
Mac/Modules/_scproxy.c [new file with mode: 0644]
Misc/NEWS
setup.py

index a56f16273ab122d906f1c0ef85e484ef4a8ad215..51e8df9c89a889f1d9042677d370d7316ab6ab49 100644 (file)
@@ -1327,38 +1327,7 @@ def proxy_bypass_environment(host):
 
 
 if sys.platform == 'darwin':
-
-    def _CFSetup(sc):
-        from ctypes import c_int32, c_void_p, c_char_p, c_int
-        sc.CFStringCreateWithCString.argtypes = [ c_void_p, c_char_p, c_int32 ]
-        sc.CFStringCreateWithCString.restype = c_void_p
-        sc.SCDynamicStoreCopyProxies.argtypes = [ c_void_p ]
-        sc.SCDynamicStoreCopyProxies.restype = c_void_p
-        sc.CFDictionaryGetValue.argtypes = [ c_void_p, c_void_p ]
-        sc.CFDictionaryGetValue.restype = c_void_p
-        sc.CFStringGetLength.argtypes = [ c_void_p ]
-        sc.CFStringGetLength.restype = c_int32
-        sc.CFStringGetCString.argtypes = [ c_void_p, c_char_p, c_int32, c_int32 ]
-        sc.CFStringGetCString.restype = c_int32
-        sc.CFNumberGetValue.argtypes = [ c_void_p, c_int, c_void_p ]
-        sc.CFNumberGetValue.restype = c_int32
-        sc.CFRelease.argtypes = [ c_void_p ]
-        sc.CFRelease.restype = None
-
-    def _CStringFromCFString(sc, value):
-        from ctypes import create_string_buffer
-        length = sc.CFStringGetLength(value) + 1
-        buff = create_string_buffer(length)
-        sc.CFStringGetCString(value, buff, length, 0)
-        return buff.value
-
-    def _CFNumberToInt32(sc, cfnum):
-        from ctypes import byref, c_int
-        val = c_int()
-        kCFNumberSInt32Type = 3
-        sc.CFNumberGetValue(cfnum, kCFNumberSInt32Type, byref(val))
-        return val.value
-
+    from _scproxy import _get_proxy_settings, _get_proxies
 
     def proxy_bypass_macosx_sysconf(host):
         """
@@ -1367,8 +1336,6 @@ if sys.platform == 'darwin':
         This function uses the MacOSX framework SystemConfiguration
         to fetch the proxy information.
         """
-        from ctypes import cdll
-        from ctypes.util import find_library
         import re
         import socket
         from fnmatch import fnmatch
@@ -1380,63 +1347,35 @@ if sys.platform == 'darwin':
                 parts = (parts + [0, 0, 0, 0])[:4]
             return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]
 
-        sc = cdll.LoadLibrary(find_library("SystemConfiguration"))
-        _CFSetup(sc)
+        proxy_settings = _get_proxy_settings()
 
-        hostIP = None
+        # Check for simple host names:
+        if '.' not in host:
+            if proxy_settings['exclude_simple']:
+                return True
 
-        if not sc:
-            return False
+        for value in proxy_settings.get('exceptions'):
+            # Items in the list are strings like these: *.local, 169.254/16
+            value = sc.CFArrayGetValueAtIndex(exceptions, index)
+            if not value: continue
 
-        kSCPropNetProxiesExceptionsList = sc.CFStringCreateWithCString(0, "ExceptionsList", 0)
-        kSCPropNetProxiesExcludeSimpleHostnames = sc.CFStringCreateWithCString(0,
-                "ExcludeSimpleHostnames", 0)
+            m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
+            if m is not None:
+                if hostIP is None:
+                    hostIP = socket.gethostbyname(host)
+                    hostIP = ip2num(hostIP)
 
+                base = ip2num(m.group(1))
+                mask = int(m.group(2)[1:])
+                mask = 32 - mask
 
-        proxyDict = sc.SCDynamicStoreCopyProxies(None)
-        if proxyDict is None:
-            return False
-
-        try:
-            # Check for simple host names:
-            if '.' not in host:
-                exclude_simple = sc.CFDictionaryGetValue(proxyDict,
-                        kSCPropNetProxiesExcludeSimpleHostnames)
-                if exclude_simple and _CFNumberToInt32(sc, exclude_simple):
+                if (hostIP >> mask) == (base >> mask):
                     return True
 
+            elif fnmatch(host, value):
+                return True
 
-            # Check the exceptions list:
-            exceptions = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesExceptionsList)
-            if exceptions:
-                # Items in the list are strings like these: *.local, 169.254/16
-                for index in xrange(sc.CFArrayGetCount(exceptions)):
-                    value = sc.CFArrayGetValueAtIndex(exceptions, index)
-                    if not value: continue
-                    value = _CStringFromCFString(sc, value)
-
-                    m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
-                    if m is not None:
-                        if hostIP is None:
-                            hostIP = socket.gethostbyname(host)
-                            hostIP = ip2num(hostIP)
-
-                        base = ip2num(m.group(1))
-                        mask = int(m.group(2)[1:])
-                        mask = 32 - mask
-
-                        if (hostIP >> mask) == (base >> mask):
-                            return True
-
-                    elif fnmatch(host, value):
-                        return True
-
-            return False
-
-        finally:
-            sc.CFRelease(kSCPropNetProxiesExceptionsList)
-            sc.CFRelease(kSCPropNetProxiesExcludeSimpleHostnames)
-
+        return False
 
 
     def getproxies_macosx_sysconf():
@@ -1445,106 +1384,7 @@ if sys.platform == 'darwin':
         This function uses the MacOSX framework SystemConfiguration
         to fetch the proxy information.
         """
-        from ctypes import cdll
-        from ctypes.util import find_library
-
-        sc = cdll.LoadLibrary(find_library("SystemConfiguration"))
-        _CFSetup(sc)
-
-        if not sc:
-            return {}
-
-        kSCPropNetProxiesHTTPEnable = sc.CFStringCreateWithCString(0, "HTTPEnable", 0)
-        kSCPropNetProxiesHTTPProxy = sc.CFStringCreateWithCString(0, "HTTPProxy", 0)
-        kSCPropNetProxiesHTTPPort = sc.CFStringCreateWithCString(0, "HTTPPort", 0)
-
-        kSCPropNetProxiesHTTPSEnable = sc.CFStringCreateWithCString(0, "HTTPSEnable", 0)
-        kSCPropNetProxiesHTTPSProxy = sc.CFStringCreateWithCString(0, "HTTPSProxy", 0)
-        kSCPropNetProxiesHTTPSPort = sc.CFStringCreateWithCString(0, "HTTPSPort", 0)
-
-        kSCPropNetProxiesFTPEnable = sc.CFStringCreateWithCString(0, "FTPEnable", 0)
-        kSCPropNetProxiesFTPPassive = sc.CFStringCreateWithCString(0, "FTPPassive", 0)
-        kSCPropNetProxiesFTPPort = sc.CFStringCreateWithCString(0, "FTPPort", 0)
-        kSCPropNetProxiesFTPProxy = sc.CFStringCreateWithCString(0, "FTPProxy", 0)
-
-        kSCPropNetProxiesGopherEnable = sc.CFStringCreateWithCString(0, "GopherEnable", 0)
-        kSCPropNetProxiesGopherPort = sc.CFStringCreateWithCString(0, "GopherPort", 0)
-        kSCPropNetProxiesGopherProxy = sc.CFStringCreateWithCString(0, "GopherProxy", 0)
-
-        proxies = {}
-        proxyDict = sc.SCDynamicStoreCopyProxies(None)
-
-        try:
-            # HTTP:
-            enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPEnable)
-            if enabled and _CFNumberToInt32(sc, enabled):
-                proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPProxy)
-                port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPPort)
-
-                if proxy:
-                    proxy = _CStringFromCFString(sc, proxy)
-                    if port:
-                        port = _CFNumberToInt32(sc, port)
-                        proxies["http"] = "http://%s:%i" % (proxy, port)
-                    else:
-                        proxies["http"] = "http://%s" % (proxy, )
-
-            # HTTPS:
-            enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSEnable)
-            if enabled and _CFNumberToInt32(sc, enabled):
-                proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSProxy)
-                port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSPort)
-
-                if proxy:
-                    proxy = _CStringFromCFString(sc, proxy)
-                    if port:
-                        port = _CFNumberToInt32(sc, port)
-                        proxies["https"] = "http://%s:%i" % (proxy, port)
-                    else:
-                        proxies["https"] = "http://%s" % (proxy, )
-
-            # FTP:
-            enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPEnable)
-            if enabled and _CFNumberToInt32(sc, enabled):
-                proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPProxy)
-                port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPPort)
-
-                if proxy:
-                    proxy = _CStringFromCFString(sc, proxy)
-                    if port:
-                        port = _CFNumberToInt32(sc, port)
-                        proxies["ftp"] = "http://%s:%i" % (proxy, port)
-                    else:
-                        proxies["ftp"] = "http://%s" % (proxy, )
-
-            # Gopher:
-            enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherEnable)
-            if enabled and _CFNumberToInt32(sc, enabled):
-                proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherProxy)
-                port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherPort)
-
-                if proxy:
-                    proxy = _CStringFromCFString(sc, proxy)
-                    if port:
-                        port = _CFNumberToInt32(sc, port)
-                        proxies["gopher"] = "http://%s:%i" % (proxy, port)
-                    else:
-                        proxies["gopher"] = "http://%s" % (proxy, )
-        finally:
-            sc.CFRelease(proxyDict)
-
-        sc.CFRelease(kSCPropNetProxiesHTTPEnable)
-        sc.CFRelease(kSCPropNetProxiesHTTPProxy)
-        sc.CFRelease(kSCPropNetProxiesHTTPPort)
-        sc.CFRelease(kSCPropNetProxiesFTPEnable)
-        sc.CFRelease(kSCPropNetProxiesFTPPassive)
-        sc.CFRelease(kSCPropNetProxiesFTPPort)
-        sc.CFRelease(kSCPropNetProxiesFTPProxy)
-        sc.CFRelease(kSCPropNetProxiesGopherEnable)
-        sc.CFRelease(kSCPropNetProxiesGopherPort)
-        sc.CFRelease(kSCPropNetProxiesGopherProxy)
-
-        return proxies
+        return _get_proxies()
 
 
 
diff --git a/Mac/Modules/_scproxy.c b/Mac/Modules/_scproxy.c
new file mode 100644 (file)
index 0000000..003f6a4
--- /dev/null
@@ -0,0 +1,226 @@
+/*
+ * Helper method for urllib to fetch the proxy configuration settings
+ * using the SystemConfiguration framework.
+ */
+#include <Python.h>
+#include <SystemConfiguration/SystemConfiguration.h>
+
+static int32_t 
+cfnum_to_int32(CFNumberRef num)
+{
+       int32_t result;
+
+       CFNumberGetValue(num, kCFNumberSInt32Type, &result);
+       return result;
+}
+
+static PyObject*
+cfstring_to_pystring(CFStringRef ref)
+{
+       const char* s; 
+
+       s = CFStringGetCStringPtr(ref, kCFStringEncodingUTF8);
+       if (s) {
+               return PyString_FromString(s);
+
+       } else {
+               CFIndex len = CFStringGetLength(ref);
+               Boolean ok;
+               PyObject* result;
+               result = PyString_FromStringAndSize(NULL, len*4);
+
+               ok = CFStringGetCString(ref, 
+                               PyString_AS_STRING(result),
+                               PyString_GET_SIZE(result),
+                               kCFStringEncodingUTF8);
+               if (!ok) {
+                       Py_DECREF(result);
+                       return NULL;
+               } else {
+                       _PyString_Resize(&result, 
+                               strlen(PyString_AS_STRING(result)));
+               }
+               return result;
+       }
+}
+
+
+static PyObject*
+get_proxy_settings(PyObject* mod __attribute__((__unused__)))
+{
+       CFDictionaryRef proxyDict = NULL;
+       CFNumberRef aNum = NULL;
+       CFArrayRef anArray = NULL;
+       PyObject* result = NULL;
+       PyObject* v;
+       int r;
+
+       proxyDict = SCDynamicStoreCopyProxies(NULL);
+       if (!proxyDict) {
+               Py_INCREF(Py_None);
+               return Py_None;
+       }
+
+       result = PyDict_New();
+       if (result == NULL) goto error;
+
+       aNum = CFDictionaryGetValue(proxyDict, 
+                       kSCPropNetProxiesExcludeSimpleHostnames);
+       if (aNum == NULL) {
+               v = PyBool_FromLong(0);
+       } else {
+               v = PyBool_FromLong(cfnum_to_int32(aNum));
+       }
+       if (v == NULL) goto error;
+
+       r = PyDict_SetItemString(result, "exclude_simple", v);
+       Py_DECREF(v); v = NULL;
+       if (r == -1) goto error;
+
+       anArray = CFDictionaryGetValue(proxyDict, 
+                       kSCPropNetProxiesExceptionsList);
+       if (anArray != NULL) {
+               CFIndex len = CFArrayGetCount(anArray);
+               CFIndex i;
+               v = PyTuple_New(len);
+               if (v == NULL) goto error;
+
+               r = PyDict_SetItemString(result, "exceptions", v);
+               Py_DECREF(v);
+               if (r == -1) goto error;
+
+               for (i = 0; i < len; i++) {
+                       CFStringRef aString = NULL;
+
+                       aString = CFArrayGetValueAtIndex(anArray, i);
+                       if (aString == NULL) {
+                               PyTuple_SetItem(v, i, Py_None);
+                               Py_INCREF(Py_None);
+                       } else {
+                               PyObject* t = cfstring_to_pystring(aString);
+                               if (!t) {
+                                       PyTuple_SetItem(v, i, Py_None);
+                                       Py_INCREF(Py_None);
+                               } else {
+                                       PyTuple_SetItem(v, i, t);
+                               }
+                       }
+               }
+       }
+
+       CFRelease(proxyDict);
+       return result;
+
+error:
+       if (proxyDict)  CFRelease(proxyDict);
+       Py_XDECREF(result);
+       return NULL;
+}
+
+static int
+set_proxy(PyObject* proxies, char* proto, CFDictionaryRef proxyDict,
+               CFStringRef enabledKey, 
+               CFStringRef hostKey, CFStringRef portKey)
+{
+       CFNumberRef aNum;
+
+       aNum = CFDictionaryGetValue(proxyDict, enabledKey);
+       if (aNum && cfnum_to_int32(aNum)) {
+               CFStringRef hostString;
+
+               hostString = CFDictionaryGetValue(proxyDict, hostKey);
+               aNum = CFDictionaryGetValue(proxyDict, portKey);
+
+               if (hostString) {
+                       int r;
+                       PyObject* h = cfstring_to_pystring(hostString);
+                       PyObject* v;
+                       if (h) {
+                               if (aNum) {
+                                       int32_t port = cfnum_to_int32(aNum);
+                                       v = PyString_FromFormat("http://%s:%ld",
+                                               PyString_AS_STRING(h),
+                                               (long)port);
+                               } else {
+                                       v = PyString_FromFormat("http://%s",
+                                               PyString_AS_STRING(h));
+                               }
+                               Py_DECREF(h);
+                               if (!v) return -1;
+                               r = PyDict_SetItemString(proxies, proto,
+                                       v);
+                               Py_DECREF(v);
+                               return r;
+                       }
+               }
+
+       }
+       return 0;
+}
+
+
+
+static PyObject*
+get_proxies(PyObject* mod __attribute__((__unused__)))
+{
+       PyObject* result = NULL;
+       int r;
+       CFDictionaryRef proxyDict = NULL;
+
+       proxyDict = SCDynamicStoreCopyProxies(NULL);
+       if (proxyDict == NULL) {
+               return PyDict_New();
+       }
+
+       result = PyDict_New();
+       if (result == NULL) goto error;
+
+       r = set_proxy(result, "http", proxyDict,
+               kSCPropNetProxiesHTTPEnable,
+               kSCPropNetProxiesHTTPProxy,
+               kSCPropNetProxiesHTTPPort);
+       if (r == -1) goto error;
+       r = set_proxy(result, "https", proxyDict,
+               kSCPropNetProxiesHTTPSEnable,
+               kSCPropNetProxiesHTTPSProxy,
+               kSCPropNetProxiesHTTPSPort);
+       if (r == -1) goto error;
+       r = set_proxy(result, "ftp", proxyDict,
+               kSCPropNetProxiesFTPEnable,
+               kSCPropNetProxiesFTPProxy,
+               kSCPropNetProxiesFTPPort);
+       if (r == -1) goto error;
+       r = set_proxy(result, "gopher", proxyDict,
+               kSCPropNetProxiesGopherEnable,
+               kSCPropNetProxiesGopherProxy,
+               kSCPropNetProxiesGopherPort);
+       if (r == -1) goto error;
+
+       CFRelease(proxyDict);
+       return result;
+error:
+       if (proxyDict)  CFRelease(proxyDict);
+       Py_XDECREF(result);
+       return NULL;
+}
+
+static PyMethodDef mod_methods[] = {
+       {
+               "_get_proxy_settings",
+               (PyCFunction)get_proxy_settings,
+               METH_NOARGS,
+               NULL,
+       },
+       {
+               "_get_proxies",
+               (PyCFunction)get_proxies,
+               METH_NOARGS,
+               NULL,
+       },
+       { 0, 0, 0, 0 }
+};
+
+void init_scproxy(void)
+{
+       (void)Py_InitModule4("_scproxy", mod_methods, NULL, NULL, PYTHON_API_VERSION);
+}
index cac78e1c947746e7d86bc06fd578b3d7629894f5..bdd121cb8d34a536e79fc717c37c12168a4a833b 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -379,6 +379,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #6851: Fix urllib.urlopen crash on secondairy threads on OSX 10.6
+
 - Issue #4606: Passing 'None' if ctypes argtype is set to POINTER(...)
   does now always result in NULL.
 
index e12869721d3fcb3c8b0855384593bea03224d253..adc72c894ed711500ec870bd890f529b7f7e1a38 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -1400,6 +1400,17 @@ class PyBuildExt(build_ext):
             addMacExtension('_CF', core_kwds, ['cf/pycfbridge.c'])
             addMacExtension('autoGIL', core_kwds)
 
+            # _scproxy
+            sc_kwds = {
+                'extra_compile_args': carbon_extra_compile_args,
+                'extra_link_args': [
+                    '-framework', 'SystemConfiguration',
+                    '-framework', 'CoreFoundation'
+                ],
+            }
+            addMacExtension("_scproxy", sc_kwds)
+
+
             # Carbon
             carbon_kwds = {'extra_compile_args': carbon_extra_compile_args,
                            'extra_link_args': ['-framework', 'Carbon'],