PyObject *im_func; /* The callable object implementing the method */
PyObject *im_self; /* The instance it is bound to */
PyObject *im_weakreflist; /* List of weak references */
+ vectorcallfunc vectorcall;
} PyMethodObject;
PyAPI_DATA(PyTypeObject) PyMethod_Type;
/* Suggested size (number of positional arguments) for arrays of PyObject*
allocated on a C stack to avoid allocating memory on the heap memory. Such
array is used to pass positional arguments to call functions of the
- _PyObject_FastCall() family.
+ _PyObject_Vectorcall() family.
The size is chosen to not abuse the C stack and so limit the risk of stack
overflow. The size is also chosen to allow using the small stack for most
#define _PY_FASTCALL_SMALL_STACK 5
/* Return 1 if callable supports FASTCALL calling convention for positional
- arguments: see _PyObject_FastCallDict() and _PyObject_FastCallKeywords() */
+ arguments: see _PyObject_Vectorcall() and _PyObject_FastCallDict() */
PyAPI_FUNC(int) _PyObject_HasFastCall(PyObject *callable);
-/* Call the callable object 'callable' with the "fast call" calling convention:
- args is a C array for positional arguments (nargs is the number of
- positional arguments), kwargs is a dictionary for keyword arguments.
+PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult(PyObject *callable,
+ PyObject *result,
+ const char *where);
- If nargs is equal to zero, args can be NULL. kwargs can be NULL.
- nargs must be greater or equal to zero.
+/* === Vectorcall protocol (PEP 590) ============================= */
- Return the result on success. Raise an exception and return NULL on
- error. */
-PyAPI_FUNC(PyObject *) _PyObject_FastCallDict(
+/* Call callable using tp_call. Arguments are like _PyObject_Vectorcall()
+ or _PyObject_FastCallDict() (both forms are supported),
+ except that nargs is plainly the number of arguments without flags. */
+PyAPI_FUNC(PyObject *) _PyObject_MakeTpCall(
PyObject *callable,
- PyObject *const *args,
- Py_ssize_t nargs,
- PyObject *kwargs);
+ PyObject *const *args, Py_ssize_t nargs,
+ PyObject *keywords);
-/* Call the callable object 'callable' with the "fast call" calling convention:
- args is a C array for positional arguments followed by values of
- keyword arguments. Keys of keyword arguments are stored as a tuple
- of strings in kwnames. nargs is the number of positional parameters at
- the beginning of stack. The size of kwnames gives the number of keyword
- values in the stack after positional arguments.
+#define PY_VECTORCALL_ARGUMENTS_OFFSET ((size_t)1 << (8 * sizeof(size_t) - 1))
- kwnames must only contains str strings, no subclass, and all keys must
- be unique.
+static inline Py_ssize_t
+PyVectorcall_NARGS(size_t n)
+{
+ return n & ~PY_VECTORCALL_ARGUMENTS_OFFSET;
+}
+
+static inline vectorcallfunc
+_PyVectorcall_Function(PyObject *callable)
+{
+ PyTypeObject *tp = Py_TYPE(callable);
+ if (!PyType_HasFeature(tp, _Py_TPFLAGS_HAVE_VECTORCALL)) {
+ return NULL;
+ }
+ assert(PyCallable_Check(callable));
+ Py_ssize_t offset = tp->tp_vectorcall_offset;
+ assert(offset > 0);
+ vectorcallfunc *ptr = (vectorcallfunc *)(((char *)callable) + offset);
+ return *ptr;
+}
+
+/* Call the callable object 'callable' with the "vectorcall" calling
+ convention.
+
+ args is a C array for positional arguments.
+
+ nargsf is the number of positional arguments plus optionally the flag
+ PY_VECTORCALL_ARGUMENTS_OFFSET which means that the caller is allowed to
+ modify args[-1].
- If nargs is equal to zero and there is no keyword argument (kwnames is
- NULL or its size is zero), args can be NULL.
+ kwnames is a tuple of keyword names. The values of the keyword arguments
+ are stored in "args" after the positional arguments (note that the number
+ of keyword arguments does not change nargsf). kwnames can also be NULL if
+ there are no keyword arguments.
+
+ keywords must only contains str strings (no subclass), and all keys must
+ be unique.
Return the result on success. Raise an exception and return NULL on
error. */
-PyAPI_FUNC(PyObject *) _PyObject_FastCallKeywords(
+static inline PyObject *
+_PyObject_Vectorcall(PyObject *callable, PyObject *const *args,
+ size_t nargsf, PyObject *kwnames)
+{
+ assert(kwnames == NULL || PyTuple_Check(kwnames));
+ assert(args != NULL || PyVectorcall_NARGS(nargsf) == 0);
+ vectorcallfunc func = _PyVectorcall_Function(callable);
+ if (func == NULL) {
+ Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
+ return _PyObject_MakeTpCall(callable, args, nargs, kwnames);
+ }
+ PyObject *res = func(callable, args, nargsf, kwnames);
+ return _Py_CheckFunctionResult(callable, res, NULL);
+}
+
+/* Same as _PyObject_Vectorcall except that keyword arguments are passed as
+ dict, which may be NULL if there are no keyword arguments. */
+PyAPI_FUNC(PyObject *) _PyObject_FastCallDict(
PyObject *callable,
PyObject *const *args,
- Py_ssize_t nargs,
- PyObject *kwnames);
+ size_t nargsf,
+ PyObject *kwargs);
-#define _PyObject_FastCall(func, args, nargs) \
- _PyObject_FastCallDict((func), (args), (nargs), NULL)
+/* Call "callable" (which must support vectorcall) with positional arguments
+ "tuple" and keyword arguments "dict". "dict" may also be NULL */
+PyAPI_FUNC(PyObject *) PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *dict);
-#define _PyObject_CallNoArg(func) \
- _PyObject_FastCallDict((func), NULL, 0, NULL)
+/* Same as _PyObject_Vectorcall except without keyword arguments */
+static inline PyObject *
+_PyObject_FastCall(PyObject *func, PyObject *const *args, Py_ssize_t nargs)
+{
+ return _PyObject_Vectorcall(func, args, (size_t)nargs, NULL);
+}
+
+/* Call a callable without any arguments */
+static inline PyObject *
+_PyObject_CallNoArg(PyObject *func) {
+ return _PyObject_Vectorcall(func, NULL, 0, NULL);
+}
PyAPI_FUNC(PyObject *) _PyObject_Call_Prepend(
PyObject *callable,
PyObject *const *args,
Py_ssize_t nargs);
-PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult(PyObject *callable,
- PyObject *result,
- const char *where);
-
/* Like PyObject_CallMethod(), but expect a _Py_Identifier*
as the method name. */
PyAPI_FUNC(PyObject *) _PyObject_CallMethodId(PyObject *obj,
typedef int (*getbufferproc)(PyObject *, Py_buffer *, int);
typedef void (*releasebufferproc)(PyObject *, Py_buffer *);
+typedef PyObject *(*vectorcallfunc)(PyObject *callable, PyObject *const *args,
+ size_t nargsf, PyObject *kwnames);
+
/* Maximum number of dimensions */
#define PyBUF_MAX_NDIM 64
releasebufferproc bf_releasebuffer;
} PyBufferProcs;
-/* We can't provide a full compile-time check that limited-API
- users won't implement tp_print. However, not defining printfunc
- and making tp_print of a different function pointer type
- if Py_LIMITED_API is set should at least cause a warning
- in most cases. */
-typedef int (*printfunc)(PyObject *, FILE *, int);
+/* Allow printfunc in the tp_vectorcall_offset slot for
+ * backwards-compatibility */
+typedef Py_ssize_t printfunc;
typedef struct _typeobject {
PyObject_VAR_HEAD
/* Methods to implement standard operations */
destructor tp_dealloc;
- printfunc tp_print;
+ Py_ssize_t tp_vectorcall_offset;
getattrfunc tp_getattr;
setattrfunc tp_setattr;
PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2)
unsigned int tp_version_tag;
destructor tp_finalize;
+ vectorcallfunc tp_vectorcall;
#ifdef COUNT_ALLOCS
/* these must be last and never explicitly initialized */
typedef struct {
PyDescr_COMMON;
PyMethodDef *d_method;
+ vectorcallfunc vectorcall;
} PyMethodDescrObject;
typedef struct {
#ifndef Py_LIMITED_API
PyAPI_FUNC(PyObject *) _PyMethodDescr_FastCallKeywords(
- PyObject *descrobj, PyObject *const *stack, Py_ssize_t nargs, PyObject *kwnames);
+ PyObject *descrobj, PyObject *const *args, size_t nargsf, PyObject *kwnames);
PyAPI_FUNC(PyObject *) PyDescr_NewWrapper(PyTypeObject *,
struct wrapperbase *, void *);
#define PyDescr_IsData(d) (Py_TYPE(d)->tp_descr_set != NULL)
PyObject *func_module; /* The __module__ attribute, can be anything */
PyObject *func_annotations; /* Annotations, a dict or NULL */
PyObject *func_qualname; /* The qualified name */
+ vectorcallfunc vectorcall;
/* Invariant:
* func_closure contains the bindings for func_code->co_freevars, so
PyAPI_FUNC(PyObject *) _PyFunction_FastCallKeywords(
PyObject *func,
PyObject *const *stack,
- Py_ssize_t nargs,
+ size_t nargsf,
PyObject *kwnames);
#endif
PyAPI_FUNC(PyObject *) _PyCFunction_FastCallKeywords(PyObject *func,
PyObject *const *stack,
- Py_ssize_t nargs,
+ size_t nargsf,
PyObject *kwnames);
#endif
PyObject *m_self; /* Passed as 'self' arg to the C func, can be NULL */
PyObject *m_module; /* The __module__ attribute, can be anything */
PyObject *m_weakreflist; /* List of weak references */
+ vectorcallfunc vectorcall;
} PyCFunctionObject;
PyAPI_FUNC(PyObject *) _PyMethodDef_RawFastCallDict(
/* Set if the type allows subclassing */
#define Py_TPFLAGS_BASETYPE (1UL << 10)
+/* Set if the type implements the vectorcall protocol (PEP 590) */
+#ifndef Py_LIMITED_API
+#define _Py_TPFLAGS_HAVE_VECTORCALL (1UL << 11)
+#endif
+
/* Set if the type is 'ready' -- fully initialized */
#define Py_TPFLAGS_READY (1UL << 12)
result = _testcapi.pyobject_fastcall(func, None)
self.check_result(result, expected)
- def test_fastcall_dict(self):
+ def test_vectorcall_dict(self):
# Test _PyObject_FastCallDict()
for func, args, expected in self.CALLS_POSARGS:
result = _testcapi.pyobject_fastcalldict(func, args, kwargs)
self.check_result(result, expected)
- def test_fastcall_keywords(self):
- # Test _PyObject_FastCallKeywords()
+ def test_vectorcall(self):
+ # Test _PyObject_Vectorcall()
for func, args, expected in self.CALLS_POSARGS:
with self.subTest(func=func, args=args):
# kwnames=NULL
- result = _testcapi.pyobject_fastcallkeywords(func, args, None)
+ result = _testcapi.pyobject_vectorcall(func, args, None)
self.check_result(result, expected)
# kwnames=()
- result = _testcapi.pyobject_fastcallkeywords(func, args, ())
+ result = _testcapi.pyobject_vectorcall(func, args, ())
self.check_result(result, expected)
if not args:
# kwnames=NULL
- result = _testcapi.pyobject_fastcallkeywords(func, None, None)
+ result = _testcapi.pyobject_vectorcall(func, None, None)
self.check_result(result, expected)
# kwnames=()
- result = _testcapi.pyobject_fastcallkeywords(func, None, ())
+ result = _testcapi.pyobject_vectorcall(func, None, ())
self.check_result(result, expected)
for func, args, kwargs, expected in self.CALLS_KWARGS:
with self.subTest(func=func, args=args, kwargs=kwargs):
kwnames = tuple(kwargs.keys())
args = args + tuple(kwargs.values())
- result = _testcapi.pyobject_fastcallkeywords(func, args, kwnames)
+ result = _testcapi.pyobject_vectorcall(func, args, kwnames)
self.check_result(result, expected)
def test_fastcall_clearing_dict(self):
"""some doc"""
return self
+def testfunction_kw(self, *, kw):
+ """some doc"""
+ return self
+
+
class InstanceMethod:
id = _testcapi.instancemethod(id)
testfunction = _testcapi.instancemethod(testfunction)
pass
self.assertFalse(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_METHOD_DESCRIPTOR)
+ def test_vectorcall(self):
+ # Test a bunch of different ways to call objects:
+ # 1. normal call
+ # 2. vectorcall using _PyObject_Vectorcall()
+ # 3. vectorcall using PyVectorcall_Call()
+ # 4. call as bound method
+ # 5. call using functools.partial
+
+ # A list of (function, args, kwargs, result) calls to test
+ calls = [(len, (range(42),), {}, 42),
+ (list.append, ([], 0), {}, None),
+ ([].append, (0,), {}, None),
+ (sum, ([36],), {"start":6}, 42),
+ (testfunction, (42,), {}, 42),
+ (testfunction_kw, (42,), {"kw":None}, 42)]
+
+ from _testcapi import pyobject_vectorcall, pyvectorcall_call
+ from types import MethodType
+ from functools import partial
+
+ def vectorcall(func, args, kwargs):
+ args = *args, *kwargs.values()
+ kwnames = tuple(kwargs)
+ return pyobject_vectorcall(func, args, kwnames)
+
+ for (func, args, kwargs, expected) in calls:
+ with self.subTest(str(func)):
+ args1 = args[1:]
+ meth = MethodType(func, args[0])
+ wrapped = partial(func)
+ if not kwargs:
+ self.assertEqual(expected, func(*args))
+ self.assertEqual(expected, pyobject_vectorcall(func, args, None))
+ self.assertEqual(expected, pyvectorcall_call(func, args))
+ self.assertEqual(expected, meth(*args1))
+ self.assertEqual(expected, wrapped(*args))
+ self.assertEqual(expected, func(*args, **kwargs))
+ self.assertEqual(expected, vectorcall(func, args, kwargs))
+ self.assertEqual(expected, pyvectorcall_call(func, args, kwargs))
+ self.assertEqual(expected, meth(*args1, **kwargs))
+ self.assertEqual(expected, wrapped(*args, **kwargs))
+
class SubinterpreterTest(unittest.TestCase):
# buffer
# XXX
# builtin_function_or_method
- check(len, size('4P')) # XXX check layout
+ check(len, size('5P'))
# bytearray
samples = [b'', b'u'*100000]
for sample in samples:
# complex
check(complex(0,1), size('2d'))
# method_descriptor (descriptor object)
- check(str.lower, size('3PP'))
+ check(str.lower, size('3PPP'))
# classmethod_descriptor (descriptor object)
# XXX
# member_descriptor (descriptor object)
check(x, vsize('5P2c4P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
# function
def func(): pass
- check(func, size('12P'))
+ check(func, size('13P'))
class c():
@staticmethod
def foo():
check((1,2,3), vsize('') + 3*self.P)
# type
# static type: PyTypeObject
- fmt = 'P2n15Pl4Pn9Pn11PIP'
+ fmt = 'P2nPI13Pl4Pn9Pn11PIPP'
if hasattr(sys, 'getcounts'):
fmt += '3n2P'
s = vsize(fmt)
--- /dev/null
+Implement :pep:`590`: Vectorcall: a fast calling protocol for CPython.
+This is a new protocol to optimize calls of custom callable objects.
}
stack[nargs] = (PyObject *)ctx;
- handle = _PyObject_FastCallKeywords(
- callable, stack, nargs, context_kwname);
+ handle = _PyObject_Vectorcall(callable, stack, nargs, context_kwname);
Py_DECREF(callable);
}
PyObject *stack[2];
stack[0] = wrapper;
stack[1] = (PyObject *)task->task_context;
- res = _PyObject_FastCallKeywords(
- add_cb, stack, 1, context_kwname);
+ res = _PyObject_Vectorcall(add_cb, stack, 1, context_kwname);
Py_DECREF(add_cb);
Py_DECREF(wrapper);
if (res == NULL) {
static PyObject *
-test_pyobject_fastcallkeywords(PyObject *self, PyObject *args)
+test_pyobject_vectorcall(PyObject *self, PyObject *args)
{
PyObject *func, *func_args, *kwnames = NULL;
PyObject **stack;
PyErr_SetString(PyExc_TypeError, "kwnames must be None or a tuple");
return NULL;
}
- return _PyObject_FastCallKeywords(func, stack, nargs, kwnames);
+ return _PyObject_Vectorcall(func, stack, nargs, kwnames);
+}
+
+
+static PyObject *
+test_pyvectorcall_call(PyObject *self, PyObject *args)
+{
+ PyObject *func;
+ PyObject *argstuple;
+ PyObject *kwargs = NULL;
+
+ if (!PyArg_ParseTuple(args, "OO|O", &func, &argstuple, &kwargs)) {
+ return NULL;
+ }
+
+ if (!PyTuple_Check(argstuple)) {
+ PyErr_SetString(PyExc_TypeError, "args must be a tuple");
+ return NULL;
+ }
+ if (kwargs != NULL && !PyDict_Check(kwargs)) {
+ PyErr_SetString(PyExc_TypeError, "kwargs must be a dict");
+ return NULL;
+ }
+
+ return PyVectorcall_Call(func, argstuple, kwargs);
}
{"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS},
{"pyobject_fastcall", test_pyobject_fastcall, METH_VARARGS},
{"pyobject_fastcalldict", test_pyobject_fastcalldict, METH_VARARGS},
- {"pyobject_fastcallkeywords", test_pyobject_fastcallkeywords, METH_VARARGS},
+ {"pyobject_vectorcall", test_pyobject_vectorcall, METH_VARARGS},
+ {"pyvectorcall_call", test_pyvectorcall_call, METH_VARARGS},
{"stack_pointer", stack_pointer, METH_NOARGS},
#ifdef W_STOPCODE
{"W_STOPCODE", py_w_stopcode, METH_VARARGS},
#include "frameobject.h"
+static PyObject *
+cfunction_call_varargs(PyObject *func, PyObject *args, PyObject *kwargs);
+
+
int
_PyObject_HasFastCall(PyObject *callable)
{
/* --- Core PyObject call functions ------------------------------- */
PyObject *
-_PyObject_FastCallDict(PyObject *callable, PyObject *const *args, Py_ssize_t nargs,
- PyObject *kwargs)
+_PyObject_FastCallDict(PyObject *callable, PyObject *const *args,
+ size_t nargsf, PyObject *kwargs)
{
/* _PyObject_FastCallDict() must not be called with an exception set,
because it can clear it (directly or indirectly) and so the
caller loses its exception */
assert(!PyErr_Occurred());
-
assert(callable != NULL);
+
+ Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
assert(nargs >= 0);
assert(nargs == 0 || args != NULL);
assert(kwargs == NULL || PyDict_Check(kwargs));
- if (PyFunction_Check(callable)) {
- return _PyFunction_FastCallDict(callable, args, nargs, kwargs);
+ vectorcallfunc func = _PyVectorcall_Function(callable);
+ if (func == NULL) {
+ /* Use tp_call instead */
+ return _PyObject_MakeTpCall(callable, args, nargs, kwargs);
}
- else if (PyCFunction_Check(callable)) {
- return _PyCFunction_FastCallDict(callable, args, nargs, kwargs);
+
+ PyObject *res;
+ if (kwargs == NULL) {
+ res = func(callable, args, nargsf, NULL);
}
else {
- PyObject *argstuple, *result;
- ternaryfunc call;
-
- /* Slow-path: build a temporary tuple */
- call = callable->ob_type->tp_call;
- if (call == NULL) {
- PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable",
- callable->ob_type->tp_name);
- return NULL;
- }
-
- argstuple = _PyTuple_FromArray(args, nargs);
- if (argstuple == NULL) {
+ PyObject *kwnames;
+ PyObject *const *newargs;
+ if (_PyStack_UnpackDict(args, nargs, kwargs, &newargs, &kwnames) < 0) {
return NULL;
}
-
- if (Py_EnterRecursiveCall(" while calling a Python object")) {
- Py_DECREF(argstuple);
- return NULL;
+ res = func(callable, newargs, nargs, kwnames);
+ if (kwnames != NULL) {
+ Py_ssize_t i, n = PyTuple_GET_SIZE(kwnames) + nargs;
+ for (i = 0; i < n; i++) {
+ Py_DECREF(newargs[i]);
+ }
+ PyMem_Free((PyObject **)newargs);
+ Py_DECREF(kwnames);
}
-
- result = (*call)(callable, argstuple, kwargs);
-
- Py_LeaveRecursiveCall();
- Py_DECREF(argstuple);
-
- result = _Py_CheckFunctionResult(callable, result, NULL);
- return result;
}
+ return _Py_CheckFunctionResult(callable, res, NULL);
}
PyObject *
-_PyObject_FastCallKeywords(PyObject *callable, PyObject *const *stack, Py_ssize_t nargs,
- PyObject *kwnames)
+_PyObject_MakeTpCall(PyObject *callable, PyObject *const *args, Py_ssize_t nargs, PyObject *keywords)
{
- /* _PyObject_FastCallKeywords() must not be called with an exception set,
- because it can clear it (directly or indirectly) and so the
- caller loses its exception */
- assert(!PyErr_Occurred());
+ /* Slow path: build a temporary tuple for positional arguments and a
+ * temporary dictionary for keyword arguments (if any) */
+ ternaryfunc call = Py_TYPE(callable)->tp_call;
+ if (call == NULL) {
+ PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable",
+ Py_TYPE(callable)->tp_name);
+ return NULL;
+ }
assert(nargs >= 0);
- assert(kwnames == NULL || PyTuple_CheckExact(kwnames));
-
- /* kwnames must only contains str strings, no subclass, and all keys must
- be unique: these checks are implemented in Python/ceval.c and
- _PyArg_ParseStackAndKeywords(). */
-
- if (PyFunction_Check(callable)) {
- return _PyFunction_FastCallKeywords(callable, stack, nargs, kwnames);
+ assert(nargs == 0 || args != NULL);
+ assert(keywords == NULL || PyTuple_Check(keywords) || PyDict_Check(keywords));
+ PyObject *argstuple = _PyTuple_FromArray(args, nargs);
+ if (argstuple == NULL) {
+ return NULL;
}
- if (PyCFunction_Check(callable)) {
- return _PyCFunction_FastCallKeywords(callable, stack, nargs, kwnames);
+
+ PyObject *kwdict;
+ if (keywords == NULL || PyDict_Check(keywords)) {
+ kwdict = keywords;
}
else {
- /* Slow-path: build a temporary tuple for positional arguments and a
- temporary dictionary for keyword arguments (if any) */
-
- ternaryfunc call;
- PyObject *argstuple;
- PyObject *kwdict, *result;
- Py_ssize_t nkwargs;
-
- nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames);
- assert((nargs == 0 && nkwargs == 0) || stack != NULL);
-
- call = callable->ob_type->tp_call;
- if (call == NULL) {
- PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable",
- callable->ob_type->tp_name);
- return NULL;
- }
-
- argstuple = _PyTuple_FromArray(stack, nargs);
- if (argstuple == NULL) {
- return NULL;
- }
-
- if (nkwargs > 0) {
- kwdict = _PyStack_AsDict(stack + nargs, kwnames);
+ if (PyTuple_GET_SIZE(keywords)) {
+ assert(args != NULL);
+ kwdict = _PyStack_AsDict(args + nargs, keywords);
if (kwdict == NULL) {
Py_DECREF(argstuple);
return NULL;
}
}
else {
- kwdict = NULL;
+ keywords = kwdict = NULL;
}
+ }
- if (Py_EnterRecursiveCall(" while calling a Python object")) {
- Py_DECREF(argstuple);
- Py_XDECREF(kwdict);
- return NULL;
- }
+ PyObject *result = NULL;
+ if (Py_EnterRecursiveCall(" while calling a Python object") == 0)
+ {
+ result = call(callable, argstuple, kwdict);
+ Py_LeaveRecursiveCall();
+ }
- result = (*call)(callable, argstuple, kwdict);
+ Py_DECREF(argstuple);
+ if (kwdict != keywords) {
+ Py_DECREF(kwdict);
+ }
- Py_LeaveRecursiveCall();
+ result = _Py_CheckFunctionResult(callable, result, NULL);
+ return result;
+}
- Py_DECREF(argstuple);
- Py_XDECREF(kwdict);
- result = _Py_CheckFunctionResult(callable, result, NULL);
- return result;
+PyObject *
+PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *kwargs)
+{
+ vectorcallfunc func = _PyVectorcall_Function(callable);
+ if (func == NULL) {
+ PyErr_Format(PyExc_TypeError, "'%.200s' object does not support vectorcall",
+ Py_TYPE(callable)->tp_name);
+ return NULL;
}
+ PyObject *const *args;
+ Py_ssize_t nargs = PyTuple_GET_SIZE(tuple);
+ PyObject *kwnames;
+ if (_PyStack_UnpackDict(_PyTuple_ITEMS(tuple), nargs,
+ kwargs, &args, &kwnames) < 0) {
+ return NULL;
+ }
+ PyObject *result = func(callable, args, nargs, kwnames);
+ if (kwnames != NULL) {
+ Py_ssize_t i, n = PyTuple_GET_SIZE(kwnames) + nargs;
+ for (i = 0; i < n; i++) {
+ Py_DECREF(args[i]);
+ }
+ PyMem_Free((PyObject **)args);
+ Py_DECREF(kwnames);
+ }
+
+ return result;
}
assert(PyTuple_Check(args));
assert(kwargs == NULL || PyDict_Check(kwargs));
- if (PyFunction_Check(callable)) {
- return _PyFunction_FastCallDict(callable,
- _PyTuple_ITEMS(args),
- PyTuple_GET_SIZE(args),
- kwargs);
+ if (_PyVectorcall_Function(callable) != NULL) {
+ return PyVectorcall_Call(callable, args, kwargs);
}
else if (PyCFunction_Check(callable)) {
- return PyCFunction_Call(callable, args, kwargs);
+ /* This must be a METH_VARARGS function, otherwise we would be
+ * in the previous case */
+ return cfunction_call_varargs(callable, args, kwargs);
}
else {
call = callable->ob_type->tp_call;
return result;
}
+
PyObject *
-_PyFunction_FastCallKeywords(PyObject *func, PyObject *const *stack,
- Py_ssize_t nargs, PyObject *kwnames)
+_PyFunction_FastCallKeywords(PyObject *func, PyObject* const* stack,
+ size_t nargsf, PyObject *kwnames)
{
PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func);
PyObject *globals = PyFunction_GET_GLOBALS(func);
Py_ssize_t nd;
assert(PyFunction_Check(func));
+ Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
assert(nargs >= 0);
assert(kwnames == NULL || PyTuple_CheckExact(kwnames));
assert((nargs == 0 && nkwargs == 0) || stack != NULL);
PyObject *
_PyCFunction_FastCallKeywords(PyObject *func,
- PyObject *const *args, Py_ssize_t nargs,
+ PyObject *const *args, size_t nargsf,
PyObject *kwnames)
{
PyObject *result;
assert(func != NULL);
assert(PyCFunction_Check(func));
+ Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
result = _PyMethodDef_RawFastCallKeywords(((PyCFunctionObject*)func)->m_ml,
PyCFunction_GET_SELF(func),
PyObject *self = PyCFunction_GET_SELF(func);
PyObject *result;
+ assert(PyCFunction_GET_FLAGS(func) & METH_VARARGS);
if (PyCFunction_GET_FLAGS(func) & METH_KEYWORDS) {
if (Py_EnterRecursiveCall(" while calling a Python object")) {
return NULL;
PyObject *
PyCFunction_Call(PyObject *func, PyObject *args, PyObject *kwargs)
{
- /* first try METH_VARARGS to pass directly args tuple unchanged.
- _PyMethodDef_RawFastCallDict() creates a new temporary tuple
- for METH_VARARGS. */
+ /* For METH_VARARGS, we cannot use vectorcall as the vectorcall pointer
+ * is NULL. This is intentional, since vectorcall would be slower. */
if (PyCFunction_GET_FLAGS(func) & METH_VARARGS) {
return cfunction_call_varargs(func, args, kwargs);
}
- else {
- return _PyCFunction_FastCallDict(func,
- _PyTuple_ITEMS(args),
- PyTuple_GET_SIZE(args),
- kwargs);
- }
+ return PyVectorcall_Call(func, args, kwargs);
}
return ((PyMethodObject *)im)->im_self;
}
+
+static PyObject *
+method_vectorcall(PyObject *method, PyObject *const *args,
+ size_t nargsf, PyObject *kwnames)
+{
+ assert(Py_TYPE(method) == &PyMethod_Type);
+ PyObject *self, *func, *result;
+ self = PyMethod_GET_SELF(method);
+ func = PyMethod_GET_FUNCTION(method);
+ Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
+
+ if (nargsf & PY_VECTORCALL_ARGUMENTS_OFFSET) {
+ /* PY_VECTORCALL_ARGUMENTS_OFFSET is set, so we are allowed to mutate the vector */
+ PyObject **newargs = (PyObject**)args - 1;
+ nargs += 1;
+ PyObject *tmp = newargs[0];
+ newargs[0] = self;
+ result = _PyObject_Vectorcall(func, newargs, nargs, kwnames);
+ newargs[0] = tmp;
+ }
+ else {
+ Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames);
+ PyObject **newargs;
+ Py_ssize_t totalargs = nargs + nkwargs;
+ newargs = PyMem_Malloc((totalargs+1) * sizeof(PyObject *));
+ if (newargs == NULL) {
+ PyErr_NoMemory();
+ return NULL;
+ }
+ /* use borrowed references */
+ newargs[0] = self;
+ memcpy(newargs + 1, args, totalargs * sizeof(PyObject *));
+ result = _PyObject_Vectorcall(func, newargs, nargs+1, kwnames);
+ PyMem_Free(newargs);
+ }
+ return result;
+}
+
+
/* Method objects are used for bound instance methods returned by
instancename.methodname. ClassName.methodname returns an ordinary
function.
im->im_func = func;
Py_XINCREF(self);
im->im_self = self;
+ im->vectorcall = method_vectorcall;
_PyObject_GC_TRACK(im);
return (PyObject *)im;
}
sizeof(PyMethodObject),
0,
(destructor)method_dealloc, /* tp_dealloc */
- 0, /* tp_print */
+ offsetof(PyMethodObject, vectorcall), /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_reserved */
method_getattro, /* tp_getattro */
PyObject_GenericSetAttr, /* tp_setattro */
0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
+ _Py_TPFLAGS_HAVE_VECTORCALL, /* tp_flags */
method_doc, /* tp_doc */
(traverseproc)method_traverse, /* tp_traverse */
0, /* tp_clear */
// same to methoddescr_call(), but use FASTCALL convention.
PyObject *
_PyMethodDescr_FastCallKeywords(PyObject *descrobj,
- PyObject *const *args, Py_ssize_t nargs,
+ PyObject *const *args, size_t nargsf,
PyObject *kwnames)
{
assert(Py_TYPE(descrobj) == &PyMethodDescr_Type);
PyMethodDescrObject *descr = (PyMethodDescrObject *)descrobj;
PyObject *self, *result;
+ Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
/* Make sure that the first argument is acceptable as 'self' */
if (nargs < 1) {
PyErr_Format(PyExc_TypeError,
sizeof(PyMethodDescrObject),
0,
(destructor)descr_dealloc, /* tp_dealloc */
- 0, /* tp_print */
+ offsetof(PyMethodDescrObject, vectorcall), /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_reserved */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
+ _Py_TPFLAGS_HAVE_VECTORCALL |
Py_TPFLAGS_METHOD_DESCRIPTOR, /* tp_flags */
0, /* tp_doc */
descr_traverse, /* tp_traverse */
descr = (PyMethodDescrObject *)descr_new(&PyMethodDescr_Type,
type, method->ml_name);
- if (descr != NULL)
+ if (descr != NULL) {
descr->d_method = method;
+ descr->vectorcall = &_PyMethodDescr_FastCallKeywords;
+ }
return (PyObject *)descr;
}
op->func_defaults = NULL; /* No default arguments */
op->func_kwdefaults = NULL; /* No keyword only defaults */
op->func_closure = NULL;
+ op->vectorcall = _PyFunction_FastCallKeywords;
consts = ((PyCodeObject *)code)->co_consts;
if (PyTuple_Size(consts) >= 1) {
sizeof(PyFunctionObject),
0,
(destructor)func_dealloc, /* tp_dealloc */
- 0, /* tp_print */
+ offsetof(PyFunctionObject, vectorcall), /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_reserved */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
+ _Py_TPFLAGS_HAVE_VECTORCALL |
Py_TPFLAGS_METHOD_DESCRIPTOR, /* tp_flags */
func_new__doc__, /* tp_doc */
(traverseproc)func_traverse, /* tp_traverse */
op->m_self = self;
Py_XINCREF(module);
op->m_module = module;
+ if (ml->ml_flags & METH_VARARGS) {
+ /* For METH_VARARGS functions, it's more efficient to use tp_call
+ * instead of vectorcall. */
+ op->vectorcall = NULL;
+ }
+ else {
+ op->vectorcall = &_PyCFunction_FastCallKeywords;
+ }
_PyObject_GC_TRACK(op);
return (PyObject *)op;
}
sizeof(PyCFunctionObject),
0,
(destructor)meth_dealloc, /* tp_dealloc */
- 0, /* tp_print */
+ offsetof(PyCFunctionObject, vectorcall), /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_reserved */
PyObject_GenericGetAttr, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
+ _Py_TPFLAGS_HAVE_VECTORCALL, /* tp_flags */
0, /* tp_doc */
(traverseproc)meth_traverse, /* tp_traverse */
0, /* tp_clear */
return NULL;
}
Py_INCREF(hook);
- PyObject *retval = _PyObject_FastCallKeywords(hook, args, nargs, keywords);
+ PyObject *retval = _PyObject_Vectorcall(hook, args, nargs, keywords);
Py_DECREF(hook);
return retval;
}
}
assert(nargs >= 1);
- v = _PyObject_FastCallKeywords(callable, args + 1, nargs - 1, kwnames);
+ v = _PyObject_Vectorcall(callable, args + 1, nargs - 1, kwnames);
Py_DECREF(callable);
if (v == NULL) {
Py_DECREF(newlist);
x = call; \
}
+
+static PyObject *
+trace_call_function(PyThreadState *tstate,
+ PyObject *func,
+ PyObject **args, Py_ssize_t nargs,
+ PyObject *kwnames)
+{
+ PyObject *x;
+ if (PyCFunction_Check(func)) {
+ C_TRACE(x, _PyCFunction_FastCallKeywords(func, args, nargs, kwnames));
+ return x;
+ }
+ else if (Py_TYPE(func) == &PyMethodDescr_Type && nargs > 0) {
+ /* We need to create a temporary bound method as argument
+ for profiling.
+
+ If nargs == 0, then this cannot work because we have no
+ "self". In any case, the call itself would raise
+ TypeError (foo needs an argument), so we just skip
+ profiling. */
+ PyObject *self = args[0];
+ func = Py_TYPE(func)->tp_descr_get(func, self, (PyObject*)Py_TYPE(self));
+ if (func == NULL) {
+ return NULL;
+ }
+ C_TRACE(x, _PyCFunction_FastCallKeywords(func,
+ args+1, nargs-1,
+ kwnames));
+ Py_DECREF(func);
+ return x;
+ }
+ return _PyObject_Vectorcall(func, args, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames);
+}
+
/* Issue #29227: Inline call_function() into _PyEval_EvalFrameDefault()
to reduce the stack consumption. */
Py_LOCAL_INLINE(PyObject *) _Py_HOT_FUNCTION
Py_ssize_t nargs = oparg - nkwargs;
PyObject **stack = (*pp_stack) - nargs - nkwargs;
- /* Always dispatch PyCFunction first, because these are
- presumed to be the most frequent callable object.
- */
- if (PyCFunction_Check(func)) {
- C_TRACE(x, _PyCFunction_FastCallKeywords(func, stack, nargs, kwnames));
- }
- else if (Py_TYPE(func) == &PyMethodDescr_Type) {
- if (nargs > 0 && tstate->use_tracing) {
- /* We need to create a temporary bound method as argument
- for profiling.
-
- If nargs == 0, then this cannot work because we have no
- "self". In any case, the call itself would raise
- TypeError (foo needs an argument), so we just skip
- profiling. */
- PyObject *self = stack[0];
- func = Py_TYPE(func)->tp_descr_get(func, self, (PyObject*)Py_TYPE(self));
- if (func != NULL) {
- C_TRACE(x, _PyCFunction_FastCallKeywords(func,
- stack+1, nargs-1,
- kwnames));
- Py_DECREF(func);
- }
- else {
- x = NULL;
- }
- }
- else {
- x = _PyMethodDescr_FastCallKeywords(func, stack, nargs, kwnames);
- }
+ if (tstate->use_tracing) {
+ x = trace_call_function(tstate, func, stack, nargs, kwnames);
}
else {
- if (PyMethod_Check(func) && PyMethod_GET_SELF(func) != NULL) {
- /* Optimize access to bound methods. Reuse the Python stack
- to pass 'self' as the first argument, replace 'func'
- with 'self'. It avoids the creation of a new temporary tuple
- for arguments (to replace func with self) when the method uses
- FASTCALL. */
- PyObject *self = PyMethod_GET_SELF(func);
- Py_INCREF(self);
- func = PyMethod_GET_FUNCTION(func);
- Py_INCREF(func);
- Py_SETREF(*pfunc, self);
- nargs++;
- stack--;
- }
- else {
- Py_INCREF(func);
- }
-
- if (PyFunction_Check(func)) {
- x = _PyFunction_FastCallKeywords(func, stack, nargs, kwnames);
- }
- else {
- x = _PyObject_FastCallKeywords(func, stack, nargs, kwnames);
- }
- Py_DECREF(func);
+ x = _PyObject_Vectorcall(func, stack, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames);
}
assert((x != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
return NULL;
}
- PyObject *call_result = _PyObject_FastCallKeywords(
+ PyObject *call_result = _PyObject_Vectorcall(
args[0], args + 1, nargs - 1, kwnames);
if (PyContext_Exit((PyObject *)self)) {
return NULL;
}
PyMem_RawFree(envar);
- PyObject *retval = _PyObject_FastCallKeywords(hook, args, nargs, keywords);
+ PyObject *retval = _PyObject_Vectorcall(hook, args, nargs, keywords);
Py_DECREF(hook);
return retval;
projects / python / commitdiff