bytes_types = (bytes, bytearray, memoryview)
# These are purely informational; no code uses these.
-format_version = "2.0" # File format version we write
+format_version = "3.0" # File format version we write
compatible_formats = ["1.0", # Original protocol 0
"1.1", # Protocol 0 with INST added
"1.2", # Original protocol 1
"1.3", # Protocol 1 with BINFLOAT added
"2.0", # Protocol 2
+ "3.0", # Protocol 3
] # Old format versions we can read
# This is the highest protocol number we know how to read.
-HIGHEST_PROTOCOL = 2
+HIGHEST_PROTOCOL = 3
# The protocol we write by default. May be less than HIGHEST_PROTOCOL.
-DEFAULT_PROTOCOL = 2
+# We intentionally write a protocol that Python 2.x cannot read;
+# there are too many issues with that.
+DEFAULT_PROTOCOL = 3
# Why use struct.pack() for pickling but marshal.loads() for
# unpickling? struct.pack() is 40% faster than marshal.dumps(), but
_tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]
+# Protocol 3 (Python 3.x)
+
+BINBYTES = b'B' # push bytes; counted binary string argument
+SHORT_BINBYTES = b'C' # " " ; " " " " < 256 bytes
__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$",x)])
self.write(FLOAT + repr(obj).encode("ascii") + b'\n')
dispatch[float] = save_float
- def save_string(self, obj, pack=struct.pack):
- if self.bin:
- n = len(obj)
- if n < 256:
- self.write(SHORT_BINSTRING + bytes([n]) + bytes(obj))
- else:
- self.write(BINSTRING + pack("<i", n) + bytes(obj))
+ def save_bytes(self, obj, pack=struct.pack):
+ if self.proto < 3:
+ self.save_reduce(bytes, (list(obj),))
+ return
+ n = len(obj)
+ if n < 256:
+ self.write(SHORT_BINBYTES + bytes([n]) + bytes(obj))
else:
- # Strip leading 'b' due to repr() of bytes() returning b'...'
- self.write(STRING + repr(obj).lstrip("b").encode("ascii") + b'\n')
+ self.write(BINBYTES + pack("<i", n) + bytes(obj))
self.memoize(obj)
- dispatch[bytes] = save_string
+ dispatch[bytes] = save_bytes
- def save_unicode(self, obj, pack=struct.pack):
+ def save_str(self, obj, pack=struct.pack):
if self.bin:
encoded = obj.encode('utf-8')
n = len(encoded)
self.write(UNICODE + bytes(obj.encode('raw-unicode-escape')) +
b'\n')
self.memoize(obj)
- dispatch[str] = save_unicode
+ dispatch[str] = save_str
def save_tuple(self, obj):
write = self.write
class Unpickler:
- def __init__(self, file):
+ def __init__(self, file, *, encoding="ASCII", errors="strict"):
"""This takes a binary file for reading a pickle data stream.
The protocol version of the pickle is detected automatically, so no
Thus file-like object can be a binary file object opened for
reading, a BytesIO object, or any other custom object that
meets this interface.
+
+ Optional keyword arguments are encoding and errors, which are
+ used to decode 8-bit string instances pickled by Python 2.x.
+ These default to 'ASCII' and 'strict', respectively.
"""
self.readline = file.readline
self.read = file.read
self.memo = {}
+ self.encoding = encoding
+ self.errors = errors
def load(self):
"""Read a pickled object representation from the open file.
def load_proto(self):
proto = ord(self.read(1))
- if not 0 <= proto <= 2:
+ if not 0 <= proto <= HIGHEST_PROTOCOL:
raise ValueError("unsupported pickle protocol: %d" % proto)
dispatch[PROTO[0]] = load_proto
def load_binstring(self):
len = mloads(b'i' + self.read(4))
- self.append(self.read(len))
+ data = self.read(len)
+ value = str(data, self.encoding, self.errors)
+ self.append(value)
dispatch[BINSTRING[0]] = load_binstring
+ def load_binbytes(self):
+ len = mloads(b'i' + self.read(4))
+ self.append(self.read(len))
+ dispatch[BINBYTES[0]] = load_binbytes
+
def load_unicode(self):
self.append(str(self.readline()[:-1], 'raw-unicode-escape'))
dispatch[UNICODE[0]] = load_unicode
def load_short_binstring(self):
len = ord(self.read(1))
- self.append(bytes(self.read(len)))
+ data = bytes(self.read(len))
+ value = str(data, self.encoding, self.errors)
+ self.append(value)
dispatch[SHORT_BINSTRING[0]] = load_short_binstring
+ def load_short_binbytes(self):
+ len = ord(self.read(1))
+ self.append(bytes(self.read(len)))
+ dispatch[SHORT_BINBYTES[0]] = load_short_binbytes
+
def load_tuple(self):
k = self.marker()
self.stack[k:] = [tuple(self.stack[k+1:])]
doc="A Python float object.")
pystring = StackObject(
+ name='string',
+ obtype=bytes,
+ doc="A Python (8-bit) string object.")
+
+pybytes = StackObject(
name='bytes',
obtype=bytes,
doc="A Python bytes object.")
pyunicode = StackObject(
name='str',
obtype=str,
- doc="A Python string object.")
+ doc="A Python (Unicode) string object.")
pynone = StackObject(
name="None",
assert isinstance(x, StackObject)
self.stack_after = stack_after
- assert isinstance(proto, int) and 0 <= proto <= 2
+ assert isinstance(proto, int) and 0 <= proto <= 3
self.proto = proto
assert isinstance(doc, str)
The argument is a repr-style string, with bracketing quote characters,
and perhaps embedded escapes. The argument extends until the next
- newline character.
+ newline character. (Actually, they are decoded into a str instance
+ using the encoding given to the Unpickler constructor. or the default,
+ 'ASCII'.)
"""),
I(name='BINSTRING',
There are two arguments: the first is a 4-byte little-endian signed int
giving the number of bytes in the string, and the second is that many
- bytes, which are taken literally as the string content.
+ bytes, which are taken literally as the string content. (Actually,
+ they are decoded into a str instance using the encoding given to the
+ Unpickler constructor. or the default, 'ASCII'.)
"""),
I(name='SHORT_BINSTRING',
proto=1,
doc="""Push a Python string object.
+ There are two arguments: the first is a 1-byte unsigned int giving
+ the number of bytes in the string, and the second is that many bytes,
+ which are taken literally as the string content. (Actually, they
+ are decoded into a str instance using the encoding given to the
+ Unpickler constructor. or the default, 'ASCII'.)
+ """),
+
+ # Bytes (protocol 3 only; older protocols don't support bytes at all)
+
+ I(name='BINBYTES',
+ code='B',
+ arg=string4,
+ stack_before=[],
+ stack_after=[pybytes],
+ proto=3,
+ doc="""Push a Python bytes object.
+
+ There are two arguments: the first is a 4-byte little-endian signed int
+ giving the number of bytes in the string, and the second is that many
+ bytes, which are taken literally as the bytes content.
+ """),
+
+ I(name='SHORT_BINBYTES',
+ code='C',
+ arg=string1,
+ stack_before=[],
+ stack_after=[pybytes],
+ proto=1,
+ doc="""Push a Python string object.
+
There are two arguments: the first is a 1-byte unsigned int giving
the number of bytes in the string, and the second is that many bytes,
which are taken literally as the string content.
_dis_test = r"""
>>> import pickle
->>> x = [1, 2, (3, 4), {bytes(b'abc'): "def"}]
->>> pkl = pickle.dumps(x, 0)
->>> dis(pkl)
+>>> x = [1, 2, (3, 4), {b'abc': "def"}]
+>>> pkl0 = pickle.dumps(x, 0)
+>>> dis(pkl0)
0: ( MARK
1: l LIST (MARK at 0)
2: p PUT 0
25: ( MARK
26: d DICT (MARK at 25)
27: p PUT 2
- 30: S STRING 'abc'
- 37: p PUT 3
- 40: V UNICODE 'def'
- 45: p PUT 4
- 48: s SETITEM
- 49: a APPEND
- 50: . STOP
+ 30: c GLOBAL 'builtins bytes'
+ 46: p PUT 3
+ 49: ( MARK
+ 50: ( MARK
+ 51: l LIST (MARK at 50)
+ 52: p PUT 4
+ 55: L LONG 97
+ 59: a APPEND
+ 60: L LONG 98
+ 64: a APPEND
+ 65: L LONG 99
+ 69: a APPEND
+ 70: t TUPLE (MARK at 49)
+ 71: p PUT 5
+ 74: R REDUCE
+ 75: V UNICODE 'def'
+ 80: p PUT 6
+ 83: s SETITEM
+ 84: a APPEND
+ 85: . STOP
highest protocol among opcodes = 0
Try again with a "binary" pickle.
->>> pkl = pickle.dumps(x, 1)
->>> dis(pkl)
+>>> pkl1 = pickle.dumps(x, 1)
+>>> dis(pkl1)
0: ] EMPTY_LIST
1: q BINPUT 0
3: ( MARK
14: q BINPUT 1
16: } EMPTY_DICT
17: q BINPUT 2
- 19: U SHORT_BINSTRING 'abc'
- 24: q BINPUT 3
- 26: X BINUNICODE 'def'
- 34: q BINPUT 4
- 36: s SETITEM
- 37: e APPENDS (MARK at 3)
- 38: . STOP
+ 19: c GLOBAL 'builtins bytes'
+ 35: q BINPUT 3
+ 37: ( MARK
+ 38: ] EMPTY_LIST
+ 39: q BINPUT 4
+ 41: ( MARK
+ 42: K BININT1 97
+ 44: K BININT1 98
+ 46: K BININT1 99
+ 48: e APPENDS (MARK at 41)
+ 49: t TUPLE (MARK at 37)
+ 50: q BINPUT 5
+ 52: R REDUCE
+ 53: X BINUNICODE 'def'
+ 61: q BINPUT 6
+ 63: s SETITEM
+ 64: e APPENDS (MARK at 3)
+ 65: . STOP
highest protocol among opcodes = 1
Exercise the INST/OBJ/BUILD family.
u2 = self.loads(p)
self.assertEqual(u2, u)
+ def test_bytes(self):
+ for proto in protocols:
+ for u in b'', b'xyz', b'xyz'*100:
+ p = self.dumps(u)
+ self.assertEqual(self.loads(p), u)
+
def test_ints(self):
import sys
for proto in protocols:
@run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
def test_float_format(self):
- # make sure that floats are formatted locale independent
- self.assertEqual(self.dumps(1.2)[0:3], b'F1.')
+ # make sure that floats are formatted locale independent with proto 0
+ self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.')
def test_reduce(self):
pass
(2, 2): pickle.TUPLE2,
(2, 3): pickle.TUPLE3,
(2, 4): pickle.TUPLE,
+
+ (3, 0): pickle.EMPTY_TUPLE,
+ (3, 1): pickle.TUPLE1,
+ (3, 2): pickle.TUPLE2,
+ (3, 3): pickle.TUPLE3,
+ (3, 4): pickle.TUPLE,
}
a = ()
b = (1,)
expected_opcode = {(0, None): pickle.NONE,
(1, None): pickle.NONE,
(2, None): pickle.NONE,
+ (3, None): pickle.NONE,
(0, True): pickle.INT,
(1, True): pickle.INT,
(2, True): pickle.NEWTRUE,
+ (3, True): pickle.NEWTRUE,
(0, False): pickle.INT,
(1, False): pickle.INT,
(2, False): pickle.NEWFALSE,
+ (3, False): pickle.NEWFALSE,
}
for proto in protocols:
for x in None, False, True:
def test_highest_protocol(self):
# Of course this needs to be changed when HIGHEST_PROTOCOL changes.
- self.assertEqual(self.module.HIGHEST_PROTOCOL, 2)
+ self.assertEqual(self.module.HIGHEST_PROTOCOL, 3)
def test_callapi(self):
from io import BytesIO
module = pickle
error = KeyError
- def dumps(self, arg, proto=0, fast=0):
- # Ignore fast
+ def dumps(self, arg, proto=None):
return pickle.dumps(arg, proto)
def loads(self, buf):
- # Ignore fast
return pickle.loads(buf)
class PicklerTests(AbstractPickleTests):
error = KeyError
- def dumps(self, arg, proto=0, fast=0):
+ def dumps(self, arg, proto=None):
f = io.BytesIO()
p = pickle.Pickler(f, proto)
- if fast:
- p.fast = fast
p.dump(arg)
f.seek(0)
return bytes(f.read())
class PersPicklerTests(AbstractPersistentPicklerTests):
- def dumps(self, arg, proto=0, fast=0):
+ def dumps(self, arg, proto=None):
class PersPickler(pickle.Pickler):
def persistent_id(subself, obj):
return self.persistent_id(obj)
f = io.BytesIO()
p = PersPickler(f, proto)
- if fast:
- p.fast = fast
p.dump(arg)
f.seek(0)
return f.read()
class OptimizedPickleTests(AbstractPickleTests, AbstractPickleModuleTests):
- def dumps(self, arg, proto=0, fast=0):
+ def dumps(self, arg, proto=None):
return pickletools.optimize(pickle.dumps(arg, proto))
def loads(self, buf):
Library
-------
+- A new pickle protocol (protocol 3) is added with explicit support
+ for bytes. This is the default protocol. It intentionally cannot
+ be unpickled by Python 2.x.
+
+- When a pickle written by Python 2.x contains an (8-bit) str
+ instance, this is now decoded to a (Unicode) str instance. The
+ encoding used to do this defaults to ASCII, but can be overridden
+ via two new keyword arguments to the Unpickler class. Previously
+ this would create bytes instances, which is usually wrong: str
+ instances are often used to pickle attribute names etc., and text is
+ more common than binary data anyway.
+
- Default to ASCII as the locale.getpreferredencoding, if the POSIX
system doesn't support CODESET and LANG isn't set or doesn't
allow deduction of an encoding.
projects / python / commitdiff