+++ /dev/null
-#!/usr/bin/env python
-#
-# ESP32 partition table generation tool
-#
-# Takes partition input from a CSV file of this format:
-#
-# Name,Type,SubType,Offset,Size
-#
-# Any row which starts with # is ignored as a comment. Whitespace at beginning/end of field is ignored.
-#
-# EXAMPLE CSV FILE CONTENTS
-# -------------------------
-"""
-# Name, Type, SubType, Offset, Size
-factory, 0, 0, , 1M
-ota_0, 0, 0x11, , 1M
-ota_1, 0, 0x12, , 1M
-ota_data, 1, 0, 5M, 128K
-"""
-#
-# SAMPLE USAGE
-# ------------
-#
-# Convert CSV layout to binary:
-# gen_esp32part.py table.csv table.bin
-#
-# Convert binary to CSV, print on stdout:
-# gen_esp32part.py table.bin
-#
-# FIELD DETAILS
-# -------------
-#
-# Name: Human-readable name of partition. Values longer than 16 bytes will be truncated.
-#
-# Type: Numeric value for type field, or special keywords 'app' (for 0x00) or 'data' (for 0x01).
-#
-# SubType: Numeric value for subtype field, or special keywords depending on known types:
-# app (0x00) : factory, ota_0 through ota_15, test.
-# data (0x01) : ota, rf, wifi
-#
-# Default is zero if left blank.
-#
-#
-# Offset: Start of partition. Examples: 0x800, 0x10000, 512, 2K, 1M
-# - Offset can be left blank, and gen_esp32part will find the next valid offset (aligned to 64KB for app partitions.)
-#
-# Size: Size of partition. Examples: 0x800, 0x10000, 512, 2K, 1M
-#
-# SubType: 0
-#
-# Offset: End of previous partition, aligned to 64kB if Type = 0 (app) & 4 bytes otherwise.
-#
-#
-#
-import struct
-import argparse
-import sys
-
-__version__ = '1.0'
-
-def debug(msg):
- """ Print to stderr """
- sys.stderr.write(msg)
- sys.stderr.write('\n')
-
-class PartitionTable(list):
- def __init__(self):
- super(PartitionTable, self).__init__(self)
-
- @classmethod
- def from_csv(cls, csv_contents):
- res = PartitionTable()
- lines = csv_contents.split("\n")
- for line_no in range(len(lines)):
- line = lines[line_no].strip()
- if line.startswith("#") or len(line) == 0:
- continue
- try:
- res.append(PartitionDefinition.from_csv(line))
- except InputError as e:
- raise InputError("Error at line %d: %s" % (line_no+1, e))
- except Exception:
- debug("Unexpected error parsing line %d: %s" % (line_no+1, line))
- raise
-
- # fix up missing offsets & negative sizes
- last_end = 0x5000 # first offset after partition table
- for e in res:
- if e.offset is None:
- pad_to = 0x10000 if e.type == PartitionDefinition.APP_TYPE else 4
- if last_end % pad_to != 0:
- last_end += pad_to - (last_end % pad_to)
- e.offset = last_end
- if e.size < 0:
- e.size = -e.size - e.offset
- last_end = e.offset + e.size
-
- return res
-
- def __getitem__(self, item):
- """ Allow partition table access via name as well as by
- numeric index. """
- if isinstance(item, str):
- for x in self:
- if x.name == item:
- return x
- raise ValueError("No partition entry named '%s'" % item)
- else:
- return super(PartitionTable, self).__getitem__(item)
-
- def verify(self):
- # verify each partition individually
- for p in self:
- p.verify()
- # check for overlaps
- last = None
- for p in sorted(self):
- if p.offset < 0x5000:
- raise InputError("Partition offset 0x%x is below 0x5000" % p.offset)
- if last is not None and p.offset < last.offset + last.size:
- raise InputError("Partition at 0x%x overlaps 0x%x-0x%x" % (p.offset, last.offset, last.offset+last.size-1))
- last = p
-
- @classmethod
- def from_binary(cls, b):
- if len(b) % 32 != 0:
- raise InputError("Partition table length must be a multiple of 32 bytes. Got %d bytes." % len(b))
- result = cls()
- for o in range(0,len(b),32):
- result.append(PartitionDefinition.from_binary(b[o:o+32]))
- return result
-
- def to_binary(self):
- return "".join(e.to_binary() for e in self)
-
- def to_csv(self, simple_formatting=False):
- rows = [ "# Espressif ESP32 Partition Table",
- "# Name, Type, SubType, Offset, Size" ]
- rows += [ x.to_csv(simple_formatting) for x in self ]
- return "\n".join(rows)
-
-class PartitionDefinition(object):
- APP_TYPE = 0x00
- DATA_TYPE = 0x01
- TYPES = {
- "app" : APP_TYPE,
- "data" : DATA_TYPE,
- }
-
- SUBTYPES = {
- APP_TYPE : {
- "factory" : 0x00,
- "test" : 0x20,
- },
- DATA_TYPE : {
- "ota" : 0x00,
- "rf" : 0x01,
- "wifi" : 0x02,
- },
- }
-
- MAGIC_BYTES = "\xAA\x50"
-
- ALIGNMENT = {
- APP_TYPE : 0x1000,
- DATA_TYPE : 0x04,
- }
-
- # add subtypes for the 16 OTA slot values ("ota_XXX, etc.")
- for ota_slot in range(16):
- SUBTYPES[TYPES["app"]]["ota_%d" % ota_slot] = 0x10 + ota_slot
-
- def __init__(self):
- self.name = ""
- self.type = None
- self.subtype = None
- self.offset = None
- self.size = None
-
- @classmethod
- def from_csv(cls, line):
- """ Parse a line from the CSV """
- line_w_defaults = line + ",,," # lazy way to support default fields
- fields = [ f.strip() for f in line_w_defaults.split(",") ]
-
- res = PartitionDefinition()
- res.name = fields[0]
- res.type = res.parse_type(fields[1])
- res.subtype = res.parse_subtype(fields[2])
- res.offset = res.parse_address(fields[3])
- res.size = res.parse_address(fields[4])
- if res.size is None:
- raise InputError("Size field can't be empty")
- return res
-
- def __eq__(self, other):
- return self.name == other.name and self.type == other.type \
- and self.subtype == other.subtype and self.offset == other.offset \
- and self.size == other.size
-
- def __repr__(self):
- def maybe_hex(x):
- return "0x%x" % x if x is not None else "None"
- return "PartitionDefinition('%s', 0x%x, 0x%x, %s, %s)" % (self.name, self.type, self.subtype or 0,
- maybe_hex(self.offset), maybe_hex(self.size))
-
- def __str__(self):
- return "Part '%s' %d/%d @ 0x%x size 0x%x" % (self.name, self.type, self.subtype, self.offset or -1, self.size or -1)
-
- def __cmp__(self, other):
- return self.offset - other.offset
-
- def parse_type(self, strval):
- if strval == "":
- raise InputError("Field 'type' can't be left empty.")
- return parse_int(strval, self.TYPES)
-
- def parse_subtype(self, strval):
- if strval == "":
- return 0 # default
- return parse_int(strval, self.SUBTYPES.get(self.type, {}))
-
- def parse_address(self, strval):
- if strval == "":
- return None # PartitionTable will fill in default
- return parse_int(strval)
-
- def verify(self):
- if self.type is None:
- raise ValidationError("Type field is not set")
- if self.subtype is None:
- raise ValidationError("Subtype field is not set")
- if self.offset is None:
- raise ValidationError("Offset field is not set")
- align = self.ALIGNMENT.get(self.type, 4)
- if self.offset % align:
- raise ValidationError("%s offset 0x%x is not aligned to 0x%x" % (self.name, self.offset, align))
- if self.size is None:
- raise ValidationError("Size field is not set")
-
- STRUCT_FORMAT = "<2sBBLL16sL"
-
- @classmethod
- def from_binary(cls, b):
- if len(b) != 32:
- raise InputError("Partition definition length must be exactly 32 bytes. Got %d bytes." % len(b))
- res = cls()
- (magic, res.type, res.subtype, res.offset,
- res.size, res.name, reserved) = struct.unpack(cls.STRUCT_FORMAT, b)
- if "\x00" in res.name: # strip null byte padding from name string
- res.name = res.name[:res.name.index("\x00")]
- if magic != cls.MAGIC_BYTES:
- raise InputError("Invalid magic bytes (%r) for partition definition" % magic)
- if reserved != 0:
- debug("WARNING: Partition definition had unexpected reserved value 0x%08x. Newer binary format?" % reserved)
- return res
-
- def to_binary(self):
- return struct.pack(self.STRUCT_FORMAT,
- self.MAGIC_BYTES,
- self.type, self.subtype,
- self.offset, self.size,
- self.name,
- 0) # reserved
-
- def to_csv(self, simple_formatting=False):
- def addr_format(a):
- if not simple_formatting:
- for (val, suffix) in [ (0x100000, "M"), (0x400, "K") ]:
- if a % val == 0:
- return "%d%s" % (a / val, suffix)
- return "0x%x" % a
-
- def lookup_keyword(t, keywords):
- for k,v in keywords.items():
- if simple_formatting == False and t == v:
- return k
- return "%d" % t
-
- return ",".join([ self.name,
- lookup_keyword(self.type, self.TYPES),
- lookup_keyword(self.subtype, self.SUBTYPES),
- addr_format(self.offset),
- addr_format(self.size) ])
-
-class InputError(RuntimeError):
- def __init__(self, e):
- super(InputError, self).__init__(e)
-
-def parse_int(v, keywords={}):
- """Generic parser for integer fields - int(x,0) with provision for
- k/m/K/M suffixes and 'keyword' value lookup.
- """
- try:
- for letter, multiplier in [ ("k",1024), ("m",1024*1024) ]:
- if v.lower().endswith(letter):
- return parse_int(v[:-1], keywords) * multiplier
- return int(v, 0)
- except ValueError:
- if len(keywords) == 0:
- raise InputError("Invalid field value %s" % v)
- try:
- return keywords[v.lower()]
- except KeyError:
- raise InputError("Value '%s' is not valid. Known keywords: %s" % (v, ", ".join(keywords)))
-
-def main():
- parser = argparse.ArgumentParser(description='ESP32 partition table utility')
-
- parser.add_argument('--verify', '-v', help='Verify partition table fields', default=True, action='store_false')
-
- parser.add_argument('input', help='Path to CSV or binary file to parse. Will use stdin if omitted.', type=argparse.FileType('r'), default=sys.stdin)
- parser.add_argument('output', help='Path to output converted binary or CSV file. Will use stdout if omitted, unless the --display argument is also passed (in which case only the summary is printed.)',
- type=argparse.FileType('w'), nargs='?',
- default=sys.stdout)
-
- args = parser.parse_args()
-
- input = args.input.read()
- input_is_binary = input[0:2] == PartitionDefinition.MAGIC_BYTES
- if input_is_binary:
- debug("Parsing binary partition input...")
- table = PartitionTable.from_binary(input)
- else:
- debug("Parsing CSV input...")
- table = PartitionTable.from_csv(input)
-
- if args.verify:
- debug("Verifying table...")
- table.verify()
-
- if input_is_binary:
- output = table.to_csv()
- else:
- output = table.to_binary()
- args.output.write(output)
-
-if __name__ == '__main__':
- try:
- main()
- except InputError as e:
- print(e)
- sys.exit(2)
+++ /dev/null
-#!/usr/bin/env python
-import unittest
-import struct
-import csv
-import sys
-import subprocess
-import tempfile
-import os
-sys.path.append("..")
-from gen_esp32part import *
-
-SIMPLE_CSV = """
-# Name,Type,SubType,Offset,Size
-factory,0,2,65536,1048576
-"""
-
-LONGER_BINARY_TABLE = ""
-# type 0x00, subtype 0x00,
-# offset 64KB, size 1MB
-LONGER_BINARY_TABLE += "\xAA\x50\x00\x00" + \
- "\x00\x00\x01\x00" + \
- "\x00\x00\x10\x00" + \
- "factory\0" + ("\0"*8) + \
- "\x00\x00\x00\x00"
-# type 0x01, subtype 0x20,
-# offset 0x110000, size 128KB
-LONGER_BINARY_TABLE += "\xAA\x50\x01\x20" + \
- "\x00\x00\x11\x00" + \
- "\x00\x02\x00\x00" + \
- "data" + ("\0"*12) + \
- "\x00\x00\x00\x00"
-# type 0x10, subtype 0x00,
-# offset 0x150000, size 1MB
-LONGER_BINARY_TABLE += "\xAA\x50\x10\x00" + \
- "\x00\x00\x15\x00" + \
- "\x00\x10\x00\x00" + \
- "second" + ("\0"*10) + \
- "\x00\x00\x00\x00"
-
-
-class CSVParserTests(unittest.TestCase):
-
- def test_simple_partition(self):
- table = PartitionTable.from_csv(SIMPLE_CSV)
- self.assertEqual(len(table), 1)
- self.assertEqual(table[0].name, "factory")
- self.assertEqual(table[0].type, 0)
- self.assertEqual(table[0].subtype, 2)
- self.assertEqual(table[0].offset, 65536)
- self.assertEqual(table[0].size, 1048576)
-
-
- def test_require_type(self):
- csv = """
-# Name,Type, SubType,Offset,Size
-ihavenotype,
-"""
- with self.assertRaisesRegexp(InputError, "type"):
- PartitionTable.from_csv(csv)
-
-
- def test_type_subtype_names(self):
- csv_magicnumbers = """
-# Name, Type, SubType, Offset, Size
-myapp, 0, 0,, 0x100000
-myota_0, 0, 0x10,, 0x100000
-myota_1, 0, 0x11,, 0x100000
-myota_15, 0, 0x1f,, 0x100000
-mytest, 0, 0x20,, 0x100000
-myota_status, 1, 0,, 0x100000
- """
- csv_nomagicnumbers = """
-# Name, Type, SubType, Offset, Size
-myapp, app, factory,, 0x100000
-myota_0, app, ota_0,, 0x100000
-myota_1, app, ota_1,, 0x100000
-myota_15, app, ota_15,, 0x100000
-mytest, app, test,, 0x100000
-myota_status, data, ota,, 0x100000
-"""
- # make two equivalent partition tables, one using
- # magic numbers and one using shortcuts. Ensure they match
- magic = PartitionTable.from_csv(csv_magicnumbers)
- magic.verify()
- nomagic = PartitionTable.from_csv(csv_nomagicnumbers)
- nomagic.verify()
-
- self.assertEqual(nomagic["myapp"].type, 0)
- self.assertEqual(nomagic["myapp"].subtype, 0)
- self.assertEqual(nomagic["myapp"], magic["myapp"])
- self.assertEqual(nomagic["myota_0"].type, 0)
- self.assertEqual(nomagic["myota_0"].subtype, 0x10)
- self.assertEqual(nomagic["myota_0"], magic["myota_0"])
- self.assertEqual(nomagic["myota_15"], magic["myota_15"])
- self.assertEqual(nomagic["mytest"], magic["mytest"])
- self.assertEqual(nomagic["myota_status"], magic["myota_status"])
-
- #self.assertEqual(nomagic.to_binary(), magic.to_binary())
-
- def test_unit_suffixes(self):
- csv = """
-# Name, Type, Subtype, Offset, Size
-one_megabyte, app, factory, 32k, 1M
-"""
- t = PartitionTable.from_csv(csv)
- t.verify()
- self.assertEqual(t[0].offset, 32*1024)
- self.assertEqual(t[0].size, 1*1024*1024)
-
- def test_default_offsets(self):
- csv = """
-# Name, Type, Subtype, Offset, Size
-first, app, factory,, 1M
-second, data, 0x15,, 1M
-minidata, data, 0x40,, 32K
-otherapp, app, factory,, 1M
- """
- t = PartitionTable.from_csv(csv)
- # 'first'
- self.assertEqual(t[0].offset, 0x010000) # 64KB boundary as it's an app image
- self.assertEqual(t[0].size, 0x100000) # Size specified in CSV
- # 'second'
- self.assertEqual(t[1].offset, 0x110000) # prev offset+size
- self.assertEqual(t[1].size, 0x100000) # Size specified in CSV
- # 'minidata'
- self.assertEqual(t[2].offset, 0x210000)
- # 'otherapp'
- self.assertEqual(t[3].offset, 0x220000) # 64KB boundary as it's an app image
-
- def test_negative_size_to_offset(self):
- csv = """
-# Name, Type, Subtype, Offset, Size
-first, app, factory, 0x10000, -2M
-second, data, 0x15, , 1M
- """
- t = PartitionTable.from_csv(csv)
- t.verify()
- # 'first'
- self.assertEqual(t[0].offset, 0x10000) # in CSV
- self.assertEqual(t[0].size, 0x200000 - t[0].offset) # Up to 2M
- # 'second'
- self.assertEqual(t[1].offset, 0x200000) # prev offset+size
-
- def test_overlapping_offsets_fail(self):
- csv = """
-first, app, factory, 0x100000, 2M
-second, app, ota_0, 0x200000, 1M
-"""
- t = PartitionTable.from_csv(csv)
- with self.assertRaisesRegexp(InputError, "overlap"):
- t.verify()
-
-class BinaryOutputTests(unittest.TestCase):
- def test_binary_entry(self):
- csv = """
-first, 0x30, 0xEE, 0x100400, 0x300000
-"""
- t = PartitionTable.from_csv(csv)
- tb = t.to_binary()
- self.assertEqual(len(tb), 32)
- self.assertEqual('\xAA\x50', tb[0:2]) # magic
- self.assertEqual('\x30\xee', tb[2:4]) # type, subtype
- eo, es = struct.unpack("<LL", tb[4:12])
- self.assertEqual(eo, 0x100400) # offset
- self.assertEqual(es, 0x300000) # size
-
- def test_multiple_entries(self):
- csv = """
-first, 0x30, 0xEE, 0x100400, 0x300000
-second,0x31, 0xEF, , 0x100000
-"""
- t = PartitionTable.from_csv(csv)
- tb = t.to_binary()
- self.assertEqual(len(tb), 64)
- self.assertEqual('\xAA\x50', tb[0:2])
- self.assertEqual('\xAA\x50', tb[32:34])
-
-
-class BinaryParserTests(unittest.TestCase):
- def test_parse_one_entry(self):
- # type 0x30, subtype 0xee,
- # offset 1MB, size 2MB
- entry = "\xAA\x50\x30\xee" + \
- "\x00\x00\x10\x00" + \
- "\x00\x00\x20\x00" + \
- "0123456789abc\0\0\0" + \
- "\x00\x00\x00\x00"
- # verify that parsing 32 bytes as a table
- # or as a single Definition are the same thing
- t = PartitionTable.from_binary(entry)
- self.assertEqual(len(t), 1)
- t[0].verify()
-
- e = PartitionDefinition.from_binary(entry)
- self.assertEqual(t[0], e)
- e.verify()
-
- self.assertEqual(e.type, 0x30)
- self.assertEqual(e.subtype, 0xEE)
- self.assertEqual(e.offset, 0x100000)
- self.assertEqual(e.size, 0x200000)
- self.assertEqual(e.name, "0123456789abc")
-
- def test_multiple_entries(self):
- t = PartitionTable.from_binary(LONGER_BINARY_TABLE)
- t.verify()
-
- self.assertEqual(3, len(t))
- self.assertEqual(t[0].type, PartitionDefinition.APP_TYPE)
- self.assertEqual(t[0].name, "factory")
-
- self.assertEqual(t[1].type, PartitionDefinition.DATA_TYPE)
- self.assertEqual(t[1].name, "data")
-
- self.assertEqual(t[2].type, 0x10)
- self.assertEqual(t[2].name, "second")
-
- round_trip = t.to_binary()
- self.assertEqual(round_trip, LONGER_BINARY_TABLE)
-
- def test_bad_magic(self):
- bad_magic = "OHAI" + \
- "\x00\x00\x10\x00" + \
- "\x00\x00\x20\x00" + \
- "0123456789abc\0\0\0" + \
- "\x00\x00\x00\x00"
- with self.assertRaisesRegexp(InputError, "Invalid magic bytes"):
- PartitionTable.from_binary(bad_magic)
-
- def test_bad_length(self):
- bad_length = "OHAI" + \
- "\x00\x00\x10\x00" + \
- "\x00\x00\x20\x00" + \
- "0123456789"
- with self.assertRaisesRegexp(InputError, "32 bytes"):
- PartitionTable.from_binary(bad_length)
-
-
-class CSVOutputTests(unittest.TestCase):
-
- def test_output_simple_formatting(self):
- table = PartitionTable.from_csv(SIMPLE_CSV)
- as_csv = table.to_csv(True)
- c = csv.reader(as_csv.split("\n"))
- # first two lines should start with comments
- self.assertEqual(c.next()[0][0], "#")
- self.assertEqual(c.next()[0][0], "#")
- row = c.next()
- self.assertEqual(row[0], "factory")
- self.assertEqual(row[1], "0")
- self.assertEqual(row[2], "2")
- self.assertEqual(row[3], "0x10000") # reformatted as hex
- self.assertEqual(row[4], "0x100000") # also hex
-
- # round trip back to a PartitionTable and check is identical
- roundtrip = PartitionTable.from_csv(as_csv)
- self.assertEqual(roundtrip, table)
-
- def test_output_smart_formatting(self):
- table = PartitionTable.from_csv(SIMPLE_CSV)
- as_csv = table.to_csv(False)
- c = csv.reader(as_csv.split("\n"))
- # first two lines should start with comments
- self.assertEqual(c.next()[0][0], "#")
- self.assertEqual(c.next()[0][0], "#")
- row = c.next()
- self.assertEqual(row[0], "factory")
- self.assertEqual(row[1], "app")
- self.assertEqual(row[2], "2")
- self.assertEqual(row[3], "64K")
- self.assertEqual(row[4], "1M")
-
- # round trip back to a PartitionTable and check is identical
- roundtrip = PartitionTable.from_csv(as_csv)
- self.assertEqual(roundtrip, table)
-
-class CommandLineTests(unittest.TestCase):
-
- def test_basic_cmdline(self):
- try:
- binpath = tempfile.mktemp()
- csvpath = tempfile.mktemp()
-
- # copy binary contents to temp file
- with open(binpath, 'w') as f:
- f.write(LONGER_BINARY_TABLE)
-
- # run gen_esp32part.py to convert binary file to CSV
- subprocess.check_call([sys.executable, "../gen_esp32part.py",
- binpath, csvpath])
- # reopen the CSV and check the generated binary is identical
- with open(csvpath, 'r') as f:
- from_csv = PartitionTable.from_csv(f.read())
- self.assertEqual(from_csv.to_binary(), LONGER_BINARY_TABLE)
-
- # run gen_esp32part.py to conver the CSV to binary again
- subprocess.check_call([sys.executable, "../gen_esp32part.py",
- csvpath, binpath])
- # assert that file reads back as identical
- with open(binpath, 'rb') as f:
- binary_readback = f.read()
- self.assertEqual(binary_readback, LONGER_BINARY_TABLE)
-
- finally:
- for path in binpath, csvpath:
- try:
- os.remove(path)
- except OSError:
- pass
-
-
-if __name__ =="__main__":
- unittest.main()
projects / esp-idf / commitdiff