From: Angus Gratton Date: Thu, 18 Aug 2016 04:38:51 +0000 (+0800) Subject: Fix gen_esp32part.py locations & documentation (thanks @wujiangang\!) X-Git-Tag: v0.9~101^2~5 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=9e0c80688bbf37d0b6e419990270ddb840005b3a;p=esp-idf Fix gen_esp32part.py locations & documentation (thanks @wujiangang\!) --- diff --git a/README.md b/README.md index 42b5ee7e83..87cc77c564 100644 --- a/README.md +++ b/README.md @@ -122,26 +122,26 @@ Sizes and offsets can be specified as decimal numbers, hex numbers with the pref ## Generating Binary Partition Table -The partition table which is flashed to the ESP32 is in a binary format, not CSV. The tool components/partition_table/gen_esp32part.py is used to convert between CSV and binary formats. +The partition table which is flashed to the ESP32 is in a binary format, not CSV. The tool bin/gen_esp32part.py is used to convert between CSV and binary formats. If you configure the partition table CSV name in `make menuconfig` and then `make partition_table`, this conversion is done for you. To convert CSV to Binary manually: ``` shell -python components/partition_table/gen_esp32part.py --verify input_partitions.csv binary_partitions.bin +python bin/gen_esp32part.py --verify input_partitions.csv binary_partitions.bin ``` To convert binary format back to CSV: ``` shell -python components/partition_table/gen_esp32part.py --verify binary_partitions.bin input_partitions.csv +python bin/gen_esp32part.py --verify binary_partitions.bin input_partitions.csv ``` To display the contents of a binary partition table on stdout (this is how the summaries displayed when running `make partition_table` are generated: ``` shell -python components/partition_table/gen_esp32part.py binary_partitions.bin +python bin/gen_esp32part.py binary_partitions.bin ``` `gen_esp32part.py` takes one optional argument, `--verify`, which will also verify the partition table during conversion (checking for overlapping partitions, unaligned partitions, etc.) diff --git a/tools/gen_esp32part.py b/tools/gen_esp32part.py deleted file mode 100755 index 89da746710..0000000000 --- a/tools/gen_esp32part.py +++ /dev/null @@ -1,343 +0,0 @@ -#!/usr/bin/env python -# -# ESP32 partition table generation tool -# -# Takes partition input from a CSV file of this format: -# -# Name,Type,SubType,Offset,Size -# -# Any row which starts with # is ignored as a comment. Whitespace at beginning/end of field is ignored. -# -# EXAMPLE CSV FILE CONTENTS -# ------------------------- -""" -# Name, Type, SubType, Offset, Size -factory, 0, 0, , 1M -ota_0, 0, 0x11, , 1M -ota_1, 0, 0x12, , 1M -ota_data, 1, 0, 5M, 128K -""" -# -# SAMPLE USAGE -# ------------ -# -# Convert CSV layout to binary: -# gen_esp32part.py table.csv table.bin -# -# Convert binary to CSV, print on stdout: -# gen_esp32part.py table.bin -# -# FIELD DETAILS -# ------------- -# -# Name: Human-readable name of partition. Values longer than 16 bytes will be truncated. -# -# Type: Numeric value for type field, or special keywords 'app' (for 0x00) or 'data' (for 0x01). -# -# SubType: Numeric value for subtype field, or special keywords depending on known types: -# app (0x00) : factory, ota_0 through ota_15, test. -# data (0x01) : ota, rf, wifi -# -# Default is zero if left blank. -# -# -# Offset: Start of partition. Examples: 0x800, 0x10000, 512, 2K, 1M -# - Offset can be left blank, and gen_esp32part will find the next valid offset (aligned to 64KB for app partitions.) -# -# Size: Size of partition. Examples: 0x800, 0x10000, 512, 2K, 1M -# -# SubType: 0 -# -# Offset: End of previous partition, aligned to 64kB if Type = 0 (app) & 4 bytes otherwise. -# -# -# -import struct -import argparse -import sys - -__version__ = '1.0' - -def debug(msg): - """ Print to stderr """ - sys.stderr.write(msg) - sys.stderr.write('\n') - -class PartitionTable(list): - def __init__(self): - super(PartitionTable, self).__init__(self) - - @classmethod - def from_csv(cls, csv_contents): - res = PartitionTable() - lines = csv_contents.split("\n") - for line_no in range(len(lines)): - line = lines[line_no].strip() - if line.startswith("#") or len(line) == 0: - continue - try: - res.append(PartitionDefinition.from_csv(line)) - except InputError as e: - raise InputError("Error at line %d: %s" % (line_no+1, e)) - except Exception: - debug("Unexpected error parsing line %d: %s" % (line_no+1, line)) - raise - - # fix up missing offsets & negative sizes - last_end = 0x5000 # first offset after partition table - for e in res: - if e.offset is None: - pad_to = 0x10000 if e.type == PartitionDefinition.APP_TYPE else 4 - if last_end % pad_to != 0: - last_end += pad_to - (last_end % pad_to) - e.offset = last_end - if e.size < 0: - e.size = -e.size - e.offset - last_end = e.offset + e.size - - return res - - def __getitem__(self, item): - """ Allow partition table access via name as well as by - numeric index. """ - if isinstance(item, str): - for x in self: - if x.name == item: - return x - raise ValueError("No partition entry named '%s'" % item) - else: - return super(PartitionTable, self).__getitem__(item) - - def verify(self): - # verify each partition individually - for p in self: - p.verify() - # check for overlaps - last = None - for p in sorted(self): - if p.offset < 0x5000: - raise InputError("Partition offset 0x%x is below 0x5000" % p.offset) - if last is not None and p.offset < last.offset + last.size: - raise InputError("Partition at 0x%x overlaps 0x%x-0x%x" % (p.offset, last.offset, last.offset+last.size-1)) - last = p - - @classmethod - def from_binary(cls, b): - if len(b) % 32 != 0: - raise InputError("Partition table length must be a multiple of 32 bytes. Got %d bytes." % len(b)) - result = cls() - for o in range(0,len(b),32): - result.append(PartitionDefinition.from_binary(b[o:o+32])) - return result - - def to_binary(self): - return "".join(e.to_binary() for e in self) - - def to_csv(self, simple_formatting=False): - rows = [ "# Espressif ESP32 Partition Table", - "# Name, Type, SubType, Offset, Size" ] - rows += [ x.to_csv(simple_formatting) for x in self ] - return "\n".join(rows) - -class PartitionDefinition(object): - APP_TYPE = 0x00 - DATA_TYPE = 0x01 - TYPES = { - "app" : APP_TYPE, - "data" : DATA_TYPE, - } - - SUBTYPES = { - APP_TYPE : { - "factory" : 0x00, - "test" : 0x20, - }, - DATA_TYPE : { - "ota" : 0x00, - "rf" : 0x01, - "wifi" : 0x02, - }, - } - - MAGIC_BYTES = "\xAA\x50" - - ALIGNMENT = { - APP_TYPE : 0x1000, - DATA_TYPE : 0x04, - } - - # add subtypes for the 16 OTA slot values ("ota_XXX, etc.") - for ota_slot in range(16): - SUBTYPES[TYPES["app"]]["ota_%d" % ota_slot] = 0x10 + ota_slot - - def __init__(self): - self.name = "" - self.type = None - self.subtype = None - self.offset = None - self.size = None - - @classmethod - def from_csv(cls, line): - """ Parse a line from the CSV """ - line_w_defaults = line + ",,," # lazy way to support default fields - fields = [ f.strip() for f in line_w_defaults.split(",") ] - - res = PartitionDefinition() - res.name = fields[0] - res.type = res.parse_type(fields[1]) - res.subtype = res.parse_subtype(fields[2]) - res.offset = res.parse_address(fields[3]) - res.size = res.parse_address(fields[4]) - if res.size is None: - raise InputError("Size field can't be empty") - return res - - def __eq__(self, other): - return self.name == other.name and self.type == other.type \ - and self.subtype == other.subtype and self.offset == other.offset \ - and self.size == other.size - - def __repr__(self): - def maybe_hex(x): - return "0x%x" % x if x is not None else "None" - return "PartitionDefinition('%s', 0x%x, 0x%x, %s, %s)" % (self.name, self.type, self.subtype or 0, - maybe_hex(self.offset), maybe_hex(self.size)) - - def __str__(self): - return "Part '%s' %d/%d @ 0x%x size 0x%x" % (self.name, self.type, self.subtype, self.offset or -1, self.size or -1) - - def __cmp__(self, other): - return self.offset - other.offset - - def parse_type(self, strval): - if strval == "": - raise InputError("Field 'type' can't be left empty.") - return parse_int(strval, self.TYPES) - - def parse_subtype(self, strval): - if strval == "": - return 0 # default - return parse_int(strval, self.SUBTYPES.get(self.type, {})) - - def parse_address(self, strval): - if strval == "": - return None # PartitionTable will fill in default - return parse_int(strval) - - def verify(self): - if self.type is None: - raise ValidationError("Type field is not set") - if self.subtype is None: - raise ValidationError("Subtype field is not set") - if self.offset is None: - raise ValidationError("Offset field is not set") - align = self.ALIGNMENT.get(self.type, 4) - if self.offset % align: - raise ValidationError("%s offset 0x%x is not aligned to 0x%x" % (self.name, self.offset, align)) - if self.size is None: - raise ValidationError("Size field is not set") - - STRUCT_FORMAT = "<2sBBLL16sL" - - @classmethod - def from_binary(cls, b): - if len(b) != 32: - raise InputError("Partition definition length must be exactly 32 bytes. Got %d bytes." % len(b)) - res = cls() - (magic, res.type, res.subtype, res.offset, - res.size, res.name, reserved) = struct.unpack(cls.STRUCT_FORMAT, b) - if "\x00" in res.name: # strip null byte padding from name string - res.name = res.name[:res.name.index("\x00")] - if magic != cls.MAGIC_BYTES: - raise InputError("Invalid magic bytes (%r) for partition definition" % magic) - if reserved != 0: - debug("WARNING: Partition definition had unexpected reserved value 0x%08x. Newer binary format?" % reserved) - return res - - def to_binary(self): - return struct.pack(self.STRUCT_FORMAT, - self.MAGIC_BYTES, - self.type, self.subtype, - self.offset, self.size, - self.name, - 0) # reserved - - def to_csv(self, simple_formatting=False): - def addr_format(a): - if not simple_formatting: - for (val, suffix) in [ (0x100000, "M"), (0x400, "K") ]: - if a % val == 0: - return "%d%s" % (a / val, suffix) - return "0x%x" % a - - def lookup_keyword(t, keywords): - for k,v in keywords.items(): - if simple_formatting == False and t == v: - return k - return "%d" % t - - return ",".join([ self.name, - lookup_keyword(self.type, self.TYPES), - lookup_keyword(self.subtype, self.SUBTYPES), - addr_format(self.offset), - addr_format(self.size) ]) - -class InputError(RuntimeError): - def __init__(self, e): - super(InputError, self).__init__(e) - -def parse_int(v, keywords={}): - """Generic parser for integer fields - int(x,0) with provision for - k/m/K/M suffixes and 'keyword' value lookup. - """ - try: - for letter, multiplier in [ ("k",1024), ("m",1024*1024) ]: - if v.lower().endswith(letter): - return parse_int(v[:-1], keywords) * multiplier - return int(v, 0) - except ValueError: - if len(keywords) == 0: - raise InputError("Invalid field value %s" % v) - try: - return keywords[v.lower()] - except KeyError: - raise InputError("Value '%s' is not valid. Known keywords: %s" % (v, ", ".join(keywords))) - -def main(): - parser = argparse.ArgumentParser(description='ESP32 partition table utility') - - parser.add_argument('--verify', '-v', help='Verify partition table fields', default=True, action='store_false') - - parser.add_argument('input', help='Path to CSV or binary file to parse. Will use stdin if omitted.', type=argparse.FileType('r'), default=sys.stdin) - parser.add_argument('output', help='Path to output converted binary or CSV file. Will use stdout if omitted, unless the --display argument is also passed (in which case only the summary is printed.)', - type=argparse.FileType('w'), nargs='?', - default=sys.stdout) - - args = parser.parse_args() - - input = args.input.read() - input_is_binary = input[0:2] == PartitionDefinition.MAGIC_BYTES - if input_is_binary: - debug("Parsing binary partition input...") - table = PartitionTable.from_binary(input) - else: - debug("Parsing CSV input...") - table = PartitionTable.from_csv(input) - - if args.verify: - debug("Verifying table...") - table.verify() - - if input_is_binary: - output = table.to_csv() - else: - output = table.to_binary() - args.output.write(output) - -if __name__ == '__main__': - try: - main() - except InputError as e: - print(e) - sys.exit(2) diff --git a/tools/tests/gen_esp32part_tests.py b/tools/tests/gen_esp32part_tests.py deleted file mode 100755 index 413f1aac91..0000000000 --- a/tools/tests/gen_esp32part_tests.py +++ /dev/null @@ -1,313 +0,0 @@ -#!/usr/bin/env python -import unittest -import struct -import csv -import sys -import subprocess -import tempfile -import os -sys.path.append("..") -from gen_esp32part import * - -SIMPLE_CSV = """ -# Name,Type,SubType,Offset,Size -factory,0,2,65536,1048576 -""" - -LONGER_BINARY_TABLE = "" -# type 0x00, subtype 0x00, -# offset 64KB, size 1MB -LONGER_BINARY_TABLE += "\xAA\x50\x00\x00" + \ - "\x00\x00\x01\x00" + \ - "\x00\x00\x10\x00" + \ - "factory\0" + ("\0"*8) + \ - "\x00\x00\x00\x00" -# type 0x01, subtype 0x20, -# offset 0x110000, size 128KB -LONGER_BINARY_TABLE += "\xAA\x50\x01\x20" + \ - "\x00\x00\x11\x00" + \ - "\x00\x02\x00\x00" + \ - "data" + ("\0"*12) + \ - "\x00\x00\x00\x00" -# type 0x10, subtype 0x00, -# offset 0x150000, size 1MB -LONGER_BINARY_TABLE += "\xAA\x50\x10\x00" + \ - "\x00\x00\x15\x00" + \ - "\x00\x10\x00\x00" + \ - "second" + ("\0"*10) + \ - "\x00\x00\x00\x00" - - -class CSVParserTests(unittest.TestCase): - - def test_simple_partition(self): - table = PartitionTable.from_csv(SIMPLE_CSV) - self.assertEqual(len(table), 1) - self.assertEqual(table[0].name, "factory") - self.assertEqual(table[0].type, 0) - self.assertEqual(table[0].subtype, 2) - self.assertEqual(table[0].offset, 65536) - self.assertEqual(table[0].size, 1048576) - - - def test_require_type(self): - csv = """ -# Name,Type, SubType,Offset,Size -ihavenotype, -""" - with self.assertRaisesRegexp(InputError, "type"): - PartitionTable.from_csv(csv) - - - def test_type_subtype_names(self): - csv_magicnumbers = """ -# Name, Type, SubType, Offset, Size -myapp, 0, 0,, 0x100000 -myota_0, 0, 0x10,, 0x100000 -myota_1, 0, 0x11,, 0x100000 -myota_15, 0, 0x1f,, 0x100000 -mytest, 0, 0x20,, 0x100000 -myota_status, 1, 0,, 0x100000 - """ - csv_nomagicnumbers = """ -# Name, Type, SubType, Offset, Size -myapp, app, factory,, 0x100000 -myota_0, app, ota_0,, 0x100000 -myota_1, app, ota_1,, 0x100000 -myota_15, app, ota_15,, 0x100000 -mytest, app, test,, 0x100000 -myota_status, data, ota,, 0x100000 -""" - # make two equivalent partition tables, one using - # magic numbers and one using shortcuts. Ensure they match - magic = PartitionTable.from_csv(csv_magicnumbers) - magic.verify() - nomagic = PartitionTable.from_csv(csv_nomagicnumbers) - nomagic.verify() - - self.assertEqual(nomagic["myapp"].type, 0) - self.assertEqual(nomagic["myapp"].subtype, 0) - self.assertEqual(nomagic["myapp"], magic["myapp"]) - self.assertEqual(nomagic["myota_0"].type, 0) - self.assertEqual(nomagic["myota_0"].subtype, 0x10) - self.assertEqual(nomagic["myota_0"], magic["myota_0"]) - self.assertEqual(nomagic["myota_15"], magic["myota_15"]) - self.assertEqual(nomagic["mytest"], magic["mytest"]) - self.assertEqual(nomagic["myota_status"], magic["myota_status"]) - - #self.assertEqual(nomagic.to_binary(), magic.to_binary()) - - def test_unit_suffixes(self): - csv = """ -# Name, Type, Subtype, Offset, Size -one_megabyte, app, factory, 32k, 1M -""" - t = PartitionTable.from_csv(csv) - t.verify() - self.assertEqual(t[0].offset, 32*1024) - self.assertEqual(t[0].size, 1*1024*1024) - - def test_default_offsets(self): - csv = """ -# Name, Type, Subtype, Offset, Size -first, app, factory,, 1M -second, data, 0x15,, 1M -minidata, data, 0x40,, 32K -otherapp, app, factory,, 1M - """ - t = PartitionTable.from_csv(csv) - # 'first' - self.assertEqual(t[0].offset, 0x010000) # 64KB boundary as it's an app image - self.assertEqual(t[0].size, 0x100000) # Size specified in CSV - # 'second' - self.assertEqual(t[1].offset, 0x110000) # prev offset+size - self.assertEqual(t[1].size, 0x100000) # Size specified in CSV - # 'minidata' - self.assertEqual(t[2].offset, 0x210000) - # 'otherapp' - self.assertEqual(t[3].offset, 0x220000) # 64KB boundary as it's an app image - - def test_negative_size_to_offset(self): - csv = """ -# Name, Type, Subtype, Offset, Size -first, app, factory, 0x10000, -2M -second, data, 0x15, , 1M - """ - t = PartitionTable.from_csv(csv) - t.verify() - # 'first' - self.assertEqual(t[0].offset, 0x10000) # in CSV - self.assertEqual(t[0].size, 0x200000 - t[0].offset) # Up to 2M - # 'second' - self.assertEqual(t[1].offset, 0x200000) # prev offset+size - - def test_overlapping_offsets_fail(self): - csv = """ -first, app, factory, 0x100000, 2M -second, app, ota_0, 0x200000, 1M -""" - t = PartitionTable.from_csv(csv) - with self.assertRaisesRegexp(InputError, "overlap"): - t.verify() - -class BinaryOutputTests(unittest.TestCase): - def test_binary_entry(self): - csv = """ -first, 0x30, 0xEE, 0x100400, 0x300000 -""" - t = PartitionTable.from_csv(csv) - tb = t.to_binary() - self.assertEqual(len(tb), 32) - self.assertEqual('\xAA\x50', tb[0:2]) # magic - self.assertEqual('\x30\xee', tb[2:4]) # type, subtype - eo, es = struct.unpack("