From 943bf78a48ac22ce0277697976fef4f89457b446 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 23 Nov 2021 21:09:50 -0700 Subject: [PATCH] binman: Allow extracting a file in an alternative format In some cases entries encapsulate other data and it is useful to access the data within. An example is the fdtmap which consists of a 16-byte header, followed by a devicetree. Provide an option to specify an alternative format when extracting files. In the case of fdtmap, this is 'fdt', which produces an FDT file which can be viewed with fdtdump. Signed-off-by: Simon Glass --- tools/binman/binman.rst | 29 ++++++++++++++++ tools/binman/cmdline.py | 2 ++ tools/binman/control.py | 28 +++++++++++++--- tools/binman/entries.rst | 11 +++++- tools/binman/entry.py | 37 +++++++++++++++++---- tools/binman/etype/cbfs.py | 8 ++--- tools/binman/etype/fdtmap.py | 12 +++++++ tools/binman/etype/section.py | 23 ++++++++++--- tools/binman/ftest.py | 34 +++++++++++++++++++ tools/binman/image.py | 2 +- tools/binman/test/213_fdtmap_alt_format.dts | 15 +++++++++ tools/binman/test/214_no_alt_format.dts | 13 ++++++++ 12 files changed, 193 insertions(+), 21 deletions(-) create mode 100644 tools/binman/test/213_fdtmap_alt_format.dts create mode 100644 tools/binman/test/214_no_alt_format.dts diff --git a/tools/binman/binman.rst b/tools/binman/binman.rst index 26f462ae16..10389a52c4 100644 --- a/tools/binman/binman.rst +++ b/tools/binman/binman.rst @@ -942,6 +942,35 @@ or just a selection:: $ binman extract -i image.bin "*u-boot*" -O outdir +Some entry types have alternative formats, for example fdtmap which allows +extracted just the devicetree binary without the fdtmap header:: + + $ binman extract -i /tmp/b/odroid-c4/image.bin -f out.dtb -F fdt fdtmap + $ fdtdump out.dtb + /dts-v1/; + // magic: 0xd00dfeed + // totalsize: 0x8ab (2219) + // off_dt_struct: 0x38 + // off_dt_strings: 0x82c + // off_mem_rsvmap: 0x28 + // version: 17 + // last_comp_version: 2 + // boot_cpuid_phys: 0x0 + // size_dt_strings: 0x7f + // size_dt_struct: 0x7f4 + + / { + image-node = "binman"; + image-pos = <0x00000000>; + size = <0x0011162b>; + ... + +Use `-F list` to see what alternative formats are available:: + + $ binman extract -i /tmp/b/odroid-c4/image.bin -F list + Flag (-F) Entry type Description + fdt fdtmap Extract the devicetree blob from the fdtmap + Replacing files in an image --------------------------- diff --git a/tools/binman/cmdline.py b/tools/binman/cmdline.py index 2229316f10..adc17547ae 100644 --- a/tools/binman/cmdline.py +++ b/tools/binman/cmdline.py @@ -17,6 +17,8 @@ def make_extract_parser(subparsers): """ extract_parser = subparsers.add_parser('extract', help='Extract files from an image') + extract_parser.add_argument('-F', '--format', type=str, + help='Select an alternative format for extracted data') extract_parser.add_argument('-i', '--image', type=str, required=True, help='Image filename to extract') extract_parser.add_argument('-f', '--filename', type=str, diff --git a/tools/binman/control.py b/tools/binman/control.py index 7da69ba38d..dcf070da85 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -200,8 +200,24 @@ def ReadEntry(image_fname, entry_path, decomp=True): return entry.ReadData(decomp) +def ShowAltFormats(image): + """Show alternative formats available for entries in the image + + This shows a list of formats available. + + Args: + image (Image): Image to check + """ + alt_formats = {} + image.CheckAltFormats(alt_formats) + print('%-10s %-20s %s' % ('Flag (-F)', 'Entry type', 'Description')) + for name, val in alt_formats.items(): + entry, helptext = val + print('%-10s %-20s %s' % (name, entry.etype, helptext)) + + def ExtractEntries(image_fname, output_fname, outdir, entry_paths, - decomp=True): + decomp=True, alt_format=None): """Extract the data from one or more entries and write it to files Args: @@ -217,6 +233,10 @@ def ExtractEntries(image_fname, output_fname, outdir, entry_paths, """ image = Image.FromFile(image_fname) + if alt_format == 'list': + ShowAltFormats(image) + return + # Output an entry to a single file, as a special case if output_fname: if not entry_paths: @@ -224,7 +244,7 @@ def ExtractEntries(image_fname, output_fname, outdir, entry_paths, if len(entry_paths) != 1: raise ValueError('Must specify exactly one entry path to write with -f') entry = image.FindEntryPath(entry_paths[0]) - data = entry.ReadData(decomp) + data = entry.ReadData(decomp, alt_format) tools.WriteFile(output_fname, data) tout.Notice("Wrote %#x bytes to file '%s'" % (len(data), output_fname)) return @@ -236,7 +256,7 @@ def ExtractEntries(image_fname, output_fname, outdir, entry_paths, tout.Notice('%d entries match and will be written' % len(einfos)) for einfo in einfos: entry = einfo.entry - data = entry.ReadData(decomp) + data = entry.ReadData(decomp, alt_format) path = entry.GetPath()[1:] fname = os.path.join(outdir, path) @@ -584,7 +604,7 @@ def Binman(args): if args.cmd == 'extract': ExtractEntries(args.image, args.filename, args.outdir, args.paths, - not args.uncompressed) + not args.uncompressed, args.format) if args.cmd == 'replace': ReplaceEntries(args.image, args.filename, args.indir, args.paths, diff --git a/tools/binman/entries.rst b/tools/binman/entries.rst index 748277c1cd..2ebac517ce 100644 --- a/tools/binman/entries.rst +++ b/tools/binman/entries.rst @@ -314,6 +314,10 @@ Example output for a simple image with U-Boot and an FDT map:: If allow-repack is used then 'orig-offset' and 'orig-size' properties are added as necessary. See the binman README. +When extracting files, an alternative 'fdt' format is available for fdtmaps. +Use `binman extract -F fdt ...` to use this. It will export a devicetree, +without the fdtmap header, so it can be viewed with `fdtdump`. + Entry: files: A set of files arranged in a section @@ -855,7 +859,7 @@ SetImagePos(image_pos): Binman calls this after the image has been packed, to update the location that all the entries ended up at. -ReadChildData(child, decomp): +ReadChildData(child, decomp, alt_format): The default version of this may be good enough, if you are able to implement SetImagePos() correctly. But that is a bit of a bypass, so you can override this method to read from your custom file format. It @@ -868,6 +872,11 @@ ReadChildData(child, decomp): uncompress it first, then return the uncompressed data (`decomp` is True). This is used by the `binman extract -U` option. + If your entry supports alternative formats, the alt_format provides the + alternative format that the user has selected. Your function should + return data in that format. This is used by the 'binman extract -l' + option. + Binman calls this when reading in an image, in order to populate all the entries with the data from that image (`binman ls`). diff --git a/tools/binman/entry.py b/tools/binman/entry.py index e7a8365fd5..61642bf501 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -815,7 +815,7 @@ features to produce new behaviours. self.AddEntryInfo(entries, indent, self.name, self.etype, self.size, self.image_pos, self.uncomp_size, self.offset, self) - def ReadData(self, decomp=True): + def ReadData(self, decomp=True, alt_format=None): """Read the data for an entry from the image This is used when the image has been read in and we want to extract the @@ -832,19 +832,20 @@ features to produce new behaviours. # although compressed sections are currently not supported tout.Debug("ReadChildData section '%s', entry '%s'" % (self.section.GetPath(), self.GetPath())) - data = self.section.ReadChildData(self, decomp) + data = self.section.ReadChildData(self, decomp, alt_format) return data - def ReadChildData(self, child, decomp=True): + def ReadChildData(self, child, decomp=True, alt_format=None): """Read the data for a particular child entry This reads data from the parent and extracts the piece that relates to the given child. Args: - child: Child entry to read data for (must be valid) - decomp: True to decompress any compressed data before returning it; - False to return the raw, uncompressed data + child (Entry): Child entry to read data for (must be valid) + decomp (bool): True to decompress any compressed data before + returning it; False to return the raw, uncompressed data + alt_format (str): Alternative format to read in, or None Returns: Data for the child (bytes) @@ -857,6 +858,20 @@ features to produce new behaviours. self.ProcessContentsUpdate(data) self.Detail('Loaded data size %x' % len(data)) + def GetAltFormat(self, data, alt_format): + """Read the data for an extry in an alternative format + + Supported formats are list in the documentation for each entry. An + example is fdtmap which provides . + + Args: + data (bytes): Data to convert (this should have been produced by the + entry) + alt_format (str): Format to use + + """ + pass + def GetImage(self): """Get the image containing this entry @@ -997,3 +1012,13 @@ features to produce new behaviours. tout.Info("Node '%s': etype '%s': %s selected" % (node.path, etype, new_etype)) return True + + def CheckAltFormats(self, alt_formats): + """Add any alternative formats supported by this entry type + + Args: + alt_formats (dict): Dict to add alt_formats to: + key: Name of alt format + value: Help text + """ + pass diff --git a/tools/binman/etype/cbfs.py b/tools/binman/etype/cbfs.py index 2459388f84..cc1fbdf4b5 100644 --- a/tools/binman/etype/cbfs.py +++ b/tools/binman/etype/cbfs.py @@ -276,13 +276,13 @@ class Entry_cbfs(Entry): def GetEntries(self): return self._entries - def ReadData(self, decomp=True): - data = super().ReadData(True) + def ReadData(self, decomp=True, alt_format=None): + data = super().ReadData(True, alt_format) return data - def ReadChildData(self, child, decomp=True): + def ReadChildData(self, child, decomp=True, alt_format=None): if not self.reader: - data = super().ReadData(True) + data = super().ReadData(True, alt_format) self.reader = cbfs_util.CbfsReader(data) reader = self.reader cfile = reader.files.get(child.name) diff --git a/tools/binman/etype/fdtmap.py b/tools/binman/etype/fdtmap.py index 2339feeba8..aaaf2de438 100644 --- a/tools/binman/etype/fdtmap.py +++ b/tools/binman/etype/fdtmap.py @@ -74,6 +74,10 @@ class Entry_fdtmap(Entry): If allow-repack is used then 'orig-offset' and 'orig-size' properties are added as necessary. See the binman README. + + When extracting files, an alternative 'fdt' format is available for fdtmaps. + Use `binman extract -F fdt ...` to use this. It will export a devicetree, + without the fdtmap header, so it can be viewed with `fdtdump`. """ def __init__(self, section, etype, node): # Put these here to allow entry-docs and help to work without libfdt @@ -86,6 +90,10 @@ class Entry_fdtmap(Entry): from dtoc.fdt import Fdt super().__init__(section, etype, node) + self.alt_formats = ['fdt'] + + def CheckAltFormats(self, alt_formats): + alt_formats['fdt'] = self, 'Extract the devicetree blob from the fdtmap' def _GetFdtmap(self): """Build an FDT map from the entries in the current image @@ -147,3 +155,7 @@ class Entry_fdtmap(Entry): processing, e.g. the image-pos properties. """ return self.ProcessContentsUpdate(self._GetFdtmap()) + + def GetAltFormat(self, data, alt_format): + if alt_format == 'fdt': + return data[FDTMAP_HDR_LEN:] diff --git a/tools/binman/etype/section.py b/tools/binman/etype/section.py index 6ce07dd37d..43436a11f2 100644 --- a/tools/binman/etype/section.py +++ b/tools/binman/etype/section.py @@ -80,7 +80,7 @@ class Entry_section(Entry): Binman calls this after the image has been packed, to update the location that all the entries ended up at. - ReadChildData(child, decomp): + ReadChildData(child, decomp, alt_format): The default version of this may be good enough, if you are able to implement SetImagePos() correctly. But that is a bit of a bypass, so you can override this method to read from your custom file format. It @@ -93,6 +93,11 @@ class Entry_section(Entry): uncompress it first, then return the uncompressed data (`decomp` is True). This is used by the `binman extract -U` option. + If your entry supports alternative formats, the alt_format provides the + alternative format that the user has selected. Your function should + return data in that format. This is used by the 'binman extract -l' + option. + Binman calls this when reading in an image, in order to populate all the entries with the data from that image (`binman ls`). @@ -750,9 +755,9 @@ class Entry_section(Entry): """ return self._sort - def ReadData(self, decomp=True): + def ReadData(self, decomp=True, alt_format=None): tout.Info("ReadData path='%s'" % self.GetPath()) - parent_data = self.section.ReadData(True) + parent_data = self.section.ReadData(True, alt_format) offset = self.offset - self.section._skip_at_start data = parent_data[offset:offset + self.size] tout.Info( @@ -761,9 +766,9 @@ class Entry_section(Entry): self.size, len(data))) return data - def ReadChildData(self, child, decomp=True): + def ReadChildData(self, child, decomp=True, alt_format=None): tout.Debug(f"ReadChildData for child '{child.GetPath()}'") - parent_data = self.ReadData(True) + parent_data = self.ReadData(True, alt_format) offset = child.offset - self._skip_at_start tout.Debug("Extract for child '%s': offset %#x, skip_at_start %#x, result %#x" % (child.GetPath(), child.offset, self._skip_at_start, offset)) @@ -775,6 +780,10 @@ class Entry_section(Entry): tout.Info("%s: Decompressing data size %#x with algo '%s' to data size %#x" % (child.GetPath(), len(indata), child.compress, len(data))) + if alt_format: + new_data = child.GetAltFormat(data, alt_format) + if new_data is not None: + data = new_data return data def WriteChildData(self, child): @@ -846,3 +855,7 @@ class Entry_section(Entry): if not self._ignore_missing: missing = ', '.join(missing) entry.Raise(f'Missing required properties/entry args: {missing}') + + def CheckAltFormats(self, alt_formats): + for entry in self._entries.values(): + entry.CheckAltFormats(alt_formats) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 0f4330b680..d3a6cbf71d 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -4681,6 +4681,40 @@ class TestFunctional(unittest.TestCase): binary=False) self.assertEqual(version, state.GetVersion(self._indir)) + def testAltFormat(self): + """Test that alternative formats can be used to extract""" + self._DoReadFileRealDtb('213_fdtmap_alt_format.dts') + + try: + tmpdir, updated_fname = self._SetupImageInTmpdir() + with test_util.capture_sys_output() as (stdout, _): + self._DoBinman('extract', '-i', updated_fname, '-F', 'list') + self.assertEqual( + '''Flag (-F) Entry type Description +fdt fdtmap Extract the devicetree blob from the fdtmap +''', + stdout.getvalue()) + + dtb = os.path.join(tmpdir, 'fdt.dtb') + self._DoBinman('extract', '-i', updated_fname, '-F', 'fdt', '-f', + dtb, 'fdtmap') + + # Check that we can read it and it can be scanning, meaning it does + # not have a 16-byte fdtmap header + data = tools.ReadFile(dtb) + dtb = fdt.Fdt.FromData(data) + dtb.Scan() + + # Now check u-boot which has no alt_format + fname = os.path.join(tmpdir, 'fdt.dtb') + self._DoBinman('extract', '-i', updated_fname, '-F', 'dummy', + '-f', fname, 'u-boot') + data = tools.ReadFile(fname) + self.assertEqual(U_BOOT_DATA, data) + + finally: + shutil.rmtree(tmpdir) + if __name__ == "__main__": unittest.main() diff --git a/tools/binman/image.py b/tools/binman/image.py index 891e8b488e..f0a7d65299 100644 --- a/tools/binman/image.py +++ b/tools/binman/image.py @@ -223,7 +223,7 @@ class Image(section.Entry_section): entries = entry.GetEntries() return entry - def ReadData(self, decomp=True): + def ReadData(self, decomp=True, alt_format=None): tout.Debug("Image '%s' ReadData(), size=%#x" % (self.GetPath(), len(self._data))) return self._data diff --git a/tools/binman/test/213_fdtmap_alt_format.dts b/tools/binman/test/213_fdtmap_alt_format.dts new file mode 100644 index 0000000000..d9aef04bcf --- /dev/null +++ b/tools/binman/test/213_fdtmap_alt_format.dts @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + u-boot { + }; + fdtmap { + }; + }; +}; diff --git a/tools/binman/test/214_no_alt_format.dts b/tools/binman/test/214_no_alt_format.dts new file mode 100644 index 0000000000..f00bcdd576 --- /dev/null +++ b/tools/binman/test/214_no_alt_format.dts @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + u-boot { + }; + }; +}; -- 2.39.5