"""
from collections import OrderedDict
+import concurrent.futures
import re
import sys
from binman.entry import Entry
+from binman import state
from dtoc import fdt_util
from patman import tools
from patman import tout
class Entry_section(Entry):
"""Entry that contains other entries
- Properties / Entry arguments: (see binman README for more information):
- pad-byte: Pad byte to use when padding
- sort-by-offset: True if entries should be sorted by offset, False if
- they must be in-order in the device tree description
-
- end-at-4gb: Used to build an x86 ROM which ends at 4GB (2^32)
+ A section is an entry which can contain other entries, thus allowing
+ hierarchical images to be created. See 'Sections and hierarchical images'
+ in the binman README for more information.
- skip-at-start: Number of bytes before the first entry starts. These
- effectively adjust the starting offset of entries. For example,
- if this is 16, then the first entry would start at 16. An entry
- with offset = 20 would in fact be written at offset 4 in the image
- file, since the first 16 bytes are skipped when writing.
- name-prefix: Adds a prefix to the name of every entry in the section
- when writing out the map
- align_default: Default alignment for this section, if no alignment is
- given in the entry
+ The base implementation simply joins the various entries together, using
+ various rules about alignment, etc.
+
+ Subclassing
+ ~~~~~~~~~~~
- Properties:
- allow_missing: True if this section permits external blobs to be
- missing their contents. The second will produce an image but of
- course it will not work.
+ This class can be subclassed to support other file formats which hold
+ multiple entries, such as CBFS. To do this, override the following
+ functions. The documentation here describes what your function should do.
+ For example code, see etypes which subclass `Entry_section`, or `cbfs.py`
+ for a more involved example::
+
+ $ grep -l \(Entry_section tools/binman/etype/*.py
+
+ ReadNode()
+ Call `super().ReadNode()`, then read any special properties for the
+ section. Then call `self.ReadEntries()` to read the entries.
+
+ Binman calls this at the start when reading the image description.
+
+ ReadEntries()
+ Read in the subnodes of the section. This may involve creating entries
+ of a particular etype automatically, as well as reading any special
+ properties in the entries. For each entry, entry.ReadNode() should be
+ called, to read the basic entry properties. The properties should be
+ added to `self._entries[]`, in the correct order, with a suitable name.
+
+ Binman calls this at the start when reading the image description.
+
+ BuildSectionData(required)
+ Create the custom file format that you want and return it as bytes.
+ This likely sets up a file header, then loops through the entries,
+ adding them to the file. For each entry, call `entry.GetData()` to
+ obtain the data. If that returns None, and `required` is False, then
+ this method must give up and return None. But if `required` is True then
+ it should assume that all data is valid.
+
+ Binman calls this when packing the image, to find out the size of
+ everything. It is called again at the end when building the final image.
+
+ SetImagePos(image_pos):
+ Call `super().SetImagePos(image_pos)`, then set the `image_pos` values
+ for each of the entries. This should use the custom file format to find
+ the `start offset` (and `image_pos`) of each entry. If the file format
+ uses compression in such a way that there is no offset available (other
+ than reading the whole file and decompressing it), then the offsets for
+ affected entries can remain unset (`None`). The size should also be set
+ if possible.
+
+ Binman calls this after the image has been packed, to update the
+ location that all the entries ended up at.
+
+ ReadChildData(child, decomp, alt_format):
+ The default version of this may be good enough, if you are able to
+ implement SetImagePos() correctly. But that is a bit of a bypass, so
+ you can override this method to read from your custom file format. It
+ should read the entire entry containing the custom file using
+ `super().ReadData(True)`, then parse the file to get the data for the
+ given child, then return that data.
+
+ If your file format supports compression, the `decomp` argument tells
+ you whether to return the compressed data (`decomp` is False) or to
+ uncompress it first, then return the uncompressed data (`decomp` is
+ True). This is used by the `binman extract -U` option.
+
+ If your entry supports alternative formats, the alt_format provides the
+ alternative format that the user has selected. Your function should
+ return data in that format. This is used by the 'binman extract -l'
+ option.
+
+ Binman calls this when reading in an image, in order to populate all the
+ entries with the data from that image (`binman ls`).
+
+ WriteChildData(child):
+ Binman calls this after `child.data` is updated, to inform the custom
+ file format about this, in case it needs to do updates.
+
+ The default version of this does nothing and probably needs to be
+ overridden for the 'binman replace' command to work. Your version should
+ use `child.data` to update the data for that child in the custom file
+ format.
+
+ Binman calls this when updating an image that has been read in and in
+ particular to update the data for a particular entry (`binman replace`)
+
+ Properties / Entry arguments
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ See :ref:`develop/package/binman:Image description format` for more
+ information.
+
+ align-default
+ Default alignment for this section, if no alignment is given in the
+ entry
+
+ pad-byte
+ Pad byte to use when padding
+
+ sort-by-offset
+ True if entries should be sorted by offset, False if they must be
+ in-order in the device tree description
+
+ end-at-4gb
+ Used to build an x86 ROM which ends at 4GB (2^32)
+
+ name-prefix
+ Adds a prefix to the name of every entry in the section when writing out
+ the map
+
+ skip-at-start
+ Number of bytes before the first entry starts. These effectively adjust
+ the starting offset of entries. For example, if this is 16, then the
+ first entry would start at 16. An entry with offset = 20 would in fact
+ be written at offset 4 in the image file, since the first 16 bytes are
+ skipped when writing.
Since a section is also an entry, it inherits all the properies of entries
too.
- A section is an entry which can contain other entries, thus allowing
- hierarchical images to be created. See 'Sections and hierarchical images'
- in the binman README for more information.
+ Note that the `allow_missing` member controls whether this section permits
+ external blobs to be missing their contents. The option will produce an
+ image but of course it will not work. It is useful to make sure that
+ Continuous Integration systems can build without the binaries being
+ available. This is set by the `SetAllowMissing()` method, if
+ `--allow-missing` is passed to binman.
"""
def __init__(self, section, etype, node, test=False):
if not test:
self._skip_at_start = 0
self._name_prefix = fdt_util.GetString(self._node, 'name-prefix')
self.align_default = fdt_util.GetInt(self._node, 'align-default', 0)
- filename = fdt_util.GetString(self._node, 'filename')
- if filename:
- self._filename = filename
- self._ReadEntries()
+ self.ReadEntries()
- def _ReadEntries(self):
+ def ReadEntries(self):
for node in self._node.subnodes:
if node.name.startswith('hash') or node.name.startswith('signature'):
continue
entry = Entry.Create(self, node,
- expanded=self.GetImage().use_expanded)
+ expanded=self.GetImage().use_expanded,
+ missing_etype=self.GetImage().missing_etype)
entry.ReadNode()
entry.SetPrefix(self._name_prefix)
self._entries[node.name] = entry
"""Raises an error for this section
Args:
- msg: Error message to use in the raise string
+ msg (str): Error message to use in the raise string
Raises:
- ValueError()
+ ValueError: always
"""
raise ValueError("Section '%s': %s" % (self._node.path, msg))
for entry in self._entries.values():
entry.AddMissingProperties(have_image_pos)
- def ObtainContents(self):
- return self.GetEntryContents()
+ def ObtainContents(self, skip_entry=None):
+ return self.GetEntryContents(skip_entry=skip_entry)
def GetPaddedDataForEntry(self, entry, entry_data):
"""Get the data for an entry including any padding
pad_byte = (entry._pad_byte if isinstance(entry, Entry_section)
else self._pad_byte)
- data = b''
+ data = bytearray()
# Handle padding before the entry
if entry.pad_before:
data += tools.GetBytes(self._pad_byte, entry.pad_before)
return data
- def _BuildSectionData(self, required):
+ def BuildSectionData(self, required):
"""Build the contents of a section
This places all entries at the right place, dealing with padding before
pad-before and pad-after properties in the section items) since that is
handled by the parent section.
+ This should be overridden by subclasses which want to build their own
+ data structure for the section.
+
Args:
required: True if the data must be present, False if it is OK to
return None
Returns:
Contents of the section (bytes)
"""
- section_data = b''
+ section_data = bytearray()
for entry in self._entries.values():
entry_data = entry.GetData(required)
+
+ # This can happen when this section is referenced from a collection
+ # earlier in the image description. See testCollectionSection().
if not required and entry_data is None:
return None
data = self.GetPaddedDataForEntry(entry, entry_data)
This excludes any padding. If the section is compressed, the
compressed data is returned
"""
- data = self._BuildSectionData(required)
+ data = self.BuildSectionData(required)
if data is None:
return None
self.SetContents(data)
self._SortEntries()
self._ExpandEntries()
- data = self._BuildSectionData(True)
+ data = self.BuildSectionData(True)
self.SetContents(data)
self.CheckSize()
return entry
return None
- def GetEntryContents(self):
+ def GetEntryContents(self, skip_entry=None):
"""Call ObtainContents() for each entry in the section
"""
+ def _CheckDone(entry):
+ if entry != skip_entry:
+ if not entry.ObtainContents():
+ next_todo.append(entry)
+ return entry
+
todo = self._entries.values()
for passnum in range(3):
+ threads = state.GetThreads()
next_todo = []
- for entry in todo:
- if not entry.ObtainContents():
- next_todo.append(entry)
+
+ if threads == 0:
+ for entry in todo:
+ _CheckDone(entry)
+ else:
+ with concurrent.futures.ThreadPoolExecutor(
+ max_workers=threads) as executor:
+ future_to_data = {
+ entry: executor.submit(_CheckDone, entry)
+ for entry in todo}
+ timeout = 60
+ if self.GetImage().test_section_timeout:
+ timeout = 0
+ done, not_done = concurrent.futures.wait(
+ future_to_data.values(), timeout=timeout)
+ # Make sure we check the result, so any exceptions are
+ # generated. Check the results in entry order, since tests
+ # may expect earlier entries to fail first.
+ for entry in todo:
+ job = future_to_data[entry]
+ job.result()
+ if not_done:
+ self.Raise('Timed out obtaining contents')
+
todo = next_todo
if not todo:
break
+
if todo:
self.Raise('Internal error: Could not complete processing of contents: remaining %s' %
todo)
def ListEntries(self, entries, indent):
"""List the files in the section"""
- Entry.AddEntryInfo(entries, indent, self.name, 'section', self.size,
+ Entry.AddEntryInfo(entries, indent, self.name, self.etype, self.size,
self.image_pos, None, self.offset, self)
for entry in self._entries.values():
entry.ListEntries(entries, indent + 1)
"""
return self._sort
- def ReadData(self, decomp=True):
+ def ReadData(self, decomp=True, alt_format=None):
tout.Info("ReadData path='%s'" % self.GetPath())
- parent_data = self.section.ReadData(True)
+ parent_data = self.section.ReadData(True, alt_format)
offset = self.offset - self.section._skip_at_start
data = parent_data[offset:offset + self.size]
tout.Info(
self.size, len(data)))
return data
- def ReadChildData(self, child, decomp=True):
- tout.Debug("ReadChildData for child '%s'" % child.GetPath())
- parent_data = self.ReadData(True)
+ def ReadChildData(self, child, decomp=True, alt_format=None):
+ tout.Debug(f"ReadChildData for child '{child.GetPath()}'")
+ parent_data = self.ReadData(True, alt_format)
offset = child.offset - self._skip_at_start
tout.Debug("Extract for child '%s': offset %#x, skip_at_start %#x, result %#x" %
(child.GetPath(), child.offset, self._skip_at_start, offset))
tout.Info("%s: Decompressing data size %#x with algo '%s' to data size %#x" %
(child.GetPath(), len(indata), child.compress,
len(data)))
+ if alt_format:
+ new_data = child.GetAltFormat(data, alt_format)
+ if new_data is not None:
+ data = new_data
return data
def WriteChildData(self, child):
for entry in self._entries.values():
entry.SetAllowMissing(allow_missing)
+ def SetAllowFakeBlob(self, allow_fake):
+ """Set whether a section allows to create a fake blob
+
+ Args:
+ allow_fake_blob: True if allowed, False if not allowed
+ """
+ for entry in self._entries.values():
+ entry.SetAllowFakeBlob(allow_fake)
+
def CheckMissing(self, missing_list):
"""Check if any entries in this section have missing external blobs
for entry in self._entries.values():
entry.CheckMissing(missing_list)
+ def CheckFakedBlobs(self, faked_blobs_list):
+ """Check if any entries in this section have faked external blobs
+
+ If there are faked blobs, the entries are added to the list
+
+ Args:
+ fake_blobs_list: List of Entry objects to be added to
+ """
+ for entry in self._entries.values():
+ entry.CheckFakedBlobs(faked_blobs_list)
+
def _CollectEntries(self, entries, entries_by_name, add_entry):
"""Collect all the entries in an section
nothing.
Args:
- missing: List of missing properties / entry args, each a string
+ entry (Entry): Entry to raise the error on
+ missing (list of str): List of missing properties / entry args, each
+ a string
"""
if not self._ignore_missing:
- entry.Raise('Missing required properties/entry args: %s' %
- (', '.join(missing)))
+ missing = ', '.join(missing)
+ entry.Raise(f'Missing required properties/entry args: {missing}')
+
+ def CheckAltFormats(self, alt_formats):
+ for entry in self._entries.values():
+ entry.CheckAltFormats(alt_formats)
projects / thirdparty / u-boot.git / blobdiff