tools/binman/bintool.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright 2022 Google LLC
   3 # Copyright (C) 2022 Weidmüller Interface GmbH & Co. KG
   4 # Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
   5 #
   6 """Base class for all bintools
   7
   8 This defines the common functionality for all bintools, including running
   9 the tool, checking its version and fetching it if needed.
  10 """
  11
  12 import collections
  13 import glob
  14 import importlib
  15 import multiprocessing
  16 import os
  17 import shutil
  18 import tempfile
  19 import urllib.error
  20
  21 from patman import command
  22 from patman import terminal
  23 from patman import tools
  24 from patman import tout
  25
  26 BINMAN_DIR = os.path.dirname(os.path.realpath(__file__))
  27
  28 # Format string for listing bintools, see also the header in list_all()
  29 FORMAT = '%-16.16s %-12.12s %-26.26s %s'
  30
  31 # List of known modules, to avoid importing the module multiple times
  32 modules = {}
  33
  34 # Possible ways of fetching a tool (FETCH_COUNT is number of ways)
  35 FETCH_ANY, FETCH_BIN, FETCH_BUILD, FETCH_COUNT = range(4)
  36
  37 FETCH_NAMES = {
  38     FETCH_ANY: 'any method',
  39     FETCH_BIN: 'binary download',
  40     FETCH_BUILD: 'build from source'
  41     }
  42
  43 # Status of tool fetching
  44 FETCHED, FAIL, PRESENT, STATUS_COUNT = range(4)
  45
  46 DOWNLOAD_DESTDIR = os.path.join(os.getenv('HOME'), 'bin')
  47
  48 class Bintool:
  49     """Tool which operates on binaries to help produce entry contents
  50
  51     This is the base class for all bintools
  52     """
  53     # List of bintools to regard as missing
  54     missing_list = []
  55
  56     def __init__(self, name, desc, version_regex=None, version_args='-V'):
  57         self.name = name
  58         self.desc = desc
  59         self.version_regex = version_regex
  60         self.version_args = version_args
  61
  62     @staticmethod
  63     def find_bintool_class(btype):
  64         """Look up the bintool class for bintool
  65
  66         Args:
  67             byte: Bintool to use, e.g. 'mkimage'
  68
  69         Returns:
  70             The bintool class object if found, else a tuple:
  71                 module name that could not be found
  72                 exception received
  73         """
  74         # Convert something like 'u-boot' to 'u_boot' since we are only
  75         # interested in the type.
  76         module_name = btype.replace('-', '_')
  77         module = modules.get(module_name)
  78         class_name = f'Bintool{module_name}'
  79
  80         # Import the module if we have not already done so
  81         if not module:
  82             try:
  83                 module = importlib.import_module('binman.btool.' + module_name)
  84             except ImportError as exc:
  85                 try:
  86                     # Deal with classes which must be renamed due to conflicts
  87                     # with Python libraries
  88                     module = importlib.import_module('binman.btool.btool_' +
  89                                                      module_name)
  90                 except ImportError:
  91                     return module_name, exc
  92             modules[module_name] = module
  93
  94         # Look up the expected class name
  95         return getattr(module, class_name)
  96
  97     @staticmethod
  98     def create(name):
  99         """Create a new bintool object
 100
 101         Args:
 102             name (str): Bintool to create, e.g. 'mkimage'
 103
 104         Returns:
 105             A new object of the correct type (a subclass of Binutil)
 106         """
 107         cls = Bintool.find_bintool_class(name)
 108         if isinstance(cls, tuple):
 109             raise ValueError("Cannot import bintool module '%s': %s" % cls)
 110
 111         # Call its constructor to get the object we want.
 112         obj = cls(name)
 113         return obj
 114
 115     def show(self):
 116         """Show a line of information about a bintool"""
 117         if self.is_present():
 118             version = self.version()
 119         else:
 120             version = '-'
 121         print(FORMAT % (self.name, version, self.desc,
 122                         self.get_path() or '(not found)'))
 123
 124     @classmethod
 125     def set_missing_list(cls, missing_list):
 126         cls.missing_list = missing_list or []
 127
 128     @staticmethod
 129     def get_tool_list(include_testing=False):
 130         """Get a list of the known tools
 131
 132         Returns:
 133             list of str: names of all tools known to binman
 134         """
 135         files = glob.glob(os.path.join(BINMAN_DIR, 'btool/*'))
 136         names = [os.path.splitext(os.path.basename(fname))[0]
 137                  for fname in files]
 138         names = [name for name in names if name[0] != '_']
 139         names = [name[6:] if name.startswith('btool_') else name
 140                  for name in names]
 141         if include_testing:
 142             names.append('_testing')
 143         return sorted(names)
 144
 145     @staticmethod
 146     def list_all():
 147         """List all the bintools known to binman"""
 148         names = Bintool.get_tool_list()
 149         print(FORMAT % ('Name', 'Version', 'Description', 'Path'))
 150         print(FORMAT % ('-' * 15,'-' * 11, '-' * 25, '-' * 30))
 151         for name in names:
 152             btool = Bintool.create(name)
 153             btool.show()
 154
 155     def is_present(self):
 156         """Check if a bintool is available on the system
 157
 158         Returns:
 159             bool: True if available, False if not
 160         """
 161         if self.name in self.missing_list:
 162             return False
 163         return bool(self.get_path())
 164
 165     def get_path(self):
 166         """Get the path of a bintool
 167
 168         Returns:
 169             str: Path to the tool, if available, else None
 170         """
 171         return tools.tool_find(self.name)
 172
 173     def fetch_tool(self, method, col, skip_present):
 174         """Fetch a single tool
 175
 176         Args:
 177             method (FETCH_...): Method to use
 178             col (terminal.Color): Color terminal object
 179             skip_present (boo;): Skip fetching if it is already present
 180
 181         Returns:
 182             int: Result of fetch either FETCHED, FAIL, PRESENT
 183         """
 184         def try_fetch(meth):
 185             res = None
 186             try:
 187                 res = self.fetch(meth)
 188             except urllib.error.URLError as uerr:
 189                 message = uerr.reason
 190                 print(col.build(col.RED, f'- {message}'))
 191
 192             except ValueError as exc:
 193                 print(f'Exception: {exc}')
 194             return res
 195
 196         if skip_present and self.is_present():
 197             return PRESENT
 198         print(col.build(col.YELLOW, 'Fetch: %s' % self.name))
 199         if method == FETCH_ANY:
 200             for try_method in range(1, FETCH_COUNT):
 201                 print(f'- trying method: {FETCH_NAMES[try_method]}')
 202                 result = try_fetch(try_method)
 203                 if result:
 204                     break
 205         else:
 206             result = try_fetch(method)
 207         if not result:
 208             return FAIL
 209         if result is not True:
 210             fname, tmpdir = result
 211             dest = os.path.join(DOWNLOAD_DESTDIR, self.name)
 212             print(f"- writing to '{dest}'")
 213             shutil.move(fname, dest)
 214             if tmpdir:
 215                 shutil.rmtree(tmpdir)
 216         return FETCHED
 217
 218     @staticmethod
 219     def fetch_tools(method, names_to_fetch):
 220         """Fetch bintools from a suitable place
 221
 222         This fetches or builds the requested bintools so that they can be used
 223         by binman
 224
 225         Args:
 226             names_to_fetch (list of str): names of bintools to fetch
 227
 228         Returns:
 229             True on success, False on failure
 230         """
 231         def show_status(color, prompt, names):
 232             print(col.build(
 233                 color, f'{prompt}:%s{len(names):2}: %s' %
 234                 (' ' * (16 - len(prompt)), ' '.join(names))))
 235
 236         col = terminal.Color()
 237         skip_present = False
 238         name_list = names_to_fetch
 239         if len(names_to_fetch) == 1 and names_to_fetch[0] in ['all', 'missing']:
 240             name_list = Bintool.get_tool_list()
 241             if names_to_fetch[0] == 'missing':
 242                 skip_present = True
 243             print(col.build(col.YELLOW,
 244                             'Fetching tools:      %s' % ' '.join(name_list)))
 245         status = collections.defaultdict(list)
 246         for name in name_list:
 247             btool = Bintool.create(name)
 248             result = btool.fetch_tool(method, col, skip_present)
 249             status[result].append(name)
 250             if result == FAIL:
 251                 if method == FETCH_ANY:
 252                     print('- failed to fetch with all methods')
 253                 else:
 254                     print(f"- method '{FETCH_NAMES[method]}' is not supported")
 255
 256         if len(name_list) > 1:
 257             if skip_present:
 258                 show_status(col.GREEN, 'Already present', status[PRESENT])
 259             show_status(col.GREEN, 'Tools fetched', status[FETCHED])
 260             if status[FAIL]:
 261                 show_status(col.RED, 'Failures', status[FAIL])
 262         return not status[FAIL]
 263
 264     def run_cmd_result(self, *args, binary=False, raise_on_error=True):
 265         """Run the bintool using command-line arguments
 266
 267         Args:
 268             args (list of str): Arguments to provide, in addition to the bintool
 269                 name
 270             binary (bool): True to return output as bytes instead of str
 271             raise_on_error (bool): True to raise a ValueError exception if the
 272                 tool returns a non-zero return code
 273
 274         Returns:
 275             CommandResult: Resulting output from the bintool, or None if the
 276                 tool is not present
 277         """
 278         if self.name in self.missing_list:
 279             return None
 280         name = os.path.expanduser(self.name)  # Expand paths containing ~
 281         all_args = (name,) + args
 282         env = tools.get_env_with_path()
 283         tout.detail(f"bintool: {' '.join(all_args)}")
 284         result = command.run_pipe(
 285             [all_args], capture=True, capture_stderr=True, env=env,
 286             raise_on_error=False, binary=binary)
 287
 288         if result.return_code:
 289             # Return None if the tool was not found. In this case there is no
 290             # output from the tool and it does not appear on the path. We still
 291             # try to run it (as above) since RunPipe() allows faking the tool's
 292             # output
 293             if not any([result.stdout, result.stderr, tools.tool_find(name)]):
 294                 tout.info(f"bintool '{name}' not found")
 295                 return None
 296             if raise_on_error:
 297                 tout.info(f"bintool '{name}' failed")
 298                 raise ValueError("Error %d running '%s': %s" %
 299                                 (result.return_code, ' '.join(all_args),
 300                                 result.stderr or result.stdout))
 301         if result.stdout:
 302             tout.debug(result.stdout)
 303         if result.stderr:
 304             tout.debug(result.stderr)
 305         return result
 306
 307     def run_cmd(self, *args, binary=False):
 308         """Run the bintool using command-line arguments
 309
 310         Args:
 311             args (list of str): Arguments to provide, in addition to the bintool
 312                 name
 313             binary (bool): True to return output as bytes instead of str
 314
 315         Returns:
 316             str or bytes: Resulting stdout from the bintool
 317         """
 318         result = self.run_cmd_result(*args, binary=binary)
 319         if result:
 320             return result.stdout
 321
 322     @classmethod
 323     def build_from_git(cls, git_repo, make_target, bintool_path, flags=None):
 324         """Build a bintool from a git repo
 325
 326         This clones the repo in a temporary directory, builds it with 'make',
 327         then returns the filename of the resulting executable bintool
 328
 329         Args:
 330             git_repo (str): URL of git repo
 331             make_target (str): Target to pass to 'make' to build the tool
 332             bintool_path (str): Relative path of the tool in the repo, after
 333                 build is complete
 334             flags (list of str): Flags or variables to pass to make, or None
 335
 336         Returns:
 337             tuple:
 338                 str: Filename of fetched file to copy to a suitable directory
 339                 str: Name of temp directory to remove, or None
 340             or None on error
 341         """
 342         tmpdir = tempfile.mkdtemp(prefix='binmanf.')
 343         print(f"- clone git repo '{git_repo}' to '{tmpdir}'")
 344         tools.run('git', 'clone', '--depth', '1', git_repo, tmpdir)
 345         print(f"- build target '{make_target}'")
 346         cmd = ['make', '-C', tmpdir, '-j', f'{multiprocessing.cpu_count()}',
 347                make_target]
 348         if flags:
 349             cmd += flags
 350         tools.run(*cmd)
 351         fname = os.path.join(tmpdir, bintool_path)
 352         if not os.path.exists(fname):
 353             print(f"- File '{fname}' was not produced")
 354             return None
 355         return fname, tmpdir
 356
 357     @classmethod
 358     def fetch_from_url(cls, url):
 359         """Fetch a bintool from a URL
 360
 361         Args:
 362             url (str): URL to fetch from
 363
 364         Returns:
 365             tuple:
 366                 str: Filename of fetched file to copy to a suitable directory
 367                 str: Name of temp directory to remove, or None
 368         """
 369         fname, tmpdir = tools.download(url)
 370         tools.run('chmod', 'a+x', fname)
 371         return fname, tmpdir
 372
 373     @classmethod
 374     def fetch_from_drive(cls, drive_id):
 375         """Fetch a bintool from Google drive
 376
 377         Args:
 378             drive_id (str): ID of file to fetch. For a URL of the form
 379             'https://drive.google.com/file/d/xxx/view?usp=sharing' the value
 380             passed here should be 'xxx'
 381
 382         Returns:
 383             tuple:
 384                 str: Filename of fetched file to copy to a suitable directory
 385                 str: Name of temp directory to remove, or None
 386         """
 387         url = f'https://drive.google.com/uc?export=download&id={drive_id}'
 388         return cls.fetch_from_url(url)
 389
 390     @classmethod
 391     def apt_install(cls, package):
 392         """Install a bintool using the 'aot' tool
 393
 394         This requires use of servo so may request a password
 395
 396         Args:
 397             package (str): Name of package to install
 398
 399         Returns:
 400             True, assuming it completes without error
 401         """
 402         args = ['sudo', 'apt', 'install', '-y', package]
 403         print('- %s' % ' '.join(args))
 404         tools.run(*args)
 405         return True
 406
 407     @staticmethod
 408     def WriteDocs(modules, test_missing=None):
 409         """Write out documentation about the various bintools to stdout
 410
 411         Args:
 412             modules: List of modules to include
 413             test_missing: Used for testing. This is a module to report
 414                 as missing
 415         """
 416         print('''.. SPDX-License-Identifier: GPL-2.0+
 417
 418 Binman bintool Documentation
 419 ============================
 420
 421 This file describes the bintools (binary tools) supported by binman. Bintools
 422 are binman's name for external executables that it runs to generate or process
 423 binaries. It is fairly easy to create new bintools. Just add a new file to the
 424 'btool' directory. You can use existing bintools as examples.
 425
 426
 427 ''')
 428         modules = sorted(modules)
 429         missing = []
 430         for name in modules:
 431             module = Bintool.find_bintool_class(name)
 432             docs = getattr(module, '__doc__')
 433             if test_missing == name:
 434                 docs = None
 435             if docs:
 436                 lines = docs.splitlines()
 437                 first_line = lines[0]
 438                 rest = [line[4:] for line in lines[1:]]
 439                 hdr = 'Bintool: %s: %s' % (name, first_line)
 440                 print(hdr)
 441                 print('-' * len(hdr))
 442                 print('\n'.join(rest))
 443                 print()
 444                 print()
 445             else:
 446                 missing.append(name)
 447
 448         if missing:
 449             raise ValueError('Documentation is missing for modules: %s' %
 450                              ', '.join(missing))
 451
 452     # pylint: disable=W0613
 453     def fetch(self, method):
 454         """Fetch handler for a bintool
 455
 456         This should be implemented by the base class
 457
 458         Args:
 459             method (FETCH_...): Method to use
 460
 461         Returns:
 462             tuple:
 463                 str: Filename of fetched file to copy to a suitable directory
 464                 str: Name of temp directory to remove, or None
 465             or True if the file was fetched and already installed
 466             or None if no fetch() implementation is available
 467
 468         Raises:
 469             Valuerror: Fetching could not be completed
 470         """
 471         print(f"No method to fetch bintool '{self.name}'")
 472         return False
 473
 474     def version(self):
 475         """Version handler for a bintool
 476
 477         Returns:
 478             str: Version string for this bintool
 479         """
 480         if self.version_regex is None:
 481             return 'unknown'
 482
 483         import re
 484
 485         result = self.run_cmd_result(self.version_args)
 486         out = result.stdout.strip()
 487         if not out:
 488             out = result.stderr.strip()
 489         if not out:
 490             return 'unknown'
 491
 492         m_version = re.search(self.version_regex, out)
 493         return m_version.group(1) if m_version else out
 494
 495
 496 class BintoolPacker(Bintool):
 497     """Tool which compression / decompression entry contents
 498
 499     This is a bintools base class for compression / decompression packer
 500
 501     Properties:
 502         name: Name of packer tool
 503         compression: Compression type (COMPRESS_...), value of 'name' property
 504             if none
 505         compress_args: List of positional args provided to tool for compress,
 506             ['--compress'] if none
 507         decompress_args: List of positional args provided to tool for
 508             decompress, ['--decompress'] if none
 509         fetch_package: Name of the tool installed using the apt, value of 'name'
 510             property if none
 511         version_regex: Regular expressions to extract the version from tool
 512             version output,  '(v[0-9.]+)' if none
 513     """
 514     def __init__(self, name, compression=None, compress_args=None,
 515                  decompress_args=None, fetch_package=None,
 516                  version_regex=r'(v[0-9.]+)', version_args='-V'):
 517         desc = '%s compression' % (compression if compression else name)
 518         super().__init__(name, desc, version_regex, version_args)
 519         if compress_args is None:
 520             compress_args = ['--compress']
 521         self.compress_args = compress_args
 522         if decompress_args is None:
 523             decompress_args = ['--decompress']
 524         self.decompress_args = decompress_args
 525         if fetch_package is None:
 526             fetch_package = name
 527         self.fetch_package = fetch_package
 528
 529     def compress(self, indata):
 530         """Compress data
 531
 532         Args:
 533             indata (bytes): Data to compress
 534
 535         Returns:
 536             bytes: Compressed data
 537         """
 538         with tempfile.NamedTemporaryFile(prefix='comp.tmp',
 539                                          dir=tools.get_output_dir()) as tmp:
 540             tools.write_file(tmp.name, indata)
 541             args = self.compress_args + ['--stdout', tmp.name]
 542             return self.run_cmd(*args, binary=True)
 543
 544     def decompress(self, indata):
 545         """Decompress data
 546
 547         Args:
 548             indata (bytes): Data to decompress
 549
 550         Returns:
 551             bytes: Decompressed data
 552         """
 553         with tempfile.NamedTemporaryFile(prefix='decomp.tmp',
 554                                          dir=tools.get_output_dir()) as inf:
 555             tools.write_file(inf.name, indata)
 556             args = self.decompress_args + ['--stdout', inf.name]
 557             return self.run_cmd(*args, binary=True)
 558
 559     def fetch(self, method):
 560         """Fetch handler
 561
 562         This installs the gzip package using the apt utility.
 563
 564         Args:
 565             method (FETCH_...): Method to use
 566
 567         Returns:
 568             True if the file was fetched and now installed, None if a method
 569             other than FETCH_BIN was requested
 570
 571         Raises:
 572             Valuerror: Fetching could not be completed
 573         """
 574         if method != FETCH_BIN:
 575             return None
 576         return self.apt_install(self.fetch_package)