scripts/install-buildtools

   1 #!/usr/bin/env python3
   2
   3 # Buildtools and buildtools extended installer helper script
   4 #
   5 # Copyright (C) 2017-2020 Intel Corporation
   6 #
   7 # SPDX-License-Identifier: GPL-2.0-only
   8 #
   9 #  NOTE: --with-extended-buildtools is on by default
  10 #
  11 #  Example usage (extended buildtools from milestone):
  12 #    (1) using --url and --filename
  13 #        $ install-buildtools \
  14 #          --url http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools \
  15 #          --filename x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh
  16 #    (2) using --base-url, --release, --installer-version and --build-date
  17 #        $ install-buildtools \
  18 #          --base-url http://downloads.yoctoproject.org/releases/yocto \
  19 #          --release yocto-3.1_M3 \
  20 #          --installer-version 3.0+snapshot
  21 #          --build-date 202000315
  22 #
  23 #  Example usage (standard buildtools from release):
  24 #    (3) using --url and --filename
  25 #        $ install-buildtools --without-extended-buildtools \
  26 #          --url http://downloads.yoctoproject.org/releases/yocto/yocto-3.0.2/buildtools \
  27 #          --filename x86_64-buildtools-nativesdk-standalone-3.0.2.sh
  28 #    (4) using --base-url, --release and --installer-version
  29 #        $ install-buildtools --without-extended-buildtools \
  30 #          --base-url http://downloads.yoctoproject.org/releases/yocto \
  31 #          --release yocto-3.0.2 \
  32 #          --installer-version 3.0.2
  33 #
  34
  35 import argparse
  36 import logging
  37 import os
  38 import platform
  39 import re
  40 import shutil
  41 import shlex
  42 import stat
  43 import subprocess
  44 import sys
  45 import tempfile
  46 from urllib.parse import quote
  47
  48 scripts_path = os.path.dirname(os.path.realpath(__file__))
  49 lib_path = scripts_path + '/lib'
  50 sys.path = sys.path + [lib_path]
  51 import scriptutils
  52 import scriptpath
  53
  54
  55 PROGNAME = 'install-buildtools'
  56 logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout)
  57
  58 DEFAULT_INSTALL_DIR = os.path.join(os.path.split(scripts_path)[0],'buildtools')
  59 DEFAULT_BASE_URL = 'http://downloads.yoctoproject.org/releases/yocto'
  60 DEFAULT_RELEASE = 'yocto-4.1'
  61 DEFAULT_INSTALLER_VERSION = '4.1'
  62 DEFAULT_BUILDDATE = '202110XX'
  63
  64 # Python version sanity check
  65 if not (sys.version_info.major == 3 and sys.version_info.minor >= 4):
  66     logger.error("This script requires Python 3.4 or greater")
  67     logger.error("You have Python %s.%s" %
  68           (sys.version_info.major, sys.version_info.minor))
  69     sys.exit(1)
  70
  71 # The following three functions are copied directly from
  72 # bitbake/lib/bb/utils.py, in order to allow this script
  73 # to run on versions of python earlier than what bitbake
  74 # supports (e.g. less than Python 3.5 for YP 3.1 release)
  75
  76 def _hasher(method, filename):
  77     import mmap
  78
  79     with open(filename, "rb") as f:
  80         try:
  81             with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
  82                 for chunk in iter(lambda: mm.read(8192), b''):
  83                     method.update(chunk)
  84         except ValueError:
  85             # You can't mmap() an empty file so silence this exception
  86             pass
  87     return method.hexdigest()
  88
  89
  90 def md5_file(filename):
  91     """
  92     Return the hex string representation of the MD5 checksum of filename.
  93     """
  94     import hashlib
  95     return _hasher(hashlib.md5(), filename)
  96
  97 def sha256_file(filename):
  98     """
  99     Return the hex string representation of the 256-bit SHA checksum of
 100     filename.
 101     """
 102     import hashlib
 103     return _hasher(hashlib.sha256(), filename)
 104
 105
 106 def main():
 107     global DEFAULT_INSTALL_DIR
 108     global DEFAULT_BASE_URL
 109     global DEFAULT_RELEASE
 110     global DEFAULT_INSTALLER_VERSION
 111     global DEFAULT_BUILDDATE
 112     filename = ""
 113     release = ""
 114     buildtools_url = ""
 115     install_dir = ""
 116     arch = platform.machine()
 117
 118     parser = argparse.ArgumentParser(
 119         description="Buildtools installation helper",
 120         add_help=False)
 121     parser.add_argument('-u', '--url',
 122                         help='URL from where to fetch buildtools SDK installer, not '
 123                              'including filename (optional)\n'
 124                              'Requires --filename.',
 125                         action='store')
 126     parser.add_argument('-f', '--filename',
 127                         help='filename for the buildtools SDK installer to be installed '
 128                              '(optional)\nRequires --url',
 129                         action='store')
 130     parser.add_argument('-d', '--directory',
 131                         default=DEFAULT_INSTALL_DIR,
 132                         help='directory where buildtools SDK will be installed (optional)',
 133                         action='store')
 134     parser.add_argument('-r', '--release',
 135                         default=DEFAULT_RELEASE,
 136                         help='Yocto Project release string for SDK which will be '
 137                              'installed (optional)',
 138                         action='store')
 139     parser.add_argument('-V', '--installer-version',
 140                         default=DEFAULT_INSTALLER_VERSION,
 141                         help='version string for the SDK to be installed (optional)',
 142                         action='store')
 143     parser.add_argument('-b', '--base-url',
 144                         default=DEFAULT_BASE_URL,
 145                         help='base URL from which to fetch SDK (optional)', action='store')
 146     parser.add_argument('-t', '--build-date',
 147                         default=DEFAULT_BUILDDATE,
 148                         help='Build date of pre-release SDK (optional)', action='store')
 149     group = parser.add_mutually_exclusive_group()
 150     group.add_argument('--with-extended-buildtools', action='store_true',
 151                        dest='with_extended_buildtools',
 152                        default=True,
 153                        help='enable extended buildtools tarball (on by default)')
 154     group.add_argument('--without-extended-buildtools', action='store_false',
 155                        dest='with_extended_buildtools',
 156                        help='disable extended buildtools (traditional buildtools tarball)')
 157     group.add_argument('--make-only', action='store_true',
 158                        help='only install make tarball')
 159     group = parser.add_mutually_exclusive_group()
 160     group.add_argument('-c', '--check', help='enable checksum validation',
 161                         default=True, action='store_true')
 162     group.add_argument('-n', '--no-check', help='disable checksum validation',
 163                         dest="check", action='store_false')
 164     parser.add_argument('-D', '--debug', help='enable debug output',
 165                         action='store_true')
 166     parser.add_argument('-q', '--quiet', help='print only errors',
 167                         action='store_true')
 168
 169     parser.add_argument('-h', '--help', action='help',
 170                         default=argparse.SUPPRESS,
 171                         help='show this help message and exit')
 172
 173     args = parser.parse_args()
 174
 175     if args.make_only:
 176         args.with_extended_buildtools = False
 177
 178     if args.debug:
 179         logger.setLevel(logging.DEBUG)
 180     elif args.quiet:
 181         logger.setLevel(logging.ERROR)
 182
 183     if args.url and args.filename:
 184         logger.debug("--url and --filename detected. Ignoring --base-url "
 185                      "--release --installer-version  arguments.")
 186         filename = args.filename
 187         buildtools_url = "%s/%s" % (args.url, filename)
 188     else:
 189         if args.base_url:
 190             base_url = args.base_url
 191         else:
 192             base_url = DEFAULT_BASE_URL
 193         if args.release:
 194             # check if this is a pre-release "milestone" SDK
 195             m = re.search(r"^(?P<distro>[a-zA-Z\-]+)(?P<version>[0-9.]+)(?P<milestone>_M[1-9])$",
 196                           args.release)
 197             logger.debug("milestone regex: %s" % m)
 198             if m and m.group('milestone'):
 199                 logger.debug("release[distro]: %s" % m.group('distro'))
 200                 logger.debug("release[version]: %s" % m.group('version'))
 201                 logger.debug("release[milestone]: %s" % m.group('milestone'))
 202                 if not args.build_date:
 203                     logger.error("Milestone installers require --build-date")
 204                 else:
 205                     if args.make_only:
 206                         filename = "%s-buildtools-make-nativesdk-standalone-%s-%s.sh" % (
 207                             arch, args.installer_version, args.build_date)
 208                     elif args.with_extended_buildtools:
 209                         filename = "%s-buildtools-extended-nativesdk-standalone-%s-%s.sh" % (
 210                             arch, args.installer_version, args.build_date)
 211                     else:
 212                         filename = "%s-buildtools-nativesdk-standalone-%s-%s.sh" % (
 213                             arch, args.installer_version, args.build_date)
 214                     safe_filename = quote(filename)
 215                     buildtools_url = "%s/milestones/%s/buildtools/%s" % (base_url, args.release, safe_filename)
 216             # regular release SDK
 217             else:
 218                 if args.make_only:
 219                     filename = "%s-buildtools-make-nativesdk-standalone-%s.sh" % (arch, args.installer_version)
 220                 if args.with_extended_buildtools:
 221                     filename = "%s-buildtools-extended-nativesdk-standalone-%s.sh" % (arch, args.installer_version)
 222                 else:
 223                     filename = "%s-buildtools-nativesdk-standalone-%s.sh" % (arch, args.installer_version)
 224                 safe_filename = quote(filename)
 225                 buildtools_url = "%s/%s/buildtools/%s" % (base_url, args.release, safe_filename)
 226
 227     tmpsdk_dir = tempfile.mkdtemp()
 228     try:
 229         # Fetch installer
 230         logger.info("Fetching buildtools installer")
 231         tmpbuildtools = os.path.join(tmpsdk_dir, filename)
 232         ret = subprocess.call("wget -q -O %s %s" %
 233                               (tmpbuildtools, buildtools_url), shell=True)
 234         if ret != 0:
 235             logger.error("Could not download file from %s" % buildtools_url)
 236             return ret
 237
 238         # Verify checksum
 239         if args.check:
 240             logger.info("Fetching buildtools installer checksum")
 241             checksum_type = ""
 242             for checksum_type in ["md5sum", "sha256sum"]:
 243                 check_url = "{}.{}".format(buildtools_url, checksum_type)
 244                 checksum_filename = "{}.{}".format(filename, checksum_type)
 245                 tmpbuildtools_checksum = os.path.join(tmpsdk_dir, checksum_filename)
 246                 ret = subprocess.call("wget -q -O %s %s" %
 247                                       (tmpbuildtools_checksum, check_url), shell=True)
 248                 if ret == 0:
 249                     break
 250             else:
 251                 if ret != 0:
 252                     logger.error("Could not download file from %s" % check_url)
 253                     return ret
 254             regex = re.compile(r"^(?P<checksum>[0-9a-f]+)\s+(?P<path>.*/)?(?P<filename>.*)$")
 255             with open(tmpbuildtools_checksum, 'rb') as f:
 256                 original = f.read()
 257                 m = re.search(regex, original.decode("utf-8"))
 258                 logger.debug("checksum regex match: %s" % m)
 259                 logger.debug("checksum: %s" % m.group('checksum'))
 260                 logger.debug("path: %s" % m.group('path'))
 261                 logger.debug("filename: %s" % m.group('filename'))
 262                 if filename != m.group('filename'):
 263                     logger.error("Filename does not match name in checksum")
 264                     return 1
 265                 checksum = m.group('checksum')
 266             if checksum_type == "md5sum":
 267                 checksum_value = md5_file(tmpbuildtools)
 268             else:
 269                 checksum_value = sha256_file(tmpbuildtools)
 270             if checksum == checksum_value:
 271                     logger.info("Checksum success")
 272             else:
 273                 logger.error("Checksum %s expected. Actual checksum is %s." %
 274                              (checksum, checksum_value))
 275                 return 1
 276
 277         # Make installer executable
 278         logger.info("Making installer executable")
 279         st = os.stat(tmpbuildtools)
 280         os.chmod(tmpbuildtools, st.st_mode | stat.S_IEXEC)
 281         logger.debug(os.stat(tmpbuildtools))
 282         if args.directory:
 283             install_dir = args.directory
 284             ret = subprocess.call("%s -d %s -y" %
 285                                   (tmpbuildtools, install_dir), shell=True)
 286         else:
 287             install_dir = "/opt/poky/%s" % args.installer_version
 288             ret = subprocess.call("%s -y" % tmpbuildtools, shell=True)
 289         if ret != 0:
 290             logger.error("Could not run buildtools installer")
 291             return ret
 292
 293         # Setup the environment
 294         logger.info("Setting up the environment")
 295         regex = re.compile(r'^(?P<export>export )?(?P<env_var>[A-Z_]+)=(?P<env_val>.+)$')
 296         with open("%s/environment-setup-%s-pokysdk-linux" %
 297                   (install_dir, arch), 'rb') as f:
 298             for line in f:
 299                 match = regex.search(line.decode('utf-8'))
 300                 logger.debug("export regex: %s" % match)
 301                 if match:
 302                     env_var = match.group('env_var')
 303                     logger.debug("env_var: %s" % env_var)
 304                     env_val = match.group('env_val')
 305                     logger.debug("env_val: %s" % env_val)
 306                     os.environ[env_var] = env_val
 307
 308         # Test installation
 309         logger.info("Testing installation")
 310         tool = ""
 311         m = re.search("extended", tmpbuildtools)
 312         logger.debug("extended regex: %s" % m)
 313         if args.with_extended_buildtools and not m:
 314             logger.info("Ignoring --with-extended-buildtools as filename "
 315                         "does not contain 'extended'")
 316         if args.make_only:
 317             tool = 'make'
 318         elif args.with_extended_buildtools and m:
 319             tool = 'gcc'
 320         else:
 321             tool = 'tar'
 322         logger.debug("install_dir: %s" % install_dir)
 323         cmd = shlex.split("/usr/bin/which %s" % tool)
 324         logger.debug("cmd: %s" % cmd)
 325         logger.debug("tool: %s" % tool)
 326         proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
 327         output, errors = proc.communicate()
 328         logger.debug("proc.args: %s" % proc.args)
 329         logger.debug("proc.communicate(): output %s" % output)
 330         logger.debug("proc.communicate(): errors %s" % errors)
 331         which_tool = output.decode('utf-8')
 332         logger.debug("which %s: %s" % (tool, which_tool))
 333         ret = proc.returncode
 334         if not which_tool.startswith(install_dir):
 335             logger.error("Something went wrong: %s not found in %s" %
 336                          (tool, install_dir))
 337         if ret != 0:
 338             logger.error("Something went wrong: installation failed")
 339         else:
 340             logger.info("Installation successful. Remember to source the "
 341                         "environment setup script now and in any new session.")
 342         return ret
 343
 344     finally:
 345         # cleanup tmp directory
 346         shutil.rmtree(tmpsdk_dir)
 347
 348
 349 if __name__ == '__main__':
 350     try:
 351         ret = main()
 352     except Exception:
 353         ret = 1
 354         import traceback
 355
 356         traceback.print_exc()
 357     sys.exit(ret)