scripts/lib/devtool/deploy.py

   1 # Development tool - deploy/undeploy command plugin
   2 #
   3 # Copyright (C) 2014-2016 Intel Corporation
   4 #
   5 # SPDX-License-Identifier: GPL-2.0-only
   6 #
   7 """Devtool plugin containing the deploy subcommands"""
   8
   9 import logging
  10 import os
  11 import shutil
  12 import subprocess
  13 import tempfile
  14
  15 import bb.utils
  16 import argparse_oe
  17 import oe.types
  18
  19 from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError
  20
  21 logger = logging.getLogger('devtool')
  22
  23 deploylist_path = '/.devtool'
  24
  25 def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False):
  26     """
  27     Prepare a shell script for running on the target to
  28     deploy/undeploy files. We have to be careful what we put in this
  29     script - only commands that are likely to be available on the
  30     target are suitable (the target might be constrained, e.g. using
  31     busybox rather than bash with coreutils).
  32     """
  33     lines = []
  34     lines.append('#!/bin/sh')
  35     lines.append('set -e')
  36     if undeployall:
  37         # Yes, I know this is crude - but it does work
  38         lines.append('for entry in %s/*.list; do' % deploylist_path)
  39         lines.append('[ ! -f $entry ] && exit')
  40         lines.append('set `basename $entry | sed "s/.list//"`')
  41     if dryrun:
  42         if not deploy:
  43             lines.append('echo "Previously deployed files for $1:"')
  44     lines.append('manifest="%s/$1.list"' % deploylist_path)
  45     lines.append('preservedir="%s/$1.preserve"' % deploylist_path)
  46     lines.append('if [ -f $manifest ] ; then')
  47     # Read manifest in reverse and delete files / remove empty dirs
  48     lines.append('    sed \'1!G;h;$!d\' $manifest | while read file')
  49     lines.append('    do')
  50     if dryrun:
  51         lines.append('        if [ ! -d $file ] ; then')
  52         lines.append('            echo $file')
  53         lines.append('        fi')
  54     else:
  55         lines.append('        if [ -d $file ] ; then')
  56         # Avoid deleting a preserved directory in case it has special perms
  57         lines.append('            if [ ! -d $preservedir/$file ] ; then')
  58         lines.append('                rmdir $file > /dev/null 2>&1 || true')
  59         lines.append('            fi')
  60         lines.append('        else')
  61         lines.append('            rm -f $file')
  62         lines.append('        fi')
  63     lines.append('    done')
  64     if not dryrun:
  65         lines.append('    rm $manifest')
  66     if not deploy and not dryrun:
  67         # May as well remove all traces
  68         lines.append('    rmdir `dirname $manifest` > /dev/null 2>&1 || true')
  69     lines.append('fi')
  70
  71     if deploy:
  72         if not nocheckspace:
  73             # Check for available space
  74             # FIXME This doesn't take into account files spread across multiple
  75             # partitions, but doing that is non-trivial
  76             # Find the part of the destination path that exists
  77             lines.append('checkpath="$2"')
  78             lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]')
  79             lines.append('do')
  80             lines.append('    checkpath=`dirname "$checkpath"`')
  81             lines.append('done')
  82             lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")')
  83             # First line of the file is the total space
  84             lines.append('total=`head -n1 $3`')
  85             lines.append('if [ $total -gt $freespace ] ; then')
  86             lines.append('    echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"')
  87             lines.append('    exit 1')
  88             lines.append('fi')
  89         if not nopreserve:
  90             # Preserve any files that exist. Note that this will add to the
  91             # preserved list with successive deployments if the list of files
  92             # deployed changes, but because we've deleted any previously
  93             # deployed files at this point it will never preserve anything
  94             # that was deployed, only files that existed prior to any deploying
  95             # (which makes the most sense)
  96             lines.append('cat $3 | sed "1d" | while read file fsize')
  97             lines.append('do')
  98             lines.append('    if [ -e $file ] ; then')
  99             lines.append('    dest="$preservedir/$file"')
 100             lines.append('    mkdir -p `dirname $dest`')
 101             lines.append('    mv $file $dest')
 102             lines.append('    fi')
 103             lines.append('done')
 104             lines.append('rm $3')
 105         lines.append('mkdir -p `dirname $manifest`')
 106         lines.append('mkdir -p $2')
 107         if verbose:
 108             lines.append('    tar xv -C $2 -f - | tee $manifest')
 109         else:
 110             lines.append('    tar xv -C $2 -f - > $manifest')
 111         lines.append('sed -i "s!^./!$2!" $manifest')
 112     elif not dryrun:
 113         # Put any preserved files back
 114         lines.append('if [ -d $preservedir ] ; then')
 115         lines.append('    cd $preservedir')
 116         # find from busybox might not have -exec, so we don't use that
 117         lines.append('    find . -type f | while read file')
 118         lines.append('    do')
 119         lines.append('        mv $file /$file')
 120         lines.append('    done')
 121         lines.append('    cd /')
 122         lines.append('    rm -rf $preservedir')
 123         lines.append('fi')
 124
 125     if undeployall:
 126         if not dryrun:
 127             lines.append('echo "NOTE: Successfully undeployed $1"')
 128         lines.append('done')
 129
 130     # Delete the script itself
 131     lines.append('rm $0')
 132     lines.append('')
 133
 134     return '\n'.join(lines)
 135
 136
 137
 138 def deploy(args, config, basepath, workspace):
 139     """Entry point for the devtool 'deploy' subcommand"""
 140     import math
 141     import oe.recipeutils
 142     import oe.package
 143
 144     check_workspace_recipe(workspace, args.recipename, checksrc=False)
 145
 146     try:
 147         host, destdir = args.target.split(':')
 148     except ValueError:
 149         destdir = '/'
 150     else:
 151         args.target = host
 152     if not destdir.endswith('/'):
 153         destdir += '/'
 154
 155     tinfoil = setup_tinfoil(basepath=basepath)
 156     try:
 157         try:
 158             rd = tinfoil.parse_recipe(args.recipename)
 159         except Exception as e:
 160             raise DevtoolError('Exception parsing recipe %s: %s' %
 161                             (args.recipename, e))
 162         recipe_outdir = rd.getVar('D')
 163         if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
 164             raise DevtoolError('No files to deploy - have you built the %s '
 165                             'recipe? If so, the install step has not installed '
 166                             'any files.' % args.recipename)
 167
 168         if args.strip and not args.dry_run:
 169             # Fakeroot copy to new destination
 170             srcdir = recipe_outdir
 171             recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'devtool-deploy-target-stripped')
 172             if os.path.isdir(recipe_outdir):
 173                 bb.utils.remove(recipe_outdir, True)
 174             exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True)
 175             os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or ''])
 176             oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'),
 177                         rd.getVar('base_libdir'), rd)
 178
 179         filelist = []
 180         inodes = set({})
 181         ftotalsize = 0
 182         for root, _, files in os.walk(recipe_outdir):
 183             for fn in files:
 184                 fstat = os.lstat(os.path.join(root, fn))
 185                 # Get the size in kiB (since we'll be comparing it to the output of du -k)
 186                 # MUST use lstat() here not stat() or getfilesize() since we don't want to
 187                 # dereference symlinks
 188                 if fstat.st_ino in inodes:
 189                     fsize = 0
 190                 else:
 191                     fsize = int(math.ceil(float(fstat.st_size)/1024))
 192                 inodes.add(fstat.st_ino)
 193                 ftotalsize += fsize
 194                 # The path as it would appear on the target
 195                 fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)
 196                 filelist.append((fpath, fsize))
 197
 198         if args.dry_run:
 199             print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
 200             for item, _ in filelist:
 201                 print('  %s' % item)
 202             return 0
 203
 204         extraoptions = ''
 205         if args.no_host_check:
 206             extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
 207         if not args.show_status:
 208             extraoptions += ' -q'
 209
 210         scp_sshexec = ''
 211         ssh_sshexec = 'ssh'
 212         if args.ssh_exec:
 213             scp_sshexec = "-S %s" % args.ssh_exec
 214             ssh_sshexec = args.ssh_exec
 215         scp_port = ''
 216         ssh_port = ''
 217         if args.port:
 218             scp_port = "-P %s" % args.port
 219             ssh_port = "-p %s" % args.port
 220
 221         if args.key:
 222             extraoptions += ' -i %s' % args.key
 223
 224         # In order to delete previously deployed files and have the manifest file on
 225         # the target, we write out a shell script and then copy it to the target
 226         # so we can then run it (piping tar output to it).
 227         # (We cannot use scp here, because it doesn't preserve symlinks.)
 228         tmpdir = tempfile.mkdtemp(prefix='devtool')
 229         try:
 230             tmpscript = '/tmp/devtool_deploy.sh'
 231             tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list')
 232             shellscript = _prepare_remote_script(deploy=True,
 233                                                 verbose=args.show_status,
 234                                                 nopreserve=args.no_preserve,
 235                                                 nocheckspace=args.no_check_space)
 236             # Write out the script to a file
 237             with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
 238                 f.write(shellscript)
 239             # Write out the file list
 240             with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f:
 241                 f.write('%d\n' % ftotalsize)
 242                 for fpath, fsize in filelist:
 243                     f.write('%s %d\n' % (fpath, fsize))
 244             # Copy them to the target
 245             ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
 246             if ret != 0:
 247                 raise DevtoolError('Failed to copy script to %s - rerun with -s to '
 248                                 'get a complete error message' % args.target)
 249         finally:
 250             shutil.rmtree(tmpdir)
 251
 252         # Now run the script
 253         ret = exec_fakeroot(rd, 'tar cf - . | %s  %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True)
 254         if ret != 0:
 255             raise DevtoolError('Deploy failed - rerun with -s to get a complete '
 256                             'error message')
 257
 258         logger.info('Successfully deployed %s' % recipe_outdir)
 259
 260         files_list = []
 261         for root, _, files in os.walk(recipe_outdir):
 262             for filename in files:
 263                 filename = os.path.relpath(os.path.join(root, filename), recipe_outdir)
 264                 files_list.append(os.path.join(destdir, filename))
 265     finally:
 266         tinfoil.shutdown()
 267
 268     return 0
 269
 270 def undeploy(args, config, basepath, workspace):
 271     """Entry point for the devtool 'undeploy' subcommand"""
 272     if args.all and args.recipename:
 273         raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target')
 274     elif not args.recipename and not args.all:
 275         raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target')
 276
 277     extraoptions = ''
 278     if args.no_host_check:
 279         extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
 280     if not args.show_status:
 281         extraoptions += ' -q'
 282
 283     scp_sshexec = ''
 284     ssh_sshexec = 'ssh'
 285     if args.ssh_exec:
 286         scp_sshexec = "-S %s" % args.ssh_exec
 287         ssh_sshexec = args.ssh_exec
 288     scp_port = ''
 289     ssh_port = ''
 290     if args.port:
 291         scp_port = "-P %s" % args.port
 292         ssh_port = "-p %s" % args.port
 293
 294     args.target = args.target.split(':')[0]
 295
 296     tmpdir = tempfile.mkdtemp(prefix='devtool')
 297     try:
 298         tmpscript = '/tmp/devtool_undeploy.sh'
 299         shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all)
 300         # Write out the script to a file
 301         with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
 302             f.write(shellscript)
 303         # Copy it to the target
 304         ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
 305         if ret != 0:
 306             raise DevtoolError('Failed to copy script to %s - rerun with -s to '
 307                                 'get a complete error message' % args.target)
 308     finally:
 309         shutil.rmtree(tmpdir)
 310
 311     # Now run the script
 312     ret = subprocess.call('%s %s %s %s \'sh %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename), shell=True)
 313     if ret != 0:
 314         raise DevtoolError('Undeploy failed - rerun with -s to get a complete '
 315                            'error message')
 316
 317     if not args.all and not args.dry_run:
 318         logger.info('Successfully undeployed %s' % args.recipename)
 319     return 0
 320
 321
 322 def register_commands(subparsers, context):
 323     """Register devtool subcommands from the deploy plugin"""
 324
 325     parser_deploy = subparsers.add_parser('deploy-target',
 326                                           help='Deploy recipe output files to live target machine',
 327                                           description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.',
 328                                           group='testbuild')
 329     parser_deploy.add_argument('recipename', help='Recipe to deploy')
 330     parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
 331     parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
 332     parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
 333     parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true')
 334     parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true')
 335     parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true')
 336     parser_deploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh')
 337     parser_deploy.add_argument('-P', '--port', help='Specify port to use for connection to the target')
 338     parser_deploy.add_argument('-I', '--key',
 339                                help='Specify ssh private key for connection to the target')
 340
 341     strip_opts = parser_deploy.add_mutually_exclusive_group(required=False)
 342     strip_opts.add_argument('-S', '--strip',
 343                                help='Strip executables prior to deploying (default: %(default)s). '
 344                                     'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.',
 345                                default=oe.types.boolean(context.config.get('Deploy', 'strip', default='0')),
 346                                action='store_true')
 347     strip_opts.add_argument('--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false')
 348
 349     parser_deploy.set_defaults(func=deploy)
 350
 351     parser_undeploy = subparsers.add_parser('undeploy-target',
 352                                             help='Undeploy recipe output files in live target machine',
 353                                             description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.',
 354                                             group='testbuild')
 355     parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?')
 356     parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname')
 357     parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
 358     parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
 359     parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true')
 360     parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true')
 361     parser_undeploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh')
 362     parser_undeploy.add_argument('-P', '--port', help='Specify port to use for connection to the target')
 363     parser_undeploy.add_argument('-I', '--key',
 364                                help='Specify ssh private key for connection to the target')
 365
 366     parser_undeploy.set_defaults(func=undeploy)