tools/patman/gitutil.py

   1 # Copyright (c) 2011 The Chromium OS Authors.
   2 #
   3 # SPDX-License-Identifier:      GPL-2.0+
   4 #
   5
   6 import command
   7 import re
   8 import os
   9 import series
  10 import subprocess
  11 import sys
  12 import terminal
  13
  14 import checkpatch
  15 import settings
  16
  17
  18 def CountCommitsToBranch():
  19     """Returns number of commits between HEAD and the tracking branch.
  20
  21     This looks back to the tracking branch and works out the number of commits
  22     since then.
  23
  24     Return:
  25         Number of patches that exist on top of the branch
  26     """
  27     pipe = [['git', 'log', '--no-color', '--oneline', '--no-decorate',
  28              '@{upstream}..'],
  29             ['wc', '-l']]
  30     stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout
  31     patch_count = int(stdout)
  32     return patch_count
  33
  34 def GetUpstream(git_dir, branch):
  35     """Returns the name of the upstream for a branch
  36
  37     Args:
  38         git_dir: Git directory containing repo
  39         branch: Name of branch
  40
  41     Returns:
  42         Name of upstream branch (e.g. 'upstream/master') or None if none
  43     """
  44     try:
  45         remote = command.OutputOneLine('git', '--git-dir', git_dir, 'config',
  46                                        'branch.%s.remote' % branch)
  47         merge = command.OutputOneLine('git', '--git-dir', git_dir, 'config',
  48                                       'branch.%s.merge' % branch)
  49     except:
  50         return None
  51
  52     if remote == '.':
  53         return merge
  54     elif remote and merge:
  55         leaf = merge.split('/')[-1]
  56         return '%s/%s' % (remote, leaf)
  57     else:
  58         raise ValueError, ("Cannot determine upstream branch for branch "
  59                 "'%s' remote='%s', merge='%s'" % (branch, remote, merge))
  60
  61
  62 def GetRangeInBranch(git_dir, branch, include_upstream=False):
  63     """Returns an expression for the commits in the given branch.
  64
  65     Args:
  66         git_dir: Directory containing git repo
  67         branch: Name of branch
  68     Return:
  69         Expression in the form 'upstream..branch' which can be used to
  70         access the commits. If the branch does not exist, returns None.
  71     """
  72     upstream = GetUpstream(git_dir, branch)
  73     if not upstream:
  74         return None
  75     return '%s%s..%s' % (upstream, '~' if include_upstream else '', branch)
  76
  77 def CountCommitsInBranch(git_dir, branch, include_upstream=False):
  78     """Returns the number of commits in the given branch.
  79
  80     Args:
  81         git_dir: Directory containing git repo
  82         branch: Name of branch
  83     Return:
  84         Number of patches that exist on top of the branch, or None if the
  85         branch does not exist.
  86     """
  87     range_expr = GetRangeInBranch(git_dir, branch, include_upstream)
  88     if not range_expr:
  89         return None
  90     pipe = [['git', '--git-dir', git_dir, 'log', '--oneline', '--no-decorate',
  91              range_expr],
  92             ['wc', '-l']]
  93     result = command.RunPipe(pipe, capture=True, oneline=True)
  94     patch_count = int(result.stdout)
  95     return patch_count
  96
  97 def CountCommits(commit_range):
  98     """Returns the number of commits in the given range.
  99
 100     Args:
 101         commit_range: Range of commits to count (e.g. 'HEAD..base')
 102     Return:
 103         Number of patches that exist on top of the branch
 104     """
 105     pipe = [['git', 'log', '--oneline', '--no-decorate', commit_range],
 106             ['wc', '-l']]
 107     stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout
 108     patch_count = int(stdout)
 109     return patch_count
 110
 111 def Checkout(commit_hash, git_dir=None, work_tree=None, force=False):
 112     """Checkout the selected commit for this build
 113
 114     Args:
 115         commit_hash: Commit hash to check out
 116     """
 117     pipe = ['git']
 118     if git_dir:
 119         pipe.extend(['--git-dir', git_dir])
 120     if work_tree:
 121         pipe.extend(['--work-tree', work_tree])
 122     pipe.append('checkout')
 123     if force:
 124         pipe.append('-f')
 125     pipe.append(commit_hash)
 126     result = command.RunPipe([pipe], capture=True, raise_on_error=False)
 127     if result.return_code != 0:
 128         raise OSError, 'git checkout (%s): %s' % (pipe, result.stderr)
 129
 130 def Clone(git_dir, output_dir):
 131     """Checkout the selected commit for this build
 132
 133     Args:
 134         commit_hash: Commit hash to check out
 135     """
 136     pipe = ['git', 'clone', git_dir, '.']
 137     result = command.RunPipe([pipe], capture=True, cwd=output_dir)
 138     if result.return_code != 0:
 139         raise OSError, 'git clone: %s' % result.stderr
 140
 141 def Fetch(git_dir=None, work_tree=None):
 142     """Fetch from the origin repo
 143
 144     Args:
 145         commit_hash: Commit hash to check out
 146     """
 147     pipe = ['git']
 148     if git_dir:
 149         pipe.extend(['--git-dir', git_dir])
 150     if work_tree:
 151         pipe.extend(['--work-tree', work_tree])
 152     pipe.append('fetch')
 153     result = command.RunPipe([pipe], capture=True)
 154     if result.return_code != 0:
 155         raise OSError, 'git fetch: %s' % result.stderr
 156
 157 def CreatePatches(start, count, series):
 158     """Create a series of patches from the top of the current branch.
 159
 160     The patch files are written to the current directory using
 161     git format-patch.
 162
 163     Args:
 164         start: Commit to start from: 0=HEAD, 1=next one, etc.
 165         count: number of commits to include
 166     Return:
 167         Filename of cover letter
 168         List of filenames of patch files
 169     """
 170     if series.get('version'):
 171         version = '%s ' % series['version']
 172     cmd = ['git', 'format-patch', '-M', '--signoff']
 173     if series.get('cover'):
 174         cmd.append('--cover-letter')
 175     prefix = series.GetPatchPrefix()
 176     if prefix:
 177         cmd += ['--subject-prefix=%s' % prefix]
 178     cmd += ['HEAD~%d..HEAD~%d' % (start + count, start)]
 179
 180     stdout = command.RunList(cmd)
 181     files = stdout.splitlines()
 182
 183     # We have an extra file if there is a cover letter
 184     if series.get('cover'):
 185        return files[0], files[1:]
 186     else:
 187        return None, files
 188
 189 def ApplyPatch(verbose, fname):
 190     """Apply a patch with git am to test it
 191
 192     TODO: Convert these to use command, with stderr option
 193
 194     Args:
 195         fname: filename of patch file to apply
 196     """
 197     col = terminal.Color()
 198     cmd = ['git', 'am', fname]
 199     pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE,
 200             stderr=subprocess.PIPE)
 201     stdout, stderr = pipe.communicate()
 202     re_error = re.compile('^error: patch failed: (.+):(\d+)')
 203     for line in stderr.splitlines():
 204         if verbose:
 205             print line
 206         match = re_error.match(line)
 207         if match:
 208             print checkpatch.GetWarningMsg(col, 'warning', match.group(1),
 209                                            int(match.group(2)), 'Patch failed')
 210     return pipe.returncode == 0, stdout
 211
 212 def ApplyPatches(verbose, args, start_point):
 213     """Apply the patches with git am to make sure all is well
 214
 215     Args:
 216         verbose: Print out 'git am' output verbatim
 217         args: List of patch files to apply
 218         start_point: Number of commits back from HEAD to start applying.
 219             Normally this is len(args), but it can be larger if a start
 220             offset was given.
 221     """
 222     error_count = 0
 223     col = terminal.Color()
 224
 225     # Figure out our current position
 226     cmd = ['git', 'name-rev', 'HEAD', '--name-only']
 227     pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE)
 228     stdout, stderr = pipe.communicate()
 229     if pipe.returncode:
 230         str = 'Could not find current commit name'
 231         print col.Color(col.RED, str)
 232         print stdout
 233         return False
 234     old_head = stdout.splitlines()[0]
 235
 236     # Checkout the required start point
 237     cmd = ['git', 'checkout', 'HEAD~%d' % start_point]
 238     pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE,
 239             stderr=subprocess.PIPE)
 240     stdout, stderr = pipe.communicate()
 241     if pipe.returncode:
 242         str = 'Could not move to commit before patch series'
 243         print col.Color(col.RED, str)
 244         print stdout, stderr
 245         return False
 246
 247     # Apply all the patches
 248     for fname in args:
 249         ok, stdout = ApplyPatch(verbose, fname)
 250         if not ok:
 251             print col.Color(col.RED, 'git am returned errors for %s: will '
 252                     'skip this patch' % fname)
 253             if verbose:
 254                 print stdout
 255             error_count += 1
 256             cmd = ['git', 'am', '--skip']
 257             pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE)
 258             stdout, stderr = pipe.communicate()
 259             if pipe.returncode != 0:
 260                 print col.Color(col.RED, 'Unable to skip patch! Aborting...')
 261                 print stdout
 262                 break
 263
 264     # Return to our previous position
 265     cmd = ['git', 'checkout', old_head]
 266     pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 267     stdout, stderr = pipe.communicate()
 268     if pipe.returncode:
 269         print col.Color(col.RED, 'Could not move back to head commit')
 270         print stdout, stderr
 271     return error_count == 0
 272
 273 def BuildEmailList(in_list, tag=None, alias=None, raise_on_error=True):
 274     """Build a list of email addresses based on an input list.
 275
 276     Takes a list of email addresses and aliases, and turns this into a list
 277     of only email address, by resolving any aliases that are present.
 278
 279     If the tag is given, then each email address is prepended with this
 280     tag and a space. If the tag starts with a minus sign (indicating a
 281     command line parameter) then the email address is quoted.
 282
 283     Args:
 284         in_list:        List of aliases/email addresses
 285         tag:            Text to put before each address
 286         alias:          Alias dictionary
 287         raise_on_error: True to raise an error when an alias fails to match,
 288                 False to just print a message.
 289
 290     Returns:
 291         List of email addresses
 292
 293     >>> alias = {}
 294     >>> alias['fred'] = ['f.bloggs@napier.co.nz']
 295     >>> alias['john'] = ['j.bloggs@napier.co.nz']
 296     >>> alias['mary'] = ['Mary Poppins <m.poppins@cloud.net>']
 297     >>> alias['boys'] = ['fred', ' john']
 298     >>> alias['all'] = ['fred ', 'john', '   mary   ']
 299     >>> BuildEmailList(['john', 'mary'], None, alias)
 300     ['j.bloggs@napier.co.nz', 'Mary Poppins <m.poppins@cloud.net>']
 301     >>> BuildEmailList(['john', 'mary'], '--to', alias)
 302     ['--to "j.bloggs@napier.co.nz"', \
 303 '--to "Mary Poppins <m.poppins@cloud.net>"']
 304     >>> BuildEmailList(['john', 'mary'], 'Cc', alias)
 305     ['Cc j.bloggs@napier.co.nz', 'Cc Mary Poppins <m.poppins@cloud.net>']
 306     """
 307     quote = '"' if tag and tag[0] == '-' else ''
 308     raw = []
 309     for item in in_list:
 310         raw += LookupEmail(item, alias, raise_on_error=raise_on_error)
 311     result = []
 312     for item in raw:
 313         if not item in result:
 314             result.append(item)
 315     if tag:
 316         return ['%s %s%s%s' % (tag, quote, email, quote) for email in result]
 317     return result
 318
 319 def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname,
 320         self_only=False, alias=None, in_reply_to=None):
 321     """Email a patch series.
 322
 323     Args:
 324         series: Series object containing destination info
 325         cover_fname: filename of cover letter
 326         args: list of filenames of patch files
 327         dry_run: Just return the command that would be run
 328         raise_on_error: True to raise an error when an alias fails to match,
 329                 False to just print a message.
 330         cc_fname: Filename of Cc file for per-commit Cc
 331         self_only: True to just email to yourself as a test
 332         in_reply_to: If set we'll pass this to git as --in-reply-to.
 333             Should be a message ID that this is in reply to.
 334
 335     Returns:
 336         Git command that was/would be run
 337
 338     # For the duration of this doctest pretend that we ran patman with ./patman
 339     >>> _old_argv0 = sys.argv[0]
 340     >>> sys.argv[0] = './patman'
 341
 342     >>> alias = {}
 343     >>> alias['fred'] = ['f.bloggs@napier.co.nz']
 344     >>> alias['john'] = ['j.bloggs@napier.co.nz']
 345     >>> alias['mary'] = ['m.poppins@cloud.net']
 346     >>> alias['boys'] = ['fred', ' john']
 347     >>> alias['all'] = ['fred ', 'john', '   mary   ']
 348     >>> alias[os.getenv('USER')] = ['this-is-me@me.com']
 349     >>> series = series.Series()
 350     >>> series.to = ['fred']
 351     >>> series.cc = ['mary']
 352     >>> EmailPatches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \
 353             False, alias)
 354     'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \
 355 "m.poppins@cloud.net" --cc-cmd "./patman --cc-cmd cc-fname" cover p1 p2'
 356     >>> EmailPatches(series, None, ['p1'], True, True, 'cc-fname', False, \
 357             alias)
 358     'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \
 359 "m.poppins@cloud.net" --cc-cmd "./patman --cc-cmd cc-fname" p1'
 360     >>> series.cc = ['all']
 361     >>> EmailPatches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \
 362             True, alias)
 363     'git send-email --annotate --to "this-is-me@me.com" --cc-cmd "./patman \
 364 --cc-cmd cc-fname" cover p1 p2'
 365     >>> EmailPatches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \
 366             False, alias)
 367     'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \
 368 "f.bloggs@napier.co.nz" --cc "j.bloggs@napier.co.nz" --cc \
 369 "m.poppins@cloud.net" --cc-cmd "./patman --cc-cmd cc-fname" cover p1 p2'
 370
 371     # Restore argv[0] since we clobbered it.
 372     >>> sys.argv[0] = _old_argv0
 373     """
 374     to = BuildEmailList(series.get('to'), '--to', alias, raise_on_error)
 375     if not to:
 376         print ("No recipient, please add something like this to a commit\n"
 377             "Series-to: Fred Bloggs <f.blogs@napier.co.nz>")
 378         return
 379     cc = BuildEmailList(series.get('cc'), '--cc', alias, raise_on_error)
 380     if self_only:
 381         to = BuildEmailList([os.getenv('USER')], '--to', alias, raise_on_error)
 382         cc = []
 383     cmd = ['git', 'send-email', '--annotate']
 384     if in_reply_to:
 385         cmd.append('--in-reply-to="%s"' % in_reply_to)
 386
 387     cmd += to
 388     cmd += cc
 389     cmd += ['--cc-cmd', '"%s --cc-cmd %s"' % (sys.argv[0], cc_fname)]
 390     if cover_fname:
 391         cmd.append(cover_fname)
 392     cmd += args
 393     str = ' '.join(cmd)
 394     if not dry_run:
 395         os.system(str)
 396     return str
 397
 398
 399 def LookupEmail(lookup_name, alias=None, raise_on_error=True, level=0):
 400     """If an email address is an alias, look it up and return the full name
 401
 402     TODO: Why not just use git's own alias feature?
 403
 404     Args:
 405         lookup_name: Alias or email address to look up
 406         alias: Dictionary containing aliases (None to use settings default)
 407         raise_on_error: True to raise an error when an alias fails to match,
 408                 False to just print a message.
 409
 410     Returns:
 411         tuple:
 412             list containing a list of email addresses
 413
 414     Raises:
 415         OSError if a recursive alias reference was found
 416         ValueError if an alias was not found
 417
 418     >>> alias = {}
 419     >>> alias['fred'] = ['f.bloggs@napier.co.nz']
 420     >>> alias['john'] = ['j.bloggs@napier.co.nz']
 421     >>> alias['mary'] = ['m.poppins@cloud.net']
 422     >>> alias['boys'] = ['fred', ' john', 'f.bloggs@napier.co.nz']
 423     >>> alias['all'] = ['fred ', 'john', '   mary   ']
 424     >>> alias['loop'] = ['other', 'john', '   mary   ']
 425     >>> alias['other'] = ['loop', 'john', '   mary   ']
 426     >>> LookupEmail('mary', alias)
 427     ['m.poppins@cloud.net']
 428     >>> LookupEmail('arthur.wellesley@howe.ro.uk', alias)
 429     ['arthur.wellesley@howe.ro.uk']
 430     >>> LookupEmail('boys', alias)
 431     ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz']
 432     >>> LookupEmail('all', alias)
 433     ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz', 'm.poppins@cloud.net']
 434     >>> LookupEmail('odd', alias)
 435     Traceback (most recent call last):
 436     ...
 437     ValueError: Alias 'odd' not found
 438     >>> LookupEmail('loop', alias)
 439     Traceback (most recent call last):
 440     ...
 441     OSError: Recursive email alias at 'other'
 442     >>> LookupEmail('odd', alias, raise_on_error=False)
 443     \033[1;31mAlias 'odd' not found\033[0m
 444     []
 445     >>> # In this case the loop part will effectively be ignored.
 446     >>> LookupEmail('loop', alias, raise_on_error=False)
 447     \033[1;31mRecursive email alias at 'other'\033[0m
 448     \033[1;31mRecursive email alias at 'john'\033[0m
 449     \033[1;31mRecursive email alias at 'mary'\033[0m
 450     ['j.bloggs@napier.co.nz', 'm.poppins@cloud.net']
 451     """
 452     if not alias:
 453         alias = settings.alias
 454     lookup_name = lookup_name.strip()
 455     if '@' in lookup_name: # Perhaps a real email address
 456         return [lookup_name]
 457
 458     lookup_name = lookup_name.lower()
 459     col = terminal.Color()
 460
 461     out_list = []
 462     if level > 10:
 463         msg = "Recursive email alias at '%s'" % lookup_name
 464         if raise_on_error:
 465             raise OSError, msg
 466         else:
 467             print col.Color(col.RED, msg)
 468             return out_list
 469
 470     if lookup_name:
 471         if not lookup_name in alias:
 472             msg = "Alias '%s' not found" % lookup_name
 473             if raise_on_error:
 474                 raise ValueError, msg
 475             else:
 476                 print col.Color(col.RED, msg)
 477                 return out_list
 478         for item in alias[lookup_name]:
 479             todo = LookupEmail(item, alias, raise_on_error, level + 1)
 480             for new_item in todo:
 481                 if not new_item in out_list:
 482                     out_list.append(new_item)
 483
 484     #print "No match for alias '%s'" % lookup_name
 485     return out_list
 486
 487 def GetTopLevel():
 488     """Return name of top-level directory for this git repo.
 489
 490     Returns:
 491         Full path to git top-level directory
 492
 493     This test makes sure that we are running tests in the right subdir
 494
 495     >>> os.path.realpath(os.path.dirname(__file__)) == \
 496             os.path.join(GetTopLevel(), 'tools', 'patman')
 497     True
 498     """
 499     return command.OutputOneLine('git', 'rev-parse', '--show-toplevel')
 500
 501 def GetAliasFile():
 502     """Gets the name of the git alias file.
 503
 504     Returns:
 505         Filename of git alias file, or None if none
 506     """
 507     fname = command.OutputOneLine('git', 'config', 'sendemail.aliasesfile',
 508             raise_on_error=False)
 509     if fname:
 510         fname = os.path.join(GetTopLevel(), fname.strip())
 511     return fname
 512
 513 def GetDefaultUserName():
 514     """Gets the user.name from .gitconfig file.
 515
 516     Returns:
 517         User name found in .gitconfig file, or None if none
 518     """
 519     uname = command.OutputOneLine('git', 'config', '--global', 'user.name')
 520     return uname
 521
 522 def GetDefaultUserEmail():
 523     """Gets the user.email from the global .gitconfig file.
 524
 525     Returns:
 526         User's email found in .gitconfig file, or None if none
 527     """
 528     uemail = command.OutputOneLine('git', 'config', '--global', 'user.email')
 529     return uemail
 530
 531 def Setup():
 532     """Set up git utils, by reading the alias files."""
 533     # Check for a git alias file also
 534     alias_fname = GetAliasFile()
 535     if alias_fname:
 536         settings.ReadGitAliases(alias_fname)
 537
 538 def GetHead():
 539     """Get the hash of the current HEAD
 540
 541     Returns:
 542         Hash of HEAD
 543     """
 544     return command.OutputOneLine('git', 'show', '-s', '--pretty=format:%H')
 545
 546 if __name__ == "__main__":
 547     import doctest
 548
 549     doctest.testmod()