tools/patman/gitutil.py

   1 # Copyright (c) 2011 The Chromium OS Authors.
   2 #
   3 # SPDX-License-Identifier:      GPL-2.0+
   4 #
   5
   6 import command
   7 import re
   8 import os
   9 import series
  10 import subprocess
  11 import sys
  12 import terminal
  13
  14 import settings
  15
  16
  17 def CountCommitsToBranch():
  18     """Returns number of commits between HEAD and the tracking branch.
  19
  20     This looks back to the tracking branch and works out the number of commits
  21     since then.
  22
  23     Return:
  24         Number of patches that exist on top of the branch
  25     """
  26     pipe = [['git', 'log', '--no-color', '--oneline', '--no-decorate',
  27              '@{upstream}..'],
  28             ['wc', '-l']]
  29     stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout
  30     patch_count = int(stdout)
  31     return patch_count
  32
  33 def GetUpstream(git_dir, branch):
  34     """Returns the name of the upstream for a branch
  35
  36     Args:
  37         git_dir: Git directory containing repo
  38         branch: Name of branch
  39
  40     Returns:
  41         Name of upstream branch (e.g. 'upstream/master') or None if none
  42     """
  43     try:
  44         remote = command.OutputOneLine('git', '--git-dir', git_dir, 'config',
  45                                        'branch.%s.remote' % branch)
  46         merge = command.OutputOneLine('git', '--git-dir', git_dir, 'config',
  47                                       'branch.%s.merge' % branch)
  48     except:
  49         return None
  50
  51     if remote == '.':
  52         return merge
  53     elif remote and merge:
  54         leaf = merge.split('/')[-1]
  55         return '%s/%s' % (remote, leaf)
  56     else:
  57         raise ValueError, ("Cannot determine upstream branch for branch "
  58                 "'%s' remote='%s', merge='%s'" % (branch, remote, merge))
  59
  60
  61 def GetRangeInBranch(git_dir, branch, include_upstream=False):
  62     """Returns an expression for the commits in the given branch.
  63
  64     Args:
  65         git_dir: Directory containing git repo
  66         branch: Name of branch
  67     Return:
  68         Expression in the form 'upstream..branch' which can be used to
  69         access the commits. If the branch does not exist, returns None.
  70     """
  71     upstream = GetUpstream(git_dir, branch)
  72     if not upstream:
  73         return None
  74     return '%s%s..%s' % (upstream, '~' if include_upstream else '', branch)
  75
  76 def CountCommitsInBranch(git_dir, branch, include_upstream=False):
  77     """Returns the number of commits in the given branch.
  78
  79     Args:
  80         git_dir: Directory containing git repo
  81         branch: Name of branch
  82     Return:
  83         Number of patches that exist on top of the branch, or None if the
  84         branch does not exist.
  85     """
  86     range_expr = GetRangeInBranch(git_dir, branch, include_upstream)
  87     if not range_expr:
  88         return None
  89     pipe = [['git', '--git-dir', git_dir, 'log', '--oneline', '--no-decorate',
  90              range_expr],
  91             ['wc', '-l']]
  92     result = command.RunPipe(pipe, capture=True, oneline=True)
  93     patch_count = int(result.stdout)
  94     return patch_count
  95
  96 def CountCommits(commit_range):
  97     """Returns the number of commits in the given range.
  98
  99     Args:
 100         commit_range: Range of commits to count (e.g. 'HEAD..base')
 101     Return:
 102         Number of patches that exist on top of the branch
 103     """
 104     pipe = [['git', 'log', '--oneline', '--no-decorate', commit_range],
 105             ['wc', '-l']]
 106     stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout
 107     patch_count = int(stdout)
 108     return patch_count
 109
 110 def Checkout(commit_hash, git_dir=None, work_tree=None, force=False):
 111     """Checkout the selected commit for this build
 112
 113     Args:
 114         commit_hash: Commit hash to check out
 115     """
 116     pipe = ['git']
 117     if git_dir:
 118         pipe.extend(['--git-dir', git_dir])
 119     if work_tree:
 120         pipe.extend(['--work-tree', work_tree])
 121     pipe.append('checkout')
 122     if force:
 123         pipe.append('-f')
 124     pipe.append(commit_hash)
 125     result = command.RunPipe([pipe], capture=True, raise_on_error=False)
 126     if result.return_code != 0:
 127         raise OSError, 'git checkout (%s): %s' % (pipe, result.stderr)
 128
 129 def Clone(git_dir, output_dir):
 130     """Checkout the selected commit for this build
 131
 132     Args:
 133         commit_hash: Commit hash to check out
 134     """
 135     pipe = ['git', 'clone', git_dir, '.']
 136     result = command.RunPipe([pipe], capture=True, cwd=output_dir)
 137     if result.return_code != 0:
 138         raise OSError, 'git clone: %s' % result.stderr
 139
 140 def Fetch(git_dir=None, work_tree=None):
 141     """Fetch from the origin repo
 142
 143     Args:
 144         commit_hash: Commit hash to check out
 145     """
 146     pipe = ['git']
 147     if git_dir:
 148         pipe.extend(['--git-dir', git_dir])
 149     if work_tree:
 150         pipe.extend(['--work-tree', work_tree])
 151     pipe.append('fetch')
 152     result = command.RunPipe([pipe], capture=True)
 153     if result.return_code != 0:
 154         raise OSError, 'git fetch: %s' % result.stderr
 155
 156 def CreatePatches(start, count, series):
 157     """Create a series of patches from the top of the current branch.
 158
 159     The patch files are written to the current directory using
 160     git format-patch.
 161
 162     Args:
 163         start: Commit to start from: 0=HEAD, 1=next one, etc.
 164         count: number of commits to include
 165     Return:
 166         Filename of cover letter
 167         List of filenames of patch files
 168     """
 169     if series.get('version'):
 170         version = '%s ' % series['version']
 171     cmd = ['git', 'format-patch', '-M', '--signoff']
 172     if series.get('cover'):
 173         cmd.append('--cover-letter')
 174     prefix = series.GetPatchPrefix()
 175     if prefix:
 176         cmd += ['--subject-prefix=%s' % prefix]
 177     cmd += ['HEAD~%d..HEAD~%d' % (start + count, start)]
 178
 179     stdout = command.RunList(cmd)
 180     files = stdout.splitlines()
 181
 182     # We have an extra file if there is a cover letter
 183     if series.get('cover'):
 184        return files[0], files[1:]
 185     else:
 186        return None, files
 187
 188 def ApplyPatch(verbose, fname):
 189     """Apply a patch with git am to test it
 190
 191     TODO: Convert these to use command, with stderr option
 192
 193     Args:
 194         fname: filename of patch file to apply
 195     """
 196     cmd = ['git', 'am', fname]
 197     pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE,
 198             stderr=subprocess.PIPE)
 199     stdout, stderr = pipe.communicate()
 200     re_error = re.compile('^error: patch failed: (.+):(\d+)')
 201     for line in stderr.splitlines():
 202         if verbose:
 203             print line
 204         match = re_error.match(line)
 205         if match:
 206             print GetWarningMsg('warning', match.group(1), int(match.group(2)),
 207                     'Patch failed')
 208     return pipe.returncode == 0, stdout
 209
 210 def ApplyPatches(verbose, args, start_point):
 211     """Apply the patches with git am to make sure all is well
 212
 213     Args:
 214         verbose: Print out 'git am' output verbatim
 215         args: List of patch files to apply
 216         start_point: Number of commits back from HEAD to start applying.
 217             Normally this is len(args), but it can be larger if a start
 218             offset was given.
 219     """
 220     error_count = 0
 221     col = terminal.Color()
 222
 223     # Figure out our current position
 224     cmd = ['git', 'name-rev', 'HEAD', '--name-only']
 225     pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE)
 226     stdout, stderr = pipe.communicate()
 227     if pipe.returncode:
 228         str = 'Could not find current commit name'
 229         print col.Color(col.RED, str)
 230         print stdout
 231         return False
 232     old_head = stdout.splitlines()[0]
 233
 234     # Checkout the required start point
 235     cmd = ['git', 'checkout', 'HEAD~%d' % start_point]
 236     pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE,
 237             stderr=subprocess.PIPE)
 238     stdout, stderr = pipe.communicate()
 239     if pipe.returncode:
 240         str = 'Could not move to commit before patch series'
 241         print col.Color(col.RED, str)
 242         print stdout, stderr
 243         return False
 244
 245     # Apply all the patches
 246     for fname in args:
 247         ok, stdout = ApplyPatch(verbose, fname)
 248         if not ok:
 249             print col.Color(col.RED, 'git am returned errors for %s: will '
 250                     'skip this patch' % fname)
 251             if verbose:
 252                 print stdout
 253             error_count += 1
 254             cmd = ['git', 'am', '--skip']
 255             pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE)
 256             stdout, stderr = pipe.communicate()
 257             if pipe.returncode != 0:
 258                 print col.Color(col.RED, 'Unable to skip patch! Aborting...')
 259                 print stdout
 260                 break
 261
 262     # Return to our previous position
 263     cmd = ['git', 'checkout', old_head]
 264     pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 265     stdout, stderr = pipe.communicate()
 266     if pipe.returncode:
 267         print col.Color(col.RED, 'Could not move back to head commit')
 268         print stdout, stderr
 269     return error_count == 0
 270
 271 def BuildEmailList(in_list, tag=None, alias=None, raise_on_error=True):
 272     """Build a list of email addresses based on an input list.
 273
 274     Takes a list of email addresses and aliases, and turns this into a list
 275     of only email address, by resolving any aliases that are present.
 276
 277     If the tag is given, then each email address is prepended with this
 278     tag and a space. If the tag starts with a minus sign (indicating a
 279     command line parameter) then the email address is quoted.
 280
 281     Args:
 282         in_list:        List of aliases/email addresses
 283         tag:            Text to put before each address
 284         alias:          Alias dictionary
 285         raise_on_error: True to raise an error when an alias fails to match,
 286                 False to just print a message.
 287
 288     Returns:
 289         List of email addresses
 290
 291     >>> alias = {}
 292     >>> alias['fred'] = ['f.bloggs@napier.co.nz']
 293     >>> alias['john'] = ['j.bloggs@napier.co.nz']
 294     >>> alias['mary'] = ['Mary Poppins <m.poppins@cloud.net>']
 295     >>> alias['boys'] = ['fred', ' john']
 296     >>> alias['all'] = ['fred ', 'john', '   mary   ']
 297     >>> BuildEmailList(['john', 'mary'], None, alias)
 298     ['j.bloggs@napier.co.nz', 'Mary Poppins <m.poppins@cloud.net>']
 299     >>> BuildEmailList(['john', 'mary'], '--to', alias)
 300     ['--to "j.bloggs@napier.co.nz"', \
 301 '--to "Mary Poppins <m.poppins@cloud.net>"']
 302     >>> BuildEmailList(['john', 'mary'], 'Cc', alias)
 303     ['Cc j.bloggs@napier.co.nz', 'Cc Mary Poppins <m.poppins@cloud.net>']
 304     """
 305     quote = '"' if tag and tag[0] == '-' else ''
 306     raw = []
 307     for item in in_list:
 308         raw += LookupEmail(item, alias, raise_on_error=raise_on_error)
 309     result = []
 310     for item in raw:
 311         if not item in result:
 312             result.append(item)
 313     if tag:
 314         return ['%s %s%s%s' % (tag, quote, email, quote) for email in result]
 315     return result
 316
 317 def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname,
 318         self_only=False, alias=None, in_reply_to=None):
 319     """Email a patch series.
 320
 321     Args:
 322         series: Series object containing destination info
 323         cover_fname: filename of cover letter
 324         args: list of filenames of patch files
 325         dry_run: Just return the command that would be run
 326         raise_on_error: True to raise an error when an alias fails to match,
 327                 False to just print a message.
 328         cc_fname: Filename of Cc file for per-commit Cc
 329         self_only: True to just email to yourself as a test
 330         in_reply_to: If set we'll pass this to git as --in-reply-to.
 331             Should be a message ID that this is in reply to.
 332
 333     Returns:
 334         Git command that was/would be run
 335
 336     # For the duration of this doctest pretend that we ran patman with ./patman
 337     >>> _old_argv0 = sys.argv[0]
 338     >>> sys.argv[0] = './patman'
 339
 340     >>> alias = {}
 341     >>> alias['fred'] = ['f.bloggs@napier.co.nz']
 342     >>> alias['john'] = ['j.bloggs@napier.co.nz']
 343     >>> alias['mary'] = ['m.poppins@cloud.net']
 344     >>> alias['boys'] = ['fred', ' john']
 345     >>> alias['all'] = ['fred ', 'john', '   mary   ']
 346     >>> alias[os.getenv('USER')] = ['this-is-me@me.com']
 347     >>> series = series.Series()
 348     >>> series.to = ['fred']
 349     >>> series.cc = ['mary']
 350     >>> EmailPatches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \
 351             False, alias)
 352     'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \
 353 "m.poppins@cloud.net" --cc-cmd "./patman --cc-cmd cc-fname" cover p1 p2'
 354     >>> EmailPatches(series, None, ['p1'], True, True, 'cc-fname', False, \
 355             alias)
 356     'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \
 357 "m.poppins@cloud.net" --cc-cmd "./patman --cc-cmd cc-fname" p1'
 358     >>> series.cc = ['all']
 359     >>> EmailPatches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \
 360             True, alias)
 361     'git send-email --annotate --to "this-is-me@me.com" --cc-cmd "./patman \
 362 --cc-cmd cc-fname" cover p1 p2'
 363     >>> EmailPatches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \
 364             False, alias)
 365     'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \
 366 "f.bloggs@napier.co.nz" --cc "j.bloggs@napier.co.nz" --cc \
 367 "m.poppins@cloud.net" --cc-cmd "./patman --cc-cmd cc-fname" cover p1 p2'
 368
 369     # Restore argv[0] since we clobbered it.
 370     >>> sys.argv[0] = _old_argv0
 371     """
 372     to = BuildEmailList(series.get('to'), '--to', alias, raise_on_error)
 373     if not to:
 374         print ("No recipient, please add something like this to a commit\n"
 375             "Series-to: Fred Bloggs <f.blogs@napier.co.nz>")
 376         return
 377     cc = BuildEmailList(series.get('cc'), '--cc', alias, raise_on_error)
 378     if self_only:
 379         to = BuildEmailList([os.getenv('USER')], '--to', alias, raise_on_error)
 380         cc = []
 381     cmd = ['git', 'send-email', '--annotate']
 382     if in_reply_to:
 383         cmd.append('--in-reply-to="%s"' % in_reply_to)
 384
 385     cmd += to
 386     cmd += cc
 387     cmd += ['--cc-cmd', '"%s --cc-cmd %s"' % (sys.argv[0], cc_fname)]
 388     if cover_fname:
 389         cmd.append(cover_fname)
 390     cmd += args
 391     str = ' '.join(cmd)
 392     if not dry_run:
 393         os.system(str)
 394     return str
 395
 396
 397 def LookupEmail(lookup_name, alias=None, raise_on_error=True, level=0):
 398     """If an email address is an alias, look it up and return the full name
 399
 400     TODO: Why not just use git's own alias feature?
 401
 402     Args:
 403         lookup_name: Alias or email address to look up
 404         alias: Dictionary containing aliases (None to use settings default)
 405         raise_on_error: True to raise an error when an alias fails to match,
 406                 False to just print a message.
 407
 408     Returns:
 409         tuple:
 410             list containing a list of email addresses
 411
 412     Raises:
 413         OSError if a recursive alias reference was found
 414         ValueError if an alias was not found
 415
 416     >>> alias = {}
 417     >>> alias['fred'] = ['f.bloggs@napier.co.nz']
 418     >>> alias['john'] = ['j.bloggs@napier.co.nz']
 419     >>> alias['mary'] = ['m.poppins@cloud.net']
 420     >>> alias['boys'] = ['fred', ' john', 'f.bloggs@napier.co.nz']
 421     >>> alias['all'] = ['fred ', 'john', '   mary   ']
 422     >>> alias['loop'] = ['other', 'john', '   mary   ']
 423     >>> alias['other'] = ['loop', 'john', '   mary   ']
 424     >>> LookupEmail('mary', alias)
 425     ['m.poppins@cloud.net']
 426     >>> LookupEmail('arthur.wellesley@howe.ro.uk', alias)
 427     ['arthur.wellesley@howe.ro.uk']
 428     >>> LookupEmail('boys', alias)
 429     ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz']
 430     >>> LookupEmail('all', alias)
 431     ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz', 'm.poppins@cloud.net']
 432     >>> LookupEmail('odd', alias)
 433     Traceback (most recent call last):
 434     ...
 435     ValueError: Alias 'odd' not found
 436     >>> LookupEmail('loop', alias)
 437     Traceback (most recent call last):
 438     ...
 439     OSError: Recursive email alias at 'other'
 440     >>> LookupEmail('odd', alias, raise_on_error=False)
 441     \033[1;31mAlias 'odd' not found\033[0m
 442     []
 443     >>> # In this case the loop part will effectively be ignored.
 444     >>> LookupEmail('loop', alias, raise_on_error=False)
 445     \033[1;31mRecursive email alias at 'other'\033[0m
 446     \033[1;31mRecursive email alias at 'john'\033[0m
 447     \033[1;31mRecursive email alias at 'mary'\033[0m
 448     ['j.bloggs@napier.co.nz', 'm.poppins@cloud.net']
 449     """
 450     if not alias:
 451         alias = settings.alias
 452     lookup_name = lookup_name.strip()
 453     if '@' in lookup_name: # Perhaps a real email address
 454         return [lookup_name]
 455
 456     lookup_name = lookup_name.lower()
 457     col = terminal.Color()
 458
 459     out_list = []
 460     if level > 10:
 461         msg = "Recursive email alias at '%s'" % lookup_name
 462         if raise_on_error:
 463             raise OSError, msg
 464         else:
 465             print col.Color(col.RED, msg)
 466             return out_list
 467
 468     if lookup_name:
 469         if not lookup_name in alias:
 470             msg = "Alias '%s' not found" % lookup_name
 471             if raise_on_error:
 472                 raise ValueError, msg
 473             else:
 474                 print col.Color(col.RED, msg)
 475                 return out_list
 476         for item in alias[lookup_name]:
 477             todo = LookupEmail(item, alias, raise_on_error, level + 1)
 478             for new_item in todo:
 479                 if not new_item in out_list:
 480                     out_list.append(new_item)
 481
 482     #print "No match for alias '%s'" % lookup_name
 483     return out_list
 484
 485 def GetTopLevel():
 486     """Return name of top-level directory for this git repo.
 487
 488     Returns:
 489         Full path to git top-level directory
 490
 491     This test makes sure that we are running tests in the right subdir
 492
 493     >>> os.path.realpath(os.path.dirname(__file__)) == \
 494             os.path.join(GetTopLevel(), 'tools', 'patman')
 495     True
 496     """
 497     return command.OutputOneLine('git', 'rev-parse', '--show-toplevel')
 498
 499 def GetAliasFile():
 500     """Gets the name of the git alias file.
 501
 502     Returns:
 503         Filename of git alias file, or None if none
 504     """
 505     fname = command.OutputOneLine('git', 'config', 'sendemail.aliasesfile',
 506             raise_on_error=False)
 507     if fname:
 508         fname = os.path.join(GetTopLevel(), fname.strip())
 509     return fname
 510
 511 def GetDefaultUserName():
 512     """Gets the user.name from .gitconfig file.
 513
 514     Returns:
 515         User name found in .gitconfig file, or None if none
 516     """
 517     uname = command.OutputOneLine('git', 'config', '--global', 'user.name')
 518     return uname
 519
 520 def GetDefaultUserEmail():
 521     """Gets the user.email from the global .gitconfig file.
 522
 523     Returns:
 524         User's email found in .gitconfig file, or None if none
 525     """
 526     uemail = command.OutputOneLine('git', 'config', '--global', 'user.email')
 527     return uemail
 528
 529 def Setup():
 530     """Set up git utils, by reading the alias files."""
 531     # Check for a git alias file also
 532     alias_fname = GetAliasFile()
 533     if alias_fname:
 534         settings.ReadGitAliases(alias_fname)
 535
 536 def GetHead():
 537     """Get the hash of the current HEAD
 538
 539     Returns:
 540         Hash of HEAD
 541     """
 542     return command.OutputOneLine('git', 'show', '-s', '--pretty=format:%H')
 543
 544 if __name__ == "__main__":
 545     import doctest
 546
 547     doctest.testmod()