contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = ["p4"]
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += ["-u",user]
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += ["-P", password]
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += ["-p", port]
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += ["-h", host]
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += ["-c", client]
  46
  47
  48     if isinstance(cmd,basestring):
  49         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  50     else:
  51         real_cmd += cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     # P4 uses the PWD environment variable rather than getcwd(). Since we're
  56     # not using the shell, we have to set it ourselves.  This path could
  57     # be relative, so go there first, then figure out where we ended up.
  58     os.chdir(dir)
  59     os.environ['PWD'] = os.getcwd()
  60
  61 def die(msg):
  62     if verbose:
  63         raise Exception(msg)
  64     else:
  65         sys.stderr.write(msg + "\n")
  66         sys.exit(1)
  67
  68 def write_pipe(c, stdin):
  69     if verbose:
  70         sys.stderr.write('Writing pipe: %s\n' % str(c))
  71
  72     expand = isinstance(c,basestring)
  73     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  74     pipe = p.stdin
  75     val = pipe.write(stdin)
  76     pipe.close()
  77     if p.wait():
  78         die('Command failed: %s' % str(c))
  79
  80     return val
  81
  82 def p4_write_pipe(c, stdin):
  83     real_cmd = p4_build_cmd(c)
  84     return write_pipe(real_cmd, stdin)
  85
  86 def read_pipe(c, ignore_error=False):
  87     if verbose:
  88         sys.stderr.write('Reading pipe: %s\n' % str(c))
  89
  90     expand = isinstance(c,basestring)
  91     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  92     pipe = p.stdout
  93     val = pipe.read()
  94     if p.wait() and not ignore_error:
  95         die('Command failed: %s' % str(c))
  96
  97     return val
  98
  99 def p4_read_pipe(c, ignore_error=False):
 100     real_cmd = p4_build_cmd(c)
 101     return read_pipe(real_cmd, ignore_error)
 102
 103 def read_pipe_lines(c):
 104     if verbose:
 105         sys.stderr.write('Reading pipe: %s\n' % str(c))
 106
 107     expand = isinstance(c, basestring)
 108     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 109     pipe = p.stdout
 110     val = pipe.readlines()
 111     if pipe.close() or p.wait():
 112         die('Command failed: %s' % str(c))
 113
 114     return val
 115
 116 def p4_read_pipe_lines(c):
 117     """Specifically invoke p4 on the command supplied. """
 118     real_cmd = p4_build_cmd(c)
 119     return read_pipe_lines(real_cmd)
 120
 121 def system(cmd):
 122     expand = isinstance(cmd,basestring)
 123     if verbose:
 124         sys.stderr.write("executing %s\n" % str(cmd))
 125     subprocess.check_call(cmd, shell=expand)
 126
 127 def p4_system(cmd):
 128     """Specifically invoke p4 as the system command. """
 129     real_cmd = p4_build_cmd(cmd)
 130     expand = isinstance(real_cmd, basestring)
 131     subprocess.check_call(real_cmd, shell=expand)
 132
 133 def p4_integrate(src, dest):
 134     p4_system(["integrate", "-Dt", src, dest])
 135
 136 def p4_sync(path):
 137     p4_system(["sync", path])
 138
 139 def p4_add(f):
 140     p4_system(["add", f])
 141
 142 def p4_delete(f):
 143     p4_system(["delete", f])
 144
 145 def p4_edit(f):
 146     p4_system(["edit", f])
 147
 148 def p4_revert(f):
 149     p4_system(["revert", f])
 150
 151 def p4_reopen(type, file):
 152     p4_system(["reopen", "-t", type, file])
 153
 154 #
 155 # Canonicalize the p4 type and return a tuple of the
 156 # base type, plus any modifiers.  See "p4 help filetypes"
 157 # for a list and explanation.
 158 #
 159 def split_p4_type(p4type):
 160
 161     p4_filetypes_historical = {
 162         "ctempobj": "binary+Sw",
 163         "ctext": "text+C",
 164         "cxtext": "text+Cx",
 165         "ktext": "text+k",
 166         "kxtext": "text+kx",
 167         "ltext": "text+F",
 168         "tempobj": "binary+FSw",
 169         "ubinary": "binary+F",
 170         "uresource": "resource+F",
 171         "uxbinary": "binary+Fx",
 172         "xbinary": "binary+x",
 173         "xltext": "text+Fx",
 174         "xtempobj": "binary+Swx",
 175         "xtext": "text+x",
 176         "xunicode": "unicode+x",
 177         "xutf16": "utf16+x",
 178     }
 179     if p4type in p4_filetypes_historical:
 180         p4type = p4_filetypes_historical[p4type]
 181     mods = ""
 182     s = p4type.split("+")
 183     base = s[0]
 184     mods = ""
 185     if len(s) > 1:
 186         mods = s[1]
 187     return (base, mods)
 188
 189
 190 def setP4ExecBit(file, mode):
 191     # Reopens an already open file and changes the execute bit to match
 192     # the execute bit setting in the passed in mode.
 193
 194     p4Type = "+x"
 195
 196     if not isModeExec(mode):
 197         p4Type = getP4OpenedType(file)
 198         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 199         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 200         if p4Type[-1] == "+":
 201             p4Type = p4Type[0:-1]
 202
 203     p4_reopen(p4Type, file)
 204
 205 def getP4OpenedType(file):
 206     # Returns the perforce file type for the given file.
 207
 208     result = p4_read_pipe(["opened", file])
 209     match = re.match(".*\((.+)\)\r?$", result)
 210     if match:
 211         return match.group(1)
 212     else:
 213         die("Could not determine file type for %s (result: '%s')" % (file, result))
 214
 215 def diffTreePattern():
 216     # This is a simple generator for the diff tree regex pattern. This could be
 217     # a class variable if this and parseDiffTreeEntry were a part of a class.
 218     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 219     while True:
 220         yield pattern
 221
 222 def parseDiffTreeEntry(entry):
 223     """Parses a single diff tree entry into its component elements.
 224
 225     See git-diff-tree(1) manpage for details about the format of the diff
 226     output. This method returns a dictionary with the following elements:
 227
 228     src_mode - The mode of the source file
 229     dst_mode - The mode of the destination file
 230     src_sha1 - The sha1 for the source file
 231     dst_sha1 - The sha1 fr the destination file
 232     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 233     status_score - The score for the status (applicable for 'C' and 'R'
 234                    statuses). This is None if there is no score.
 235     src - The path for the source file.
 236     dst - The path for the destination file. This is only present for
 237           copy or renames. If it is not present, this is None.
 238
 239     If the pattern is not matched, None is returned."""
 240
 241     match = diffTreePattern().next().match(entry)
 242     if match:
 243         return {
 244             'src_mode': match.group(1),
 245             'dst_mode': match.group(2),
 246             'src_sha1': match.group(3),
 247             'dst_sha1': match.group(4),
 248             'status': match.group(5),
 249             'status_score': match.group(6),
 250             'src': match.group(7),
 251             'dst': match.group(10)
 252         }
 253     return None
 254
 255 def isModeExec(mode):
 256     # Returns True if the given git mode represents an executable file,
 257     # otherwise False.
 258     return mode[-3:] == "755"
 259
 260 def isModeExecChanged(src_mode, dst_mode):
 261     return isModeExec(src_mode) != isModeExec(dst_mode)
 262
 263 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 264
 265     if isinstance(cmd,basestring):
 266         cmd = "-G " + cmd
 267         expand = True
 268     else:
 269         cmd = ["-G"] + cmd
 270         expand = False
 271
 272     cmd = p4_build_cmd(cmd)
 273     if verbose:
 274         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 275
 276     # Use a temporary file to avoid deadlocks without
 277     # subprocess.communicate(), which would put another copy
 278     # of stdout into memory.
 279     stdin_file = None
 280     if stdin is not None:
 281         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 282         if isinstance(stdin,basestring):
 283             stdin_file.write(stdin)
 284         else:
 285             for i in stdin:
 286                 stdin_file.write(i + '\n')
 287         stdin_file.flush()
 288         stdin_file.seek(0)
 289
 290     p4 = subprocess.Popen(cmd,
 291                           shell=expand,
 292                           stdin=stdin_file,
 293                           stdout=subprocess.PIPE)
 294
 295     result = []
 296     try:
 297         while True:
 298             entry = marshal.load(p4.stdout)
 299             if cb is not None:
 300                 cb(entry)
 301             else:
 302                 result.append(entry)
 303     except EOFError:
 304         pass
 305     exitCode = p4.wait()
 306     if exitCode != 0:
 307         entry = {}
 308         entry["p4ExitCode"] = exitCode
 309         result.append(entry)
 310
 311     return result
 312
 313 def p4Cmd(cmd):
 314     list = p4CmdList(cmd)
 315     result = {}
 316     for entry in list:
 317         result.update(entry)
 318     return result;
 319
 320 def p4Where(depotPath):
 321     if not depotPath.endswith("/"):
 322         depotPath += "/"
 323     depotPath = depotPath + "..."
 324     outputList = p4CmdList(["where", depotPath])
 325     output = None
 326     for entry in outputList:
 327         if "depotFile" in entry:
 328             if entry["depotFile"] == depotPath:
 329                 output = entry
 330                 break
 331         elif "data" in entry:
 332             data = entry.get("data")
 333             space = data.find(" ")
 334             if data[:space] == depotPath:
 335                 output = entry
 336                 break
 337     if output == None:
 338         return ""
 339     if output["code"] == "error":
 340         return ""
 341     clientPath = ""
 342     if "path" in output:
 343         clientPath = output.get("path")
 344     elif "data" in output:
 345         data = output.get("data")
 346         lastSpace = data.rfind(" ")
 347         clientPath = data[lastSpace + 1:]
 348
 349     if clientPath.endswith("..."):
 350         clientPath = clientPath[:-3]
 351     return clientPath
 352
 353 def currentGitBranch():
 354     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 355
 356 def isValidGitDir(path):
 357     if (os.path.exists(path + "/HEAD")
 358         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 359         return True;
 360     return False
 361
 362 def parseRevision(ref):
 363     return read_pipe("git rev-parse %s" % ref).strip()
 364
 365 def branchExists(ref):
 366     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 367                      ignore_error=True)
 368     return len(rev) > 0
 369
 370 def extractLogMessageFromGitCommit(commit):
 371     logMessage = ""
 372
 373     ## fixme: title is first line of commit, not 1st paragraph.
 374     foundTitle = False
 375     for log in read_pipe_lines("git cat-file commit %s" % commit):
 376        if not foundTitle:
 377            if len(log) == 1:
 378                foundTitle = True
 379            continue
 380
 381        logMessage += log
 382     return logMessage
 383
 384 def extractSettingsGitLog(log):
 385     values = {}
 386     for line in log.split("\n"):
 387         line = line.strip()
 388         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 389         if not m:
 390             continue
 391
 392         assignments = m.group(1).split (':')
 393         for a in assignments:
 394             vals = a.split ('=')
 395             key = vals[0].strip()
 396             val = ('='.join (vals[1:])).strip()
 397             if val.endswith ('\"') and val.startswith('"'):
 398                 val = val[1:-1]
 399
 400             values[key] = val
 401
 402     paths = values.get("depot-paths")
 403     if not paths:
 404         paths = values.get("depot-path")
 405     if paths:
 406         values['depot-paths'] = paths.split(',')
 407     return values
 408
 409 def gitBranchExists(branch):
 410     proc = subprocess.Popen(["git", "rev-parse", branch],
 411                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 412     return proc.wait() == 0;
 413
 414 _gitConfig = {}
 415 def gitConfig(key, args = None): # set args to "--bool", for instance
 416     if not _gitConfig.has_key(key):
 417         argsFilter = ""
 418         if args != None:
 419             argsFilter = "%s " % args
 420         cmd = "git config %s%s" % (argsFilter, key)
 421         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 422     return _gitConfig[key]
 423
 424 def gitConfigList(key):
 425     if not _gitConfig.has_key(key):
 426         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 427     return _gitConfig[key]
 428
 429 def p4BranchesInGit(branchesAreInRemotes = True):
 430     branches = {}
 431
 432     cmdline = "git rev-parse --symbolic "
 433     if branchesAreInRemotes:
 434         cmdline += " --remotes"
 435     else:
 436         cmdline += " --branches"
 437
 438     for line in read_pipe_lines(cmdline):
 439         line = line.strip()
 440
 441         ## only import to p4/
 442         if not line.startswith('p4/') or line == "p4/HEAD":
 443             continue
 444         branch = line
 445
 446         # strip off p4
 447         branch = re.sub ("^p4/", "", line)
 448
 449         branches[branch] = parseRevision(line)
 450     return branches
 451
 452 def findUpstreamBranchPoint(head = "HEAD"):
 453     branches = p4BranchesInGit()
 454     # map from depot-path to branch name
 455     branchByDepotPath = {}
 456     for branch in branches.keys():
 457         tip = branches[branch]
 458         log = extractLogMessageFromGitCommit(tip)
 459         settings = extractSettingsGitLog(log)
 460         if settings.has_key("depot-paths"):
 461             paths = ",".join(settings["depot-paths"])
 462             branchByDepotPath[paths] = "remotes/p4/" + branch
 463
 464     settings = None
 465     parent = 0
 466     while parent < 65535:
 467         commit = head + "~%s" % parent
 468         log = extractLogMessageFromGitCommit(commit)
 469         settings = extractSettingsGitLog(log)
 470         if settings.has_key("depot-paths"):
 471             paths = ",".join(settings["depot-paths"])
 472             if branchByDepotPath.has_key(paths):
 473                 return [branchByDepotPath[paths], settings]
 474
 475         parent = parent + 1
 476
 477     return ["", settings]
 478
 479 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 480     if not silent:
 481         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 482                % localRefPrefix)
 483
 484     originPrefix = "origin/p4/"
 485
 486     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 487         line = line.strip()
 488         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 489             continue
 490
 491         headName = line[len(originPrefix):]
 492         remoteHead = localRefPrefix + headName
 493         originHead = line
 494
 495         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 496         if (not original.has_key('depot-paths')
 497             or not original.has_key('change')):
 498             continue
 499
 500         update = False
 501         if not gitBranchExists(remoteHead):
 502             if verbose:
 503                 print "creating %s" % remoteHead
 504             update = True
 505         else:
 506             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 507             if settings.has_key('change') > 0:
 508                 if settings['depot-paths'] == original['depot-paths']:
 509                     originP4Change = int(original['change'])
 510                     p4Change = int(settings['change'])
 511                     if originP4Change > p4Change:
 512                         print ("%s (%s) is newer than %s (%s). "
 513                                "Updating p4 branch from origin."
 514                                % (originHead, originP4Change,
 515                                   remoteHead, p4Change))
 516                         update = True
 517                 else:
 518                     print ("Ignoring: %s was imported from %s while "
 519                            "%s was imported from %s"
 520                            % (originHead, ','.join(original['depot-paths']),
 521                               remoteHead, ','.join(settings['depot-paths'])))
 522
 523         if update:
 524             system("git update-ref %s %s" % (remoteHead, originHead))
 525
 526 def originP4BranchesExist():
 527         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 528
 529 def p4ChangesForPaths(depotPaths, changeRange):
 530     assert depotPaths
 531     cmd = ['changes']
 532     for p in depotPaths:
 533         cmd += ["%s...%s" % (p, changeRange)]
 534     output = p4_read_pipe_lines(cmd)
 535
 536     changes = {}
 537     for line in output:
 538         changeNum = int(line.split(" ")[1])
 539         changes[changeNum] = True
 540
 541     changelist = changes.keys()
 542     changelist.sort()
 543     return changelist
 544
 545 def p4PathStartsWith(path, prefix):
 546     # This method tries to remedy a potential mixed-case issue:
 547     #
 548     # If UserA adds  //depot/DirA/file1
 549     # and UserB adds //depot/dira/file2
 550     #
 551     # we may or may not have a problem. If you have core.ignorecase=true,
 552     # we treat DirA and dira as the same directory
 553     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 554     if ignorecase:
 555         return path.lower().startswith(prefix.lower())
 556     return path.startswith(prefix)
 557
 558 class Command:
 559     def __init__(self):
 560         self.usage = "usage: %prog [options]"
 561         self.needsGit = True
 562
 563 class P4UserMap:
 564     def __init__(self):
 565         self.userMapFromPerforceServer = False
 566         self.myP4UserId = None
 567
 568     def p4UserId(self):
 569         if self.myP4UserId:
 570             return self.myP4UserId
 571
 572         results = p4CmdList("user -o")
 573         for r in results:
 574             if r.has_key('User'):
 575                 self.myP4UserId = r['User']
 576                 return r['User']
 577         die("Could not find your p4 user id")
 578
 579     def p4UserIsMe(self, p4User):
 580         # return True if the given p4 user is actually me
 581         me = self.p4UserId()
 582         if not p4User or p4User != me:
 583             return False
 584         else:
 585             return True
 586
 587     def getUserCacheFilename(self):
 588         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 589         return home + "/.gitp4-usercache.txt"
 590
 591     def getUserMapFromPerforceServer(self):
 592         if self.userMapFromPerforceServer:
 593             return
 594         self.users = {}
 595         self.emails = {}
 596
 597         for output in p4CmdList("users"):
 598             if not output.has_key("User"):
 599                 continue
 600             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 601             self.emails[output["Email"]] = output["User"]
 602
 603
 604         s = ''
 605         for (key, val) in self.users.items():
 606             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 607
 608         open(self.getUserCacheFilename(), "wb").write(s)
 609         self.userMapFromPerforceServer = True
 610
 611     def loadUserMapFromCache(self):
 612         self.users = {}
 613         self.userMapFromPerforceServer = False
 614         try:
 615             cache = open(self.getUserCacheFilename(), "rb")
 616             lines = cache.readlines()
 617             cache.close()
 618             for line in lines:
 619                 entry = line.strip().split("\t")
 620                 self.users[entry[0]] = entry[1]
 621         except IOError:
 622             self.getUserMapFromPerforceServer()
 623
 624 class P4Debug(Command):
 625     def __init__(self):
 626         Command.__init__(self)
 627         self.options = [
 628             optparse.make_option("--verbose", dest="verbose", action="store_true",
 629                                  default=False),
 630             ]
 631         self.description = "A tool to debug the output of p4 -G."
 632         self.needsGit = False
 633         self.verbose = False
 634
 635     def run(self, args):
 636         j = 0
 637         for output in p4CmdList(args):
 638             print 'Element: %d' % j
 639             j += 1
 640             print output
 641         return True
 642
 643 class P4RollBack(Command):
 644     def __init__(self):
 645         Command.__init__(self)
 646         self.options = [
 647             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 648             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 649         ]
 650         self.description = "A tool to debug the multi-branch import. Don't use :)"
 651         self.verbose = False
 652         self.rollbackLocalBranches = False
 653
 654     def run(self, args):
 655         if len(args) != 1:
 656             return False
 657         maxChange = int(args[0])
 658
 659         if "p4ExitCode" in p4Cmd("changes -m 1"):
 660             die("Problems executing p4");
 661
 662         if self.rollbackLocalBranches:
 663             refPrefix = "refs/heads/"
 664             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 665         else:
 666             refPrefix = "refs/remotes/"
 667             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 668
 669         for line in lines:
 670             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 671                 line = line.strip()
 672                 ref = refPrefix + line
 673                 log = extractLogMessageFromGitCommit(ref)
 674                 settings = extractSettingsGitLog(log)
 675
 676                 depotPaths = settings['depot-paths']
 677                 change = settings['change']
 678
 679                 changed = False
 680
 681                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 682                                                            for p in depotPaths]))) == 0:
 683                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 684                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 685                     continue
 686
 687                 while change and int(change) > maxChange:
 688                     changed = True
 689                     if self.verbose:
 690                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 691                     system("git update-ref %s \"%s^\"" % (ref, ref))
 692                     log = extractLogMessageFromGitCommit(ref)
 693                     settings =  extractSettingsGitLog(log)
 694
 695
 696                     depotPaths = settings['depot-paths']
 697                     change = settings['change']
 698
 699                 if changed:
 700                     print "%s rewound to %s" % (ref, change)
 701
 702         return True
 703
 704 class P4Submit(Command, P4UserMap):
 705     def __init__(self):
 706         Command.__init__(self)
 707         P4UserMap.__init__(self)
 708         self.options = [
 709                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 710                 optparse.make_option("--origin", dest="origin"),
 711                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 712                 # preserve the user, requires relevant p4 permissions
 713                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 714         ]
 715         self.description = "Submit changes from git to the perforce depot."
 716         self.usage += " [name of git branch to submit into perforce depot]"
 717         self.interactive = True
 718         self.origin = ""
 719         self.detectRenames = False
 720         self.verbose = False
 721         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 722         self.isWindows = (platform.system() == "Windows")
 723
 724     def check(self):
 725         if len(p4CmdList("opened ...")) > 0:
 726             die("You have files opened with perforce! Close them before starting the sync.")
 727
 728     # replaces everything between 'Description:' and the next P4 submit template field with the
 729     # commit message
 730     def prepareLogMessage(self, template, message):
 731         result = ""
 732
 733         inDescriptionSection = False
 734
 735         for line in template.split("\n"):
 736             if line.startswith("#"):
 737                 result += line + "\n"
 738                 continue
 739
 740             if inDescriptionSection:
 741                 if line.startswith("Files:") or line.startswith("Jobs:"):
 742                     inDescriptionSection = False
 743                 else:
 744                     continue
 745             else:
 746                 if line.startswith("Description:"):
 747                     inDescriptionSection = True
 748                     line += "\n"
 749                     for messageLine in message.split("\n"):
 750                         line += "\t" + messageLine + "\n"
 751
 752             result += line + "\n"
 753
 754         return result
 755
 756     def p4UserForCommit(self,id):
 757         # Return the tuple (perforce user,git email) for a given git commit id
 758         self.getUserMapFromPerforceServer()
 759         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 760         gitEmail = gitEmail.strip()
 761         if not self.emails.has_key(gitEmail):
 762             return (None,gitEmail)
 763         else:
 764             return (self.emails[gitEmail],gitEmail)
 765
 766     def checkValidP4Users(self,commits):
 767         # check if any git authors cannot be mapped to p4 users
 768         for id in commits:
 769             (user,email) = self.p4UserForCommit(id)
 770             if not user:
 771                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 772                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 773                     print "%s" % msg
 774                 else:
 775                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 776
 777     def lastP4Changelist(self):
 778         # Get back the last changelist number submitted in this client spec. This
 779         # then gets used to patch up the username in the change. If the same
 780         # client spec is being used by multiple processes then this might go
 781         # wrong.
 782         results = p4CmdList("client -o")        # find the current client
 783         client = None
 784         for r in results:
 785             if r.has_key('Client'):
 786                 client = r['Client']
 787                 break
 788         if not client:
 789             die("could not get client spec")
 790         results = p4CmdList(["changes", "-c", client, "-m", "1"])
 791         for r in results:
 792             if r.has_key('change'):
 793                 return r['change']
 794         die("Could not get changelist number for last submit - cannot patch up user details")
 795
 796     def modifyChangelistUser(self, changelist, newUser):
 797         # fixup the user field of a changelist after it has been submitted.
 798         changes = p4CmdList("change -o %s" % changelist)
 799         if len(changes) != 1:
 800             die("Bad output from p4 change modifying %s to user %s" %
 801                 (changelist, newUser))
 802
 803         c = changes[0]
 804         if c['User'] == newUser: return   # nothing to do
 805         c['User'] = newUser
 806         input = marshal.dumps(c)
 807
 808         result = p4CmdList("change -f -i", stdin=input)
 809         for r in result:
 810             if r.has_key('code'):
 811                 if r['code'] == 'error':
 812                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 813             if r.has_key('data'):
 814                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
 815                 return
 816         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 817
 818     def canChangeChangelists(self):
 819         # check to see if we have p4 admin or super-user permissions, either of
 820         # which are required to modify changelists.
 821         results = p4CmdList(["protects", self.depotPath])
 822         for r in results:
 823             if r.has_key('perm'):
 824                 if r['perm'] == 'admin':
 825                     return 1
 826                 if r['perm'] == 'super':
 827                     return 1
 828         return 0
 829
 830     def prepareSubmitTemplate(self):
 831         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 832         template = ""
 833         inFilesSection = False
 834         for line in p4_read_pipe_lines(['change', '-o']):
 835             if line.endswith("\r\n"):
 836                 line = line[:-2] + "\n"
 837             if inFilesSection:
 838                 if line.startswith("\t"):
 839                     # path starts and ends with a tab
 840                     path = line[1:]
 841                     lastTab = path.rfind("\t")
 842                     if lastTab != -1:
 843                         path = path[:lastTab]
 844                         if not p4PathStartsWith(path, self.depotPath):
 845                             continue
 846                 else:
 847                     inFilesSection = False
 848             else:
 849                 if line.startswith("Files:"):
 850                     inFilesSection = True
 851
 852             template += line
 853
 854         return template
 855
 856     def edit_template(self, template_file):
 857         """Invoke the editor to let the user change the submission
 858            message.  Return true if okay to continue with the submit."""
 859
 860         # if configured to skip the editing part, just submit
 861         if gitConfig("git-p4.skipSubmitEdit") == "true":
 862             return True
 863
 864         # look at the modification time, to check later if the user saved
 865         # the file
 866         mtime = os.stat(template_file).st_mtime
 867
 868         # invoke the editor
 869         if os.environ.has_key("P4EDITOR"):
 870             editor = os.environ.get("P4EDITOR")
 871         else:
 872             editor = read_pipe("git var GIT_EDITOR").strip()
 873         system(editor + " " + template_file)
 874
 875         # If the file was not saved, prompt to see if this patch should
 876         # be skipped.  But skip this verification step if configured so.
 877         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 878             return True
 879
 880         # modification time updated means user saved the file
 881         if os.stat(template_file).st_mtime > mtime:
 882             return True
 883
 884         while True:
 885             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 886             if response == 'y':
 887                 return True
 888             if response == 'n':
 889                 return False
 890
 891     def applyCommit(self, id):
 892         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 893
 894         (p4User, gitEmail) = self.p4UserForCommit(id)
 895
 896         if not self.detectRenames:
 897             # If not explicitly set check the config variable
 898             self.detectRenames = gitConfig("git-p4.detectRenames")
 899
 900         if self.detectRenames.lower() == "false" or self.detectRenames == "":
 901             diffOpts = ""
 902         elif self.detectRenames.lower() == "true":
 903             diffOpts = "-M"
 904         else:
 905             diffOpts = "-M%s" % self.detectRenames
 906
 907         detectCopies = gitConfig("git-p4.detectCopies")
 908         if detectCopies.lower() == "true":
 909             diffOpts += " -C"
 910         elif detectCopies != "" and detectCopies.lower() != "false":
 911             diffOpts += " -C%s" % detectCopies
 912
 913         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
 914             diffOpts += " --find-copies-harder"
 915
 916         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 917         filesToAdd = set()
 918         filesToDelete = set()
 919         editedFiles = set()
 920         filesToChangeExecBit = {}
 921         for line in diff:
 922             diff = parseDiffTreeEntry(line)
 923             modifier = diff['status']
 924             path = diff['src']
 925             if modifier == "M":
 926                 p4_edit(path)
 927                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 928                     filesToChangeExecBit[path] = diff['dst_mode']
 929                 editedFiles.add(path)
 930             elif modifier == "A":
 931                 filesToAdd.add(path)
 932                 filesToChangeExecBit[path] = diff['dst_mode']
 933                 if path in filesToDelete:
 934                     filesToDelete.remove(path)
 935             elif modifier == "D":
 936                 filesToDelete.add(path)
 937                 if path in filesToAdd:
 938                     filesToAdd.remove(path)
 939             elif modifier == "C":
 940                 src, dest = diff['src'], diff['dst']
 941                 p4_integrate(src, dest)
 942                 if diff['src_sha1'] != diff['dst_sha1']:
 943                     p4_edit(dest)
 944                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 945                     p4_edit(dest)
 946                     filesToChangeExecBit[dest] = diff['dst_mode']
 947                 os.unlink(dest)
 948                 editedFiles.add(dest)
 949             elif modifier == "R":
 950                 src, dest = diff['src'], diff['dst']
 951                 p4_integrate(src, dest)
 952                 if diff['src_sha1'] != diff['dst_sha1']:
 953                     p4_edit(dest)
 954                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 955                     p4_edit(dest)
 956                     filesToChangeExecBit[dest] = diff['dst_mode']
 957                 os.unlink(dest)
 958                 editedFiles.add(dest)
 959                 filesToDelete.add(src)
 960             else:
 961                 die("unknown modifier %s for %s" % (modifier, path))
 962
 963         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 964         patchcmd = diffcmd + " | git apply "
 965         tryPatchCmd = patchcmd + "--check -"
 966         applyPatchCmd = patchcmd + "--check --apply -"
 967
 968         if os.system(tryPatchCmd) != 0:
 969             print "Unfortunately applying the change failed!"
 970             print "What do you want to do?"
 971             response = "x"
 972             while response != "s" and response != "a" and response != "w":
 973                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 974                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 975             if response == "s":
 976                 print "Skipping! Good luck with the next patches..."
 977                 for f in editedFiles:
 978                     p4_revert(f)
 979                 for f in filesToAdd:
 980                     os.remove(f)
 981                 return
 982             elif response == "a":
 983                 os.system(applyPatchCmd)
 984                 if len(filesToAdd) > 0:
 985                     print "You may also want to call p4 add on the following files:"
 986                     print " ".join(filesToAdd)
 987                 if len(filesToDelete):
 988                     print "The following files should be scheduled for deletion with p4 delete:"
 989                     print " ".join(filesToDelete)
 990                 die("Please resolve and submit the conflict manually and "
 991                     + "continue afterwards with git-p4 submit --continue")
 992             elif response == "w":
 993                 system(diffcmd + " > patch.txt")
 994                 print "Patch saved to patch.txt in %s !" % self.clientPath
 995                 die("Please resolve and submit the conflict manually and "
 996                     "continue afterwards with git-p4 submit --continue")
 997
 998         system(applyPatchCmd)
 999
1000         for f in filesToAdd:
1001             p4_add(f)
1002         for f in filesToDelete:
1003             p4_revert(f)
1004             p4_delete(f)
1005
1006         # Set/clear executable bits
1007         for f in filesToChangeExecBit.keys():
1008             mode = filesToChangeExecBit[f]
1009             setP4ExecBit(f, mode)
1010
1011         logMessage = extractLogMessageFromGitCommit(id)
1012         logMessage = logMessage.strip()
1013
1014         template = self.prepareSubmitTemplate()
1015
1016         if self.interactive:
1017             submitTemplate = self.prepareLogMessage(template, logMessage)
1018
1019             if self.preserveUser:
1020                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1021
1022             if os.environ.has_key("P4DIFF"):
1023                 del(os.environ["P4DIFF"])
1024             diff = ""
1025             for editedFile in editedFiles:
1026                 diff += p4_read_pipe(['diff', '-du', editedFile])
1027
1028             newdiff = ""
1029             for newFile in filesToAdd:
1030                 newdiff += "==== new file ====\n"
1031                 newdiff += "--- /dev/null\n"
1032                 newdiff += "+++ %s\n" % newFile
1033                 f = open(newFile, "r")
1034                 for line in f.readlines():
1035                     newdiff += "+" + line
1036                 f.close()
1037
1038             if self.checkAuthorship and not self.p4UserIsMe(p4User):
1039                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1040                 submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
1041                 submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
1042
1043             separatorLine = "######## everything below this line is just the diff #######\n"
1044
1045             (handle, fileName) = tempfile.mkstemp()
1046             tmpFile = os.fdopen(handle, "w+")
1047             if self.isWindows:
1048                 submitTemplate = submitTemplate.replace("\n", "\r\n")
1049                 separatorLine = separatorLine.replace("\n", "\r\n")
1050                 newdiff = newdiff.replace("\n", "\r\n")
1051             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1052             tmpFile.close()
1053
1054             if self.edit_template(fileName):
1055                 # read the edited message and submit
1056                 tmpFile = open(fileName, "rb")
1057                 message = tmpFile.read()
1058                 tmpFile.close()
1059                 submitTemplate = message[:message.index(separatorLine)]
1060                 if self.isWindows:
1061                     submitTemplate = submitTemplate.replace("\r\n", "\n")
1062                 p4_write_pipe(['submit', '-i'], submitTemplate)
1063
1064                 if self.preserveUser:
1065                     if p4User:
1066                         # Get last changelist number. Cannot easily get it from
1067                         # the submit command output as the output is
1068                         # unmarshalled.
1069                         changelist = self.lastP4Changelist()
1070                         self.modifyChangelistUser(changelist, p4User)
1071             else:
1072                 # skip this patch
1073                 print "Submission cancelled, undoing p4 changes."
1074                 for f in editedFiles:
1075                     p4_revert(f)
1076                 for f in filesToAdd:
1077                     p4_revert(f)
1078                     os.remove(f)
1079
1080             os.remove(fileName)
1081         else:
1082             fileName = "submit.txt"
1083             file = open(fileName, "w+")
1084             file.write(self.prepareLogMessage(template, logMessage))
1085             file.close()
1086             print ("Perforce submit template written as %s. "
1087                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1088                    % (fileName, fileName))
1089
1090     def run(self, args):
1091         if len(args) == 0:
1092             self.master = currentGitBranch()
1093             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1094                 die("Detecting current git branch failed!")
1095         elif len(args) == 1:
1096             self.master = args[0]
1097             if not branchExists(self.master):
1098                 die("Branch %s does not exist" % self.master)
1099         else:
1100             return False
1101
1102         allowSubmit = gitConfig("git-p4.allowSubmit")
1103         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1104             die("%s is not in git-p4.allowSubmit" % self.master)
1105
1106         [upstream, settings] = findUpstreamBranchPoint()
1107         self.depotPath = settings['depot-paths'][0]
1108         if len(self.origin) == 0:
1109             self.origin = upstream
1110
1111         if self.preserveUser:
1112             if not self.canChangeChangelists():
1113                 die("Cannot preserve user names without p4 super-user or admin permissions")
1114
1115         if self.verbose:
1116             print "Origin branch is " + self.origin
1117
1118         if len(self.depotPath) == 0:
1119             print "Internal error: cannot locate perforce depot path from existing branches"
1120             sys.exit(128)
1121
1122         self.clientPath = p4Where(self.depotPath)
1123
1124         if len(self.clientPath) == 0:
1125             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
1126             sys.exit(128)
1127
1128         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1129         self.oldWorkingDirectory = os.getcwd()
1130
1131         # ensure the clientPath exists
1132         if not os.path.exists(self.clientPath):
1133             os.makedirs(self.clientPath)
1134
1135         chdir(self.clientPath)
1136         print "Synchronizing p4 checkout..."
1137         p4_sync("...")
1138         self.check()
1139
1140         commits = []
1141         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1142             commits.append(line.strip())
1143         commits.reverse()
1144
1145         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1146             self.checkAuthorship = False
1147         else:
1148             self.checkAuthorship = True
1149
1150         if self.preserveUser:
1151             self.checkValidP4Users(commits)
1152
1153         while len(commits) > 0:
1154             commit = commits[0]
1155             commits = commits[1:]
1156             self.applyCommit(commit)
1157             if not self.interactive:
1158                 break
1159
1160         if len(commits) == 0:
1161             print "All changes applied!"
1162             chdir(self.oldWorkingDirectory)
1163
1164             sync = P4Sync()
1165             sync.run([])
1166
1167             rebase = P4Rebase()
1168             rebase.rebase()
1169
1170         return True
1171
1172 class View(object):
1173     """Represent a p4 view ("p4 help views"), and map files in a
1174        repo according to the view."""
1175
1176     class Path(object):
1177         """A depot or client path, possibly containing wildcards.
1178            The only one supported is ... at the end, currently.
1179            Initialize with the full path, with //depot or //client."""
1180
1181         def __init__(self, path, is_depot):
1182             self.path = path
1183             self.is_depot = is_depot
1184             self.find_wildcards()
1185             # remember the prefix bit, useful for relative mappings
1186             m = re.match("(//[^/]+/)", self.path)
1187             if not m:
1188                 die("Path %s does not start with //prefix/" % self.path)
1189             prefix = m.group(1)
1190             if not self.is_depot:
1191                 # strip //client/ on client paths
1192                 self.path = self.path[len(prefix):]
1193
1194         def find_wildcards(self):
1195             """Make sure wildcards are valid, and set up internal
1196                variables."""
1197
1198             self.ends_triple_dot = False
1199             # There are three wildcards allowed in p4 views
1200             # (see "p4 help views").  This code knows how to
1201             # handle "..." (only at the end), but cannot deal with
1202             # "%%n" or "*".  Only check the depot_side, as p4 should
1203             # validate that the client_side matches too.
1204             if re.search(r'%%[1-9]', self.path):
1205                 die("Can't handle %%n wildcards in view: %s" % self.path)
1206             if self.path.find("*") >= 0:
1207                 die("Can't handle * wildcards in view: %s" % self.path)
1208             triple_dot_index = self.path.find("...")
1209             if triple_dot_index >= 0:
1210                 if triple_dot_index != len(self.path) - 3:
1211                     die("Can handle only single ... wildcard, at end: %s" %
1212                         self.path)
1213                 self.ends_triple_dot = True
1214
1215         def ensure_compatible(self, other_path):
1216             """Make sure the wildcards agree."""
1217             if self.ends_triple_dot != other_path.ends_triple_dot:
1218                  die("Both paths must end with ... if either does;\n" +
1219                      "paths: %s %s" % (self.path, other_path.path))
1220
1221         def match_wildcards(self, test_path):
1222             """See if this test_path matches us, and fill in the value
1223                of the wildcards if so.  Returns a tuple of
1224                (True|False, wildcards[]).  For now, only the ... at end
1225                is supported, so at most one wildcard."""
1226             if self.ends_triple_dot:
1227                 dotless = self.path[:-3]
1228                 if test_path.startswith(dotless):
1229                     wildcard = test_path[len(dotless):]
1230                     return (True, [ wildcard ])
1231             else:
1232                 if test_path == self.path:
1233                     return (True, [])
1234             return (False, [])
1235
1236         def match(self, test_path):
1237             """Just return if it matches; don't bother with the wildcards."""
1238             b, _ = self.match_wildcards(test_path)
1239             return b
1240
1241         def fill_in_wildcards(self, wildcards):
1242             """Return the relative path, with the wildcards filled in
1243                if there are any."""
1244             if self.ends_triple_dot:
1245                 return self.path[:-3] + wildcards[0]
1246             else:
1247                 return self.path
1248
1249     class Mapping(object):
1250         def __init__(self, depot_side, client_side, overlay, exclude):
1251             # depot_side is without the trailing /... if it had one
1252             self.depot_side = View.Path(depot_side, is_depot=True)
1253             self.client_side = View.Path(client_side, is_depot=False)
1254             self.overlay = overlay  # started with "+"
1255             self.exclude = exclude  # started with "-"
1256             assert not (self.overlay and self.exclude)
1257             self.depot_side.ensure_compatible(self.client_side)
1258
1259         def __str__(self):
1260             c = " "
1261             if self.overlay:
1262                 c = "+"
1263             if self.exclude:
1264                 c = "-"
1265             return "View.Mapping: %s%s -> %s" % \
1266                    (c, self.depot_side.path, self.client_side.path)
1267
1268         def map_depot_to_client(self, depot_path):
1269             """Calculate the client path if using this mapping on the
1270                given depot path; does not consider the effect of other
1271                mappings in a view.  Even excluded mappings are returned."""
1272             matches, wildcards = self.depot_side.match_wildcards(depot_path)
1273             if not matches:
1274                 return ""
1275             client_path = self.client_side.fill_in_wildcards(wildcards)
1276             return client_path
1277
1278     #
1279     # View methods
1280     #
1281     def __init__(self):
1282         self.mappings = []
1283
1284     def append(self, view_line):
1285         """Parse a view line, splitting it into depot and client
1286            sides.  Append to self.mappings, preserving order."""
1287
1288         # Split the view line into exactly two words.  P4 enforces
1289         # structure on these lines that simplifies this quite a bit.
1290         #
1291         # Either or both words may be double-quoted.
1292         # Single quotes do not matter.
1293         # Double-quote marks cannot occur inside the words.
1294         # A + or - prefix is also inside the quotes.
1295         # There are no quotes unless they contain a space.
1296         # The line is already white-space stripped.
1297         # The two words are separated by a single space.
1298         #
1299         if view_line[0] == '"':
1300             # First word is double quoted.  Find its end.
1301             close_quote_index = view_line.find('"', 1)
1302             if close_quote_index <= 0:
1303                 die("No first-word closing quote found: %s" % view_line)
1304             depot_side = view_line[1:close_quote_index]
1305             # skip closing quote and space
1306             rhs_index = close_quote_index + 1 + 1
1307         else:
1308             space_index = view_line.find(" ")
1309             if space_index <= 0:
1310                 die("No word-splitting space found: %s" % view_line)
1311             depot_side = view_line[0:space_index]
1312             rhs_index = space_index + 1
1313
1314         if view_line[rhs_index] == '"':
1315             # Second word is double quoted.  Make sure there is a
1316             # double quote at the end too.
1317             if not view_line.endswith('"'):
1318                 die("View line with rhs quote should end with one: %s" %
1319                     view_line)
1320             # skip the quotes
1321             client_side = view_line[rhs_index+1:-1]
1322         else:
1323             client_side = view_line[rhs_index:]
1324
1325         # prefix + means overlay on previous mapping
1326         overlay = False
1327         if depot_side.startswith("+"):
1328             overlay = True
1329             depot_side = depot_side[1:]
1330
1331         # prefix - means exclude this path
1332         exclude = False
1333         if depot_side.startswith("-"):
1334             exclude = True
1335             depot_side = depot_side[1:]
1336
1337         m = View.Mapping(depot_side, client_side, overlay, exclude)
1338         self.mappings.append(m)
1339
1340     def map_in_client(self, depot_path):
1341         """Return the relative location in the client where this
1342            depot file should live.  Returns "" if the file should
1343            not be mapped in the client."""
1344
1345         paths_filled = []
1346         client_path = ""
1347
1348         # look at later entries first
1349         for m in self.mappings[::-1]:
1350
1351             # see where will this path end up in the client
1352             p = m.map_depot_to_client(depot_path)
1353
1354             if p == "":
1355                 # Depot path does not belong in client.  Must remember
1356                 # this, as previous items should not cause files to
1357                 # exist in this path either.  Remember that the list is
1358                 # being walked from the end, which has higher precedence.
1359                 # Overlap mappings do not exclude previous mappings.
1360                 if not m.overlay:
1361                     paths_filled.append(m.client_side)
1362
1363             else:
1364                 # This mapping matched; no need to search any further.
1365                 # But, the mapping could be rejected if the client path
1366                 # has already been claimed by an earlier mapping (i.e.
1367                 # one later in the list, which we are walking backwards).
1368                 already_mapped_in_client = False
1369                 for f in paths_filled:
1370                     # this is View.Path.match
1371                     if f.match(p):
1372                         already_mapped_in_client = True
1373                         break
1374                 if not already_mapped_in_client:
1375                     # Include this file, unless it is from a line that
1376                     # explicitly said to exclude it.
1377                     if not m.exclude:
1378                         client_path = p
1379
1380                 # a match, even if rejected, always stops the search
1381                 break
1382
1383         return client_path
1384
1385 class P4Sync(Command, P4UserMap):
1386     delete_actions = ( "delete", "move/delete", "purge" )
1387
1388     def __init__(self):
1389         Command.__init__(self)
1390         P4UserMap.__init__(self)
1391         self.options = [
1392                 optparse.make_option("--branch", dest="branch"),
1393                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1394                 optparse.make_option("--changesfile", dest="changesFile"),
1395                 optparse.make_option("--silent", dest="silent", action="store_true"),
1396                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1397                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
1398                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1399                                      help="Import into refs/heads/ , not refs/remotes"),
1400                 optparse.make_option("--max-changes", dest="maxChanges"),
1401                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1402                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1403                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1404                                      help="Only sync files that are included in the Perforce Client Spec")
1405         ]
1406         self.description = """Imports from Perforce into a git repository.\n
1407     example:
1408     //depot/my/project/ -- to import the current head
1409     //depot/my/project/@all -- to import everything
1410     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1411
1412     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1413
1414         self.usage += " //depot/path[@revRange]"
1415         self.silent = False
1416         self.createdBranches = set()
1417         self.committedChanges = set()
1418         self.branch = ""
1419         self.detectBranches = False
1420         self.detectLabels = False
1421         self.changesFile = ""
1422         self.syncWithOrigin = True
1423         self.verbose = False
1424         self.importIntoRemotes = True
1425         self.maxChanges = ""
1426         self.isWindows = (platform.system() == "Windows")
1427         self.keepRepoPath = False
1428         self.depotPaths = None
1429         self.p4BranchesInGit = []
1430         self.cloneExclude = []
1431         self.useClientSpec = False
1432         self.clientSpecDirs = None
1433         self.tempBranches = []
1434         self.tempBranchLocation = "git-p4-tmp"
1435
1436         if gitConfig("git-p4.syncFromOrigin") == "false":
1437             self.syncWithOrigin = False
1438
1439     #
1440     # P4 wildcards are not allowed in filenames.  P4 complains
1441     # if you simply add them, but you can force it with "-f", in
1442     # which case it translates them into %xx encoding internally.
1443     # Search for and fix just these four characters.  Do % last so
1444     # that fixing it does not inadvertently create new %-escapes.
1445     #
1446     def wildcard_decode(self, path):
1447         # Cannot have * in a filename in windows; untested as to
1448         # what p4 would do in such a case.
1449         if not self.isWindows:
1450             path = path.replace("%2A", "*")
1451         path = path.replace("%23", "#") \
1452                    .replace("%40", "@") \
1453                    .replace("%25", "%")
1454         return path
1455
1456     # Force a checkpoint in fast-import and wait for it to finish
1457     def checkpoint(self):
1458         self.gitStream.write("checkpoint\n\n")
1459         self.gitStream.write("progress checkpoint\n\n")
1460         out = self.gitOutput.readline()
1461         if self.verbose:
1462             print "checkpoint finished: " + out
1463
1464     def extractFilesFromCommit(self, commit):
1465         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1466                              for path in self.cloneExclude]
1467         files = []
1468         fnum = 0
1469         while commit.has_key("depotFile%s" % fnum):
1470             path =  commit["depotFile%s" % fnum]
1471
1472             if [p for p in self.cloneExclude
1473                 if p4PathStartsWith(path, p)]:
1474                 found = False
1475             else:
1476                 found = [p for p in self.depotPaths
1477                          if p4PathStartsWith(path, p)]
1478             if not found:
1479                 fnum = fnum + 1
1480                 continue
1481
1482             file = {}
1483             file["path"] = path
1484             file["rev"] = commit["rev%s" % fnum]
1485             file["action"] = commit["action%s" % fnum]
1486             file["type"] = commit["type%s" % fnum]
1487             files.append(file)
1488             fnum = fnum + 1
1489         return files
1490
1491     def stripRepoPath(self, path, prefixes):
1492         if self.useClientSpec:
1493             return self.clientSpecDirs.map_in_client(path)
1494
1495         if self.keepRepoPath:
1496             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1497
1498         for p in prefixes:
1499             if p4PathStartsWith(path, p):
1500                 path = path[len(p):]
1501
1502         return path
1503
1504     def splitFilesIntoBranches(self, commit):
1505         branches = {}
1506         fnum = 0
1507         while commit.has_key("depotFile%s" % fnum):
1508             path =  commit["depotFile%s" % fnum]
1509             found = [p for p in self.depotPaths
1510                      if p4PathStartsWith(path, p)]
1511             if not found:
1512                 fnum = fnum + 1
1513                 continue
1514
1515             file = {}
1516             file["path"] = path
1517             file["rev"] = commit["rev%s" % fnum]
1518             file["action"] = commit["action%s" % fnum]
1519             file["type"] = commit["type%s" % fnum]
1520             fnum = fnum + 1
1521
1522             relPath = self.stripRepoPath(path, self.depotPaths)
1523
1524             for branch in self.knownBranches.keys():
1525
1526                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1527                 if relPath.startswith(branch + "/"):
1528                     if branch not in branches:
1529                         branches[branch] = []
1530                     branches[branch].append(file)
1531                     break
1532
1533         return branches
1534
1535     # output one file from the P4 stream
1536     # - helper for streamP4Files
1537
1538     def streamOneP4File(self, file, contents):
1539         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1540         relPath = self.wildcard_decode(relPath)
1541         if verbose:
1542             sys.stderr.write("%s\n" % relPath)
1543
1544         (type_base, type_mods) = split_p4_type(file["type"])
1545
1546         git_mode = "100644"
1547         if "x" in type_mods:
1548             git_mode = "100755"
1549         if type_base == "symlink":
1550             git_mode = "120000"
1551             # p4 print on a symlink contains "target\n"; remove the newline
1552             data = ''.join(contents)
1553             contents = [data[:-1]]
1554
1555         if type_base == "utf16":
1556             # p4 delivers different text in the python output to -G
1557             # than it does when using "print -o", or normal p4 client
1558             # operations.  utf16 is converted to ascii or utf8, perhaps.
1559             # But ascii text saved as -t utf16 is completely mangled.
1560             # Invoke print -o to get the real contents.
1561             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1562             contents = [ text ]
1563
1564         if type_base == "apple":
1565             # Apple filetype files will be streamed as a concatenation of
1566             # its appledouble header and the contents.  This is useless
1567             # on both macs and non-macs.  If using "print -q -o xx", it
1568             # will create "xx" with the data, and "%xx" with the header.
1569             # This is also not very useful.
1570             #
1571             # Ideally, someday, this script can learn how to generate
1572             # appledouble files directly and import those to git, but
1573             # non-mac machines can never find a use for apple filetype.
1574             print "\nIgnoring apple filetype file %s" % file['depotFile']
1575             return
1576
1577         # Perhaps windows wants unicode, utf16 newlines translated too;
1578         # but this is not doing it.
1579         if self.isWindows and type_base == "text":
1580             mangled = []
1581             for data in contents:
1582                 data = data.replace("\r\n", "\n")
1583                 mangled.append(data)
1584             contents = mangled
1585
1586         # Note that we do not try to de-mangle keywords on utf16 files,
1587         # even though in theory somebody may want that.
1588         if type_base in ("text", "unicode", "binary"):
1589             if "ko" in type_mods:
1590                 text = ''.join(contents)
1591                 text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text)
1592                 contents = [ text ]
1593             elif "k" in type_mods:
1594                 text = ''.join(contents)
1595                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text)
1596                 contents = [ text ]
1597
1598         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1599
1600         # total length...
1601         length = 0
1602         for d in contents:
1603             length = length + len(d)
1604
1605         self.gitStream.write("data %d\n" % length)
1606         for d in contents:
1607             self.gitStream.write(d)
1608         self.gitStream.write("\n")
1609
1610     def streamOneP4Deletion(self, file):
1611         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1612         if verbose:
1613             sys.stderr.write("delete %s\n" % relPath)
1614         self.gitStream.write("D %s\n" % relPath)
1615
1616     # handle another chunk of streaming data
1617     def streamP4FilesCb(self, marshalled):
1618
1619         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1620             # start of a new file - output the old one first
1621             self.streamOneP4File(self.stream_file, self.stream_contents)
1622             self.stream_file = {}
1623             self.stream_contents = []
1624             self.stream_have_file_info = False
1625
1626         # pick up the new file information... for the
1627         # 'data' field we need to append to our array
1628         for k in marshalled.keys():
1629             if k == 'data':
1630                 self.stream_contents.append(marshalled['data'])
1631             else:
1632                 self.stream_file[k] = marshalled[k]
1633
1634         self.stream_have_file_info = True
1635
1636     # Stream directly from "p4 files" into "git fast-import"
1637     def streamP4Files(self, files):
1638         filesForCommit = []
1639         filesToRead = []
1640         filesToDelete = []
1641
1642         for f in files:
1643             # if using a client spec, only add the files that have
1644             # a path in the client
1645             if self.clientSpecDirs:
1646                 if self.clientSpecDirs.map_in_client(f['path']) == "":
1647                     continue
1648
1649             filesForCommit.append(f)
1650             if f['action'] in self.delete_actions:
1651                 filesToDelete.append(f)
1652             else:
1653                 filesToRead.append(f)
1654
1655         # deleted files...
1656         for f in filesToDelete:
1657             self.streamOneP4Deletion(f)
1658
1659         if len(filesToRead) > 0:
1660             self.stream_file = {}
1661             self.stream_contents = []
1662             self.stream_have_file_info = False
1663
1664             # curry self argument
1665             def streamP4FilesCbSelf(entry):
1666                 self.streamP4FilesCb(entry)
1667
1668             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1669
1670             p4CmdList(["-x", "-", "print"],
1671                       stdin=fileArgs,
1672                       cb=streamP4FilesCbSelf)
1673
1674             # do the last chunk
1675             if self.stream_file.has_key('depotFile'):
1676                 self.streamOneP4File(self.stream_file, self.stream_contents)
1677
1678     def make_email(self, userid):
1679         if userid in self.users:
1680             return self.users[userid]
1681         else:
1682             return "%s <a@b>" % userid
1683
1684     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1685         epoch = details["time"]
1686         author = details["user"]
1687         self.branchPrefixes = branchPrefixes
1688
1689         if self.verbose:
1690             print "commit into %s" % branch
1691
1692         # start with reading files; if that fails, we should not
1693         # create a commit.
1694         new_files = []
1695         for f in files:
1696             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1697                 new_files.append (f)
1698             else:
1699                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1700
1701         self.gitStream.write("commit %s\n" % branch)
1702 #        gitStream.write("mark :%s\n" % details["change"])
1703         self.committedChanges.add(int(details["change"]))
1704         committer = ""
1705         if author not in self.users:
1706             self.getUserMapFromPerforceServer()
1707         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
1708
1709         self.gitStream.write("committer %s\n" % committer)
1710
1711         self.gitStream.write("data <<EOT\n")
1712         self.gitStream.write(details["desc"])
1713         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1714                              % (','.join (branchPrefixes), details["change"]))
1715         if len(details['options']) > 0:
1716             self.gitStream.write(": options = %s" % details['options'])
1717         self.gitStream.write("]\nEOT\n\n")
1718
1719         if len(parent) > 0:
1720             if self.verbose:
1721                 print "parent %s" % parent
1722             self.gitStream.write("from %s\n" % parent)
1723
1724         self.streamP4Files(new_files)
1725         self.gitStream.write("\n")
1726
1727         change = int(details["change"])
1728
1729         if self.labels.has_key(change):
1730             label = self.labels[change]
1731             labelDetails = label[0]
1732             labelRevisions = label[1]
1733             if self.verbose:
1734                 print "Change %s is labelled %s" % (change, labelDetails)
1735
1736             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
1737                                                     for p in branchPrefixes])
1738
1739             if len(files) == len(labelRevisions):
1740
1741                 cleanedFiles = {}
1742                 for info in files:
1743                     if info["action"] in self.delete_actions:
1744                         continue
1745                     cleanedFiles[info["depotFile"]] = info["rev"]
1746
1747                 if cleanedFiles == labelRevisions:
1748                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1749                     self.gitStream.write("from %s\n" % branch)
1750
1751                     owner = labelDetails["Owner"]
1752
1753                     # Try to use the owner of the p4 label, or failing that,
1754                     # the current p4 user id.
1755                     if owner:
1756                         email = self.make_email(owner)
1757                     else:
1758                         email = self.make_email(self.p4UserId())
1759                     tagger = "%s %s %s" % (email, epoch, self.tz)
1760
1761                     self.gitStream.write("tagger %s\n" % tagger)
1762
1763                     description = labelDetails["Description"]
1764                     self.gitStream.write("data %d\n" % len(description))
1765                     self.gitStream.write(description)
1766                     self.gitStream.write("\n")
1767
1768                 else:
1769                     if not self.silent:
1770                         print ("Tag %s does not match with change %s: files do not match."
1771                                % (labelDetails["label"], change))
1772
1773             else:
1774                 if not self.silent:
1775                     print ("Tag %s does not match with change %s: file count is different."
1776                            % (labelDetails["label"], change))
1777
1778     def getLabels(self):
1779         self.labels = {}
1780
1781         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
1782         if len(l) > 0 and not self.silent:
1783             print "Finding files belonging to labels in %s" % `self.depotPaths`
1784
1785         for output in l:
1786             label = output["label"]
1787             revisions = {}
1788             newestChange = 0
1789             if self.verbose:
1790                 print "Querying files for label %s" % label
1791             for file in p4CmdList(["files"] +
1792                                       ["%s...@%s" % (p, label)
1793                                           for p in self.depotPaths]):
1794                 revisions[file["depotFile"]] = file["rev"]
1795                 change = int(file["change"])
1796                 if change > newestChange:
1797                     newestChange = change
1798
1799             self.labels[newestChange] = [output, revisions]
1800
1801         if self.verbose:
1802             print "Label changes: %s" % self.labels.keys()
1803
1804     def guessProjectName(self):
1805         for p in self.depotPaths:
1806             if p.endswith("/"):
1807                 p = p[:-1]
1808             p = p[p.strip().rfind("/") + 1:]
1809             if not p.endswith("/"):
1810                p += "/"
1811             return p
1812
1813     def getBranchMapping(self):
1814         lostAndFoundBranches = set()
1815
1816         user = gitConfig("git-p4.branchUser")
1817         if len(user) > 0:
1818             command = "branches -u %s" % user
1819         else:
1820             command = "branches"
1821
1822         for info in p4CmdList(command):
1823             details = p4Cmd(["branch", "-o", info["branch"]])
1824             viewIdx = 0
1825             while details.has_key("View%s" % viewIdx):
1826                 paths = details["View%s" % viewIdx].split(" ")
1827                 viewIdx = viewIdx + 1
1828                 # require standard //depot/foo/... //depot/bar/... mapping
1829                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1830                     continue
1831                 source = paths[0]
1832                 destination = paths[1]
1833                 ## HACK
1834                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1835                     source = source[len(self.depotPaths[0]):-4]
1836                     destination = destination[len(self.depotPaths[0]):-4]
1837
1838                     if destination in self.knownBranches:
1839                         if not self.silent:
1840                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1841                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1842                         continue
1843
1844                     self.knownBranches[destination] = source
1845
1846                     lostAndFoundBranches.discard(destination)
1847
1848                     if source not in self.knownBranches:
1849                         lostAndFoundBranches.add(source)
1850
1851         # Perforce does not strictly require branches to be defined, so we also
1852         # check git config for a branch list.
1853         #
1854         # Example of branch definition in git config file:
1855         # [git-p4]
1856         #   branchList=main:branchA
1857         #   branchList=main:branchB
1858         #   branchList=branchA:branchC
1859         configBranches = gitConfigList("git-p4.branchList")
1860         for branch in configBranches:
1861             if branch:
1862                 (source, destination) = branch.split(":")
1863                 self.knownBranches[destination] = source
1864
1865                 lostAndFoundBranches.discard(destination)
1866
1867                 if source not in self.knownBranches:
1868                     lostAndFoundBranches.add(source)
1869
1870
1871         for branch in lostAndFoundBranches:
1872             self.knownBranches[branch] = branch
1873
1874     def getBranchMappingFromGitBranches(self):
1875         branches = p4BranchesInGit(self.importIntoRemotes)
1876         for branch in branches.keys():
1877             if branch == "master":
1878                 branch = "main"
1879             else:
1880                 branch = branch[len(self.projectName):]
1881             self.knownBranches[branch] = branch
1882
1883     def listExistingP4GitBranches(self):
1884         # branches holds mapping from name to commit
1885         branches = p4BranchesInGit(self.importIntoRemotes)
1886         self.p4BranchesInGit = branches.keys()
1887         for branch in branches.keys():
1888             self.initialParents[self.refPrefix + branch] = branches[branch]
1889
1890     def updateOptionDict(self, d):
1891         option_keys = {}
1892         if self.keepRepoPath:
1893             option_keys['keepRepoPath'] = 1
1894
1895         d["options"] = ' '.join(sorted(option_keys.keys()))
1896
1897     def readOptions(self, d):
1898         self.keepRepoPath = (d.has_key('options')
1899                              and ('keepRepoPath' in d['options']))
1900
1901     def gitRefForBranch(self, branch):
1902         if branch == "main":
1903             return self.refPrefix + "master"
1904
1905         if len(branch) <= 0:
1906             return branch
1907
1908         return self.refPrefix + self.projectName + branch
1909
1910     def gitCommitByP4Change(self, ref, change):
1911         if self.verbose:
1912             print "looking in ref " + ref + " for change %s using bisect..." % change
1913
1914         earliestCommit = ""
1915         latestCommit = parseRevision(ref)
1916
1917         while True:
1918             if self.verbose:
1919                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1920             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1921             if len(next) == 0:
1922                 if self.verbose:
1923                     print "argh"
1924                 return ""
1925             log = extractLogMessageFromGitCommit(next)
1926             settings = extractSettingsGitLog(log)
1927             currentChange = int(settings['change'])
1928             if self.verbose:
1929                 print "current change %s" % currentChange
1930
1931             if currentChange == change:
1932                 if self.verbose:
1933                     print "found %s" % next
1934                 return next
1935
1936             if currentChange < change:
1937                 earliestCommit = "^%s" % next
1938             else:
1939                 latestCommit = "%s" % next
1940
1941         return ""
1942
1943     def importNewBranch(self, branch, maxChange):
1944         # make fast-import flush all changes to disk and update the refs using the checkpoint
1945         # command so that we can try to find the branch parent in the git history
1946         self.gitStream.write("checkpoint\n\n");
1947         self.gitStream.flush();
1948         branchPrefix = self.depotPaths[0] + branch + "/"
1949         range = "@1,%s" % maxChange
1950         #print "prefix" + branchPrefix
1951         changes = p4ChangesForPaths([branchPrefix], range)
1952         if len(changes) <= 0:
1953             return False
1954         firstChange = changes[0]
1955         #print "first change in branch: %s" % firstChange
1956         sourceBranch = self.knownBranches[branch]
1957         sourceDepotPath = self.depotPaths[0] + sourceBranch
1958         sourceRef = self.gitRefForBranch(sourceBranch)
1959         #print "source " + sourceBranch
1960
1961         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1962         #print "branch parent: %s" % branchParentChange
1963         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1964         if len(gitParent) > 0:
1965             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1966             #print "parent git commit: %s" % gitParent
1967
1968         self.importChanges(changes)
1969         return True
1970
1971     def searchParent(self, parent, branch, target):
1972         parentFound = False
1973         for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
1974             blob = blob.strip()
1975             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
1976                 parentFound = True
1977                 if self.verbose:
1978                     print "Found parent of %s in commit %s" % (branch, blob)
1979                 break
1980         if parentFound:
1981             return blob
1982         else:
1983             return None
1984
1985     def importChanges(self, changes):
1986         cnt = 1
1987         for change in changes:
1988             description = p4Cmd(["describe", str(change)])
1989             self.updateOptionDict(description)
1990
1991             if not self.silent:
1992                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1993                 sys.stdout.flush()
1994             cnt = cnt + 1
1995
1996             try:
1997                 if self.detectBranches:
1998                     branches = self.splitFilesIntoBranches(description)
1999                     for branch in branches.keys():
2000                         ## HACK  --hwn
2001                         branchPrefix = self.depotPaths[0] + branch + "/"
2002
2003                         parent = ""
2004
2005                         filesForCommit = branches[branch]
2006
2007                         if self.verbose:
2008                             print "branch is %s" % branch
2009
2010                         self.updatedBranches.add(branch)
2011
2012                         if branch not in self.createdBranches:
2013                             self.createdBranches.add(branch)
2014                             parent = self.knownBranches[branch]
2015                             if parent == branch:
2016                                 parent = ""
2017                             else:
2018                                 fullBranch = self.projectName + branch
2019                                 if fullBranch not in self.p4BranchesInGit:
2020                                     if not self.silent:
2021                                         print("\n    Importing new branch %s" % fullBranch);
2022                                     if self.importNewBranch(branch, change - 1):
2023                                         parent = ""
2024                                         self.p4BranchesInGit.append(fullBranch)
2025                                     if not self.silent:
2026                                         print("\n    Resuming with change %s" % change);
2027
2028                                 if self.verbose:
2029                                     print "parent determined through known branches: %s" % parent
2030
2031                         branch = self.gitRefForBranch(branch)
2032                         parent = self.gitRefForBranch(parent)
2033
2034                         if self.verbose:
2035                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2036
2037                         if len(parent) == 0 and branch in self.initialParents:
2038                             parent = self.initialParents[branch]
2039                             del self.initialParents[branch]
2040
2041                         blob = None
2042                         if len(parent) > 0:
2043                             tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2044                             if self.verbose:
2045                                 print "Creating temporary branch: " + tempBranch
2046                             self.commit(description, filesForCommit, tempBranch, [branchPrefix])
2047                             self.tempBranches.append(tempBranch)
2048                             self.checkpoint()
2049                             blob = self.searchParent(parent, branch, tempBranch)
2050                         if blob:
2051                             self.commit(description, filesForCommit, branch, [branchPrefix], blob)
2052                         else:
2053                             if self.verbose:
2054                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2055                             self.commit(description, filesForCommit, branch, [branchPrefix], parent)
2056                 else:
2057                     files = self.extractFilesFromCommit(description)
2058                     self.commit(description, files, self.branch, self.depotPaths,
2059                                 self.initialParent)
2060                     self.initialParent = ""
2061             except IOError:
2062                 print self.gitError.read()
2063                 sys.exit(1)
2064
2065     def importHeadRevision(self, revision):
2066         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2067
2068         details = {}
2069         details["user"] = "git perforce import user"
2070         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2071                            % (' '.join(self.depotPaths), revision))
2072         details["change"] = revision
2073         newestRevision = 0
2074
2075         fileCnt = 0
2076         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2077
2078         for info in p4CmdList(["files"] + fileArgs):
2079
2080             if 'code' in info and info['code'] == 'error':
2081                 sys.stderr.write("p4 returned an error: %s\n"
2082                                  % info['data'])
2083                 if info['data'].find("must refer to client") >= 0:
2084                     sys.stderr.write("This particular p4 error is misleading.\n")
2085                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2086                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2087                 sys.exit(1)
2088             if 'p4ExitCode' in info:
2089                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2090                 sys.exit(1)
2091
2092
2093             change = int(info["change"])
2094             if change > newestRevision:
2095                 newestRevision = change
2096
2097             if info["action"] in self.delete_actions:
2098                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2099                 #fileCnt = fileCnt + 1
2100                 continue
2101
2102             for prop in ["depotFile", "rev", "action", "type" ]:
2103                 details["%s%s" % (prop, fileCnt)] = info[prop]
2104
2105             fileCnt = fileCnt + 1
2106
2107         details["change"] = newestRevision
2108
2109         # Use time from top-most change so that all git-p4 clones of
2110         # the same p4 repo have the same commit SHA1s.
2111         res = p4CmdList("describe -s %d" % newestRevision)
2112         newestTime = None
2113         for r in res:
2114             if r.has_key('time'):
2115                 newestTime = int(r['time'])
2116         if newestTime is None:
2117             die("\"describe -s\" on newest change %d did not give a time")
2118         details["time"] = newestTime
2119
2120         self.updateOptionDict(details)
2121         try:
2122             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
2123         except IOError:
2124             print "IO error with git fast-import. Is your git version recent enough?"
2125             print self.gitError.read()
2126
2127
2128     def getClientSpec(self):
2129         specList = p4CmdList("client -o")
2130         if len(specList) != 1:
2131             die('Output from "client -o" is %d lines, expecting 1' %
2132                 len(specList))
2133
2134         # dictionary of all client parameters
2135         entry = specList[0]
2136
2137         # just the keys that start with "View"
2138         view_keys = [ k for k in entry.keys() if k.startswith("View") ]
2139
2140         # hold this new View
2141         view = View()
2142
2143         # append the lines, in order, to the view
2144         for view_num in range(len(view_keys)):
2145             k = "View%d" % view_num
2146             if k not in view_keys:
2147                 die("Expected view key %s missing" % k)
2148             view.append(entry[k])
2149
2150         self.clientSpecDirs = view
2151         if self.verbose:
2152             for i, m in enumerate(self.clientSpecDirs.mappings):
2153                     print "clientSpecDirs %d: %s" % (i, str(m))
2154
2155     def run(self, args):
2156         self.depotPaths = []
2157         self.changeRange = ""
2158         self.initialParent = ""
2159         self.previousDepotPaths = []
2160
2161         # map from branch depot path to parent branch
2162         self.knownBranches = {}
2163         self.initialParents = {}
2164         self.hasOrigin = originP4BranchesExist()
2165         if not self.syncWithOrigin:
2166             self.hasOrigin = False
2167
2168         if self.importIntoRemotes:
2169             self.refPrefix = "refs/remotes/p4/"
2170         else:
2171             self.refPrefix = "refs/heads/p4/"
2172
2173         if self.syncWithOrigin and self.hasOrigin:
2174             if not self.silent:
2175                 print "Syncing with origin first by calling git fetch origin"
2176             system("git fetch origin")
2177
2178         if len(self.branch) == 0:
2179             self.branch = self.refPrefix + "master"
2180             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2181                 system("git update-ref %s refs/heads/p4" % self.branch)
2182                 system("git branch -D p4");
2183             # create it /after/ importing, when master exists
2184             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2185                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2186
2187         if not self.useClientSpec:
2188             if gitConfig("git-p4.useclientspec", "--bool") == "true":
2189                 self.useClientSpec = True
2190         if self.useClientSpec:
2191             self.getClientSpec()
2192
2193         # TODO: should always look at previous commits,
2194         # merge with previous imports, if possible.
2195         if args == []:
2196             if self.hasOrigin:
2197                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2198             self.listExistingP4GitBranches()
2199
2200             if len(self.p4BranchesInGit) > 1:
2201                 if not self.silent:
2202                     print "Importing from/into multiple branches"
2203                 self.detectBranches = True
2204
2205             if self.verbose:
2206                 print "branches: %s" % self.p4BranchesInGit
2207
2208             p4Change = 0
2209             for branch in self.p4BranchesInGit:
2210                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2211
2212                 settings = extractSettingsGitLog(logMsg)
2213
2214                 self.readOptions(settings)
2215                 if (settings.has_key('depot-paths')
2216                     and settings.has_key ('change')):
2217                     change = int(settings['change']) + 1
2218                     p4Change = max(p4Change, change)
2219
2220                     depotPaths = sorted(settings['depot-paths'])
2221                     if self.previousDepotPaths == []:
2222                         self.previousDepotPaths = depotPaths
2223                     else:
2224                         paths = []
2225                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2226                             prev_list = prev.split("/")
2227                             cur_list = cur.split("/")
2228                             for i in range(0, min(len(cur_list), len(prev_list))):
2229                                 if cur_list[i] <> prev_list[i]:
2230                                     i = i - 1
2231                                     break
2232
2233                             paths.append ("/".join(cur_list[:i + 1]))
2234
2235                         self.previousDepotPaths = paths
2236
2237             if p4Change > 0:
2238                 self.depotPaths = sorted(self.previousDepotPaths)
2239                 self.changeRange = "@%s,#head" % p4Change
2240                 if not self.detectBranches:
2241                     self.initialParent = parseRevision(self.branch)
2242                 if not self.silent and not self.detectBranches:
2243                     print "Performing incremental import into %s git branch" % self.branch
2244
2245         if not self.branch.startswith("refs/"):
2246             self.branch = "refs/heads/" + self.branch
2247
2248         if len(args) == 0 and self.depotPaths:
2249             if not self.silent:
2250                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2251         else:
2252             if self.depotPaths and self.depotPaths != args:
2253                 print ("previous import used depot path %s and now %s was specified. "
2254                        "This doesn't work!" % (' '.join (self.depotPaths),
2255                                                ' '.join (args)))
2256                 sys.exit(1)
2257
2258             self.depotPaths = sorted(args)
2259
2260         revision = ""
2261         self.users = {}
2262
2263         # Make sure no revision specifiers are used when --changesfile
2264         # is specified.
2265         bad_changesfile = False
2266         if len(self.changesFile) > 0:
2267             for p in self.depotPaths:
2268                 if p.find("@") >= 0 or p.find("#") >= 0:
2269                     bad_changesfile = True
2270                     break
2271         if bad_changesfile:
2272             die("Option --changesfile is incompatible with revision specifiers")
2273
2274         newPaths = []
2275         for p in self.depotPaths:
2276             if p.find("@") != -1:
2277                 atIdx = p.index("@")
2278                 self.changeRange = p[atIdx:]
2279                 if self.changeRange == "@all":
2280                     self.changeRange = ""
2281                 elif ',' not in self.changeRange:
2282                     revision = self.changeRange
2283                     self.changeRange = ""
2284                 p = p[:atIdx]
2285             elif p.find("#") != -1:
2286                 hashIdx = p.index("#")
2287                 revision = p[hashIdx:]
2288                 p = p[:hashIdx]
2289             elif self.previousDepotPaths == []:
2290                 # pay attention to changesfile, if given, else import
2291                 # the entire p4 tree at the head revision
2292                 if len(self.changesFile) == 0:
2293                     revision = "#head"
2294
2295             p = re.sub ("\.\.\.$", "", p)
2296             if not p.endswith("/"):
2297                 p += "/"
2298
2299             newPaths.append(p)
2300
2301         self.depotPaths = newPaths
2302
2303
2304         self.loadUserMapFromCache()
2305         self.labels = {}
2306         if self.detectLabels:
2307             self.getLabels();
2308
2309         if self.detectBranches:
2310             ## FIXME - what's a P4 projectName ?
2311             self.projectName = self.guessProjectName()
2312
2313             if self.hasOrigin:
2314                 self.getBranchMappingFromGitBranches()
2315             else:
2316                 self.getBranchMapping()
2317             if self.verbose:
2318                 print "p4-git branches: %s" % self.p4BranchesInGit
2319                 print "initial parents: %s" % self.initialParents
2320             for b in self.p4BranchesInGit:
2321                 if b != "master":
2322
2323                     ## FIXME
2324                     b = b[len(self.projectName):]
2325                 self.createdBranches.add(b)
2326
2327         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2328
2329         importProcess = subprocess.Popen(["git", "fast-import"],
2330                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2331                                          stderr=subprocess.PIPE);
2332         self.gitOutput = importProcess.stdout
2333         self.gitStream = importProcess.stdin
2334         self.gitError = importProcess.stderr
2335
2336         if revision:
2337             self.importHeadRevision(revision)
2338         else:
2339             changes = []
2340
2341             if len(self.changesFile) > 0:
2342                 output = open(self.changesFile).readlines()
2343                 changeSet = set()
2344                 for line in output:
2345                     changeSet.add(int(line))
2346
2347                 for change in changeSet:
2348                     changes.append(change)
2349
2350                 changes.sort()
2351             else:
2352                 # catch "git-p4 sync" with no new branches, in a repo that
2353                 # does not have any existing git-p4 branches
2354                 if len(args) == 0 and not self.p4BranchesInGit:
2355                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2356                 if self.verbose:
2357                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2358                                                               self.changeRange)
2359                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2360
2361                 if len(self.maxChanges) > 0:
2362                     changes = changes[:min(int(self.maxChanges), len(changes))]
2363
2364             if len(changes) == 0:
2365                 if not self.silent:
2366                     print "No changes to import!"
2367                 return True
2368
2369             if not self.silent and not self.detectBranches:
2370                 print "Import destination: %s" % self.branch
2371
2372             self.updatedBranches = set()
2373
2374             self.importChanges(changes)
2375
2376             if not self.silent:
2377                 print ""
2378                 if len(self.updatedBranches) > 0:
2379                     sys.stdout.write("Updated branches: ")
2380                     for b in self.updatedBranches:
2381                         sys.stdout.write("%s " % b)
2382                     sys.stdout.write("\n")
2383
2384         self.gitStream.close()
2385         if importProcess.wait() != 0:
2386             die("fast-import failed: %s" % self.gitError.read())
2387         self.gitOutput.close()
2388         self.gitError.close()
2389
2390         # Cleanup temporary branches created during import
2391         if self.tempBranches != []:
2392             for branch in self.tempBranches:
2393                 read_pipe("git update-ref -d %s" % branch)
2394             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
2395
2396         return True
2397
2398 class P4Rebase(Command):
2399     def __init__(self):
2400         Command.__init__(self)
2401         self.options = [ ]
2402         self.description = ("Fetches the latest revision from perforce and "
2403                             + "rebases the current work (branch) against it")
2404         self.verbose = False
2405
2406     def run(self, args):
2407         sync = P4Sync()
2408         sync.run([])
2409
2410         return self.rebase()
2411
2412     def rebase(self):
2413         if os.system("git update-index --refresh") != 0:
2414             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2415         if len(read_pipe("git diff-index HEAD --")) > 0:
2416             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2417
2418         [upstream, settings] = findUpstreamBranchPoint()
2419         if len(upstream) == 0:
2420             die("Cannot find upstream branchpoint for rebase")
2421
2422         # the branchpoint may be p4/foo~3, so strip off the parent
2423         upstream = re.sub("~[0-9]+$", "", upstream)
2424
2425         print "Rebasing the current branch onto %s" % upstream
2426         oldHead = read_pipe("git rev-parse HEAD").strip()
2427         system("git rebase %s" % upstream)
2428         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2429         return True
2430
2431 class P4Clone(P4Sync):
2432     def __init__(self):
2433         P4Sync.__init__(self)
2434         self.description = "Creates a new git repository and imports from Perforce into it"
2435         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2436         self.options += [
2437             optparse.make_option("--destination", dest="cloneDestination",
2438                                  action='store', default=None,
2439                                  help="where to leave result of the clone"),
2440             optparse.make_option("-/", dest="cloneExclude",
2441                                  action="append", type="string",
2442                                  help="exclude depot path"),
2443             optparse.make_option("--bare", dest="cloneBare",
2444                                  action="store_true", default=False),
2445         ]
2446         self.cloneDestination = None
2447         self.needsGit = False
2448         self.cloneBare = False
2449
2450     # This is required for the "append" cloneExclude action
2451     def ensure_value(self, attr, value):
2452         if not hasattr(self, attr) or getattr(self, attr) is None:
2453             setattr(self, attr, value)
2454         return getattr(self, attr)
2455
2456     def defaultDestination(self, args):
2457         ## TODO: use common prefix of args?
2458         depotPath = args[0]
2459         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2460         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2461         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2462         depotDir = re.sub(r"/$", "", depotDir)
2463         return os.path.split(depotDir)[1]
2464
2465     def run(self, args):
2466         if len(args) < 1:
2467             return False
2468
2469         if self.keepRepoPath and not self.cloneDestination:
2470             sys.stderr.write("Must specify destination for --keep-path\n")
2471             sys.exit(1)
2472
2473         depotPaths = args
2474
2475         if not self.cloneDestination and len(depotPaths) > 1:
2476             self.cloneDestination = depotPaths[-1]
2477             depotPaths = depotPaths[:-1]
2478
2479         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2480         for p in depotPaths:
2481             if not p.startswith("//"):
2482                 return False
2483
2484         if not self.cloneDestination:
2485             self.cloneDestination = self.defaultDestination(args)
2486
2487         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2488
2489         if not os.path.exists(self.cloneDestination):
2490             os.makedirs(self.cloneDestination)
2491         chdir(self.cloneDestination)
2492
2493         init_cmd = [ "git", "init" ]
2494         if self.cloneBare:
2495             init_cmd.append("--bare")
2496         subprocess.check_call(init_cmd)
2497
2498         if not P4Sync.run(self, depotPaths):
2499             return False
2500         if self.branch != "master":
2501             if self.importIntoRemotes:
2502                 masterbranch = "refs/remotes/p4/master"
2503             else:
2504                 masterbranch = "refs/heads/p4/master"
2505             if gitBranchExists(masterbranch):
2506                 system("git branch master %s" % masterbranch)
2507                 if not self.cloneBare:
2508                     system("git checkout -f")
2509             else:
2510                 print "Could not detect main branch. No checkout/master branch created."
2511
2512         return True
2513
2514 class P4Branches(Command):
2515     def __init__(self):
2516         Command.__init__(self)
2517         self.options = [ ]
2518         self.description = ("Shows the git branches that hold imports and their "
2519                             + "corresponding perforce depot paths")
2520         self.verbose = False
2521
2522     def run(self, args):
2523         if originP4BranchesExist():
2524             createOrUpdateBranchesFromOrigin()
2525
2526         cmdline = "git rev-parse --symbolic "
2527         cmdline += " --remotes"
2528
2529         for line in read_pipe_lines(cmdline):
2530             line = line.strip()
2531
2532             if not line.startswith('p4/') or line == "p4/HEAD":
2533                 continue
2534             branch = line
2535
2536             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2537             settings = extractSettingsGitLog(log)
2538
2539             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2540         return True
2541
2542 class HelpFormatter(optparse.IndentedHelpFormatter):
2543     def __init__(self):
2544         optparse.IndentedHelpFormatter.__init__(self)
2545
2546     def format_description(self, description):
2547         if description:
2548             return description + "\n"
2549         else:
2550             return ""
2551
2552 def printUsage(commands):
2553     print "usage: %s <command> [options]" % sys.argv[0]
2554     print ""
2555     print "valid commands: %s" % ", ".join(commands)
2556     print ""
2557     print "Try %s <command> --help for command specific help." % sys.argv[0]
2558     print ""
2559
2560 commands = {
2561     "debug" : P4Debug,
2562     "submit" : P4Submit,
2563     "commit" : P4Submit,
2564     "sync" : P4Sync,
2565     "rebase" : P4Rebase,
2566     "clone" : P4Clone,
2567     "rollback" : P4RollBack,
2568     "branches" : P4Branches
2569 }
2570
2571
2572 def main():
2573     if len(sys.argv[1:]) == 0:
2574         printUsage(commands.keys())
2575         sys.exit(2)
2576
2577     cmd = ""
2578     cmdName = sys.argv[1]
2579     try:
2580         klass = commands[cmdName]
2581         cmd = klass()
2582     except KeyError:
2583         print "unknown command %s" % cmdName
2584         print ""
2585         printUsage(commands.keys())
2586         sys.exit(2)
2587
2588     options = cmd.options
2589     cmd.gitdir = os.environ.get("GIT_DIR", None)
2590
2591     args = sys.argv[2:]
2592
2593     if len(options) > 0:
2594         if cmd.needsGit:
2595             options.append(optparse.make_option("--git-dir", dest="gitdir"))
2596
2597         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2598                                        options,
2599                                        description = cmd.description,
2600                                        formatter = HelpFormatter())
2601
2602         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2603     global verbose
2604     verbose = cmd.verbose
2605     if cmd.needsGit:
2606         if cmd.gitdir == None:
2607             cmd.gitdir = os.path.abspath(".git")
2608             if not isValidGitDir(cmd.gitdir):
2609                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2610                 if os.path.exists(cmd.gitdir):
2611                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2612                     if len(cdup) > 0:
2613                         chdir(cdup);
2614
2615         if not isValidGitDir(cmd.gitdir):
2616             if isValidGitDir(cmd.gitdir + "/.git"):
2617                 cmd.gitdir += "/.git"
2618             else:
2619                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2620
2621         os.environ["GIT_DIR"] = cmd.gitdir
2622
2623     if not cmd.run(args):
2624         parser.print_help()
2625         sys.exit(2)
2626
2627
2628 if __name__ == '__main__':
2629     main()