contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19 def die(msg):
  20     if verbose:
  21         raise Exception(msg)
  22     else:
  23         sys.stderr.write(msg + "\n")
  24         sys.exit(1)
  25
  26 def write_pipe(c, str):
  27     if verbose:
  28         sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30     pipe = os.popen(c, 'w')
  31     val = pipe.write(str)
  32     if pipe.close():
  33         die('Command failed: %s' % c)
  34
  35     return val
  36
  37 def read_pipe(c, ignore_error=False):
  38     if verbose:
  39         sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41     pipe = os.popen(c, 'rb')
  42     val = pipe.read()
  43     if pipe.close() and not ignore_error:
  44         die('Command failed: %s' % c)
  45
  46     return val
  47
  48
  49 def read_pipe_lines(c):
  50     if verbose:
  51         sys.stderr.write('Reading pipe: %s\n' % c)
  52     ## todo: check return status
  53     pipe = os.popen(c, 'rb')
  54     val = pipe.readlines()
  55     if pipe.close():
  56         die('Command failed: %s' % c)
  57
  58     return val
  59
  60 def p4_read_pipe_lines(c):
  61     """Specifically invoke p4 on the command supplied. """
  62     real_cmd = "%s %s" % ("p4", c)
  63     if verbose:
  64         print real_cmd
  65     return read_pipe_lines(real_cmd)
  66
  67 def system(cmd):
  68     if verbose:
  69         sys.stderr.write("executing %s\n" % cmd)
  70     if os.system(cmd) != 0:
  71         die("command failed: %s" % cmd)
  72
  73 def isP4Exec(kind):
  74     """Determine if a Perforce 'kind' should have execute permission
  75
  76     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  77     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  78     a plus sign, it is also executable"""
  79     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  80
  81 def setP4ExecBit(file, mode):
  82     # Reopens an already open file and changes the execute bit to match
  83     # the execute bit setting in the passed in mode.
  84
  85     p4Type = "+x"
  86
  87     if not isModeExec(mode):
  88         p4Type = getP4OpenedType(file)
  89         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  90         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  91         if p4Type[-1] == "+":
  92             p4Type = p4Type[0:-1]
  93
  94     system("p4 reopen -t %s %s" % (p4Type, file))
  95
  96 def getP4OpenedType(file):
  97     # Returns the perforce file type for the given file.
  98
  99     result = read_pipe("p4 opened %s" % file)
 100     match = re.match(".*\((.+)\)\r?$", result)
 101     if match:
 102         return match.group(1)
 103     else:
 104         die("Could not determine file type for %s (result: '%s')" % (file, result))
 105
 106 def diffTreePattern():
 107     # This is a simple generator for the diff tree regex pattern. This could be
 108     # a class variable if this and parseDiffTreeEntry were a part of a class.
 109     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 110     while True:
 111         yield pattern
 112
 113 def parseDiffTreeEntry(entry):
 114     """Parses a single diff tree entry into its component elements.
 115
 116     See git-diff-tree(1) manpage for details about the format of the diff
 117     output. This method returns a dictionary with the following elements:
 118
 119     src_mode - The mode of the source file
 120     dst_mode - The mode of the destination file
 121     src_sha1 - The sha1 for the source file
 122     dst_sha1 - The sha1 fr the destination file
 123     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 124     status_score - The score for the status (applicable for 'C' and 'R'
 125                    statuses). This is None if there is no score.
 126     src - The path for the source file.
 127     dst - The path for the destination file. This is only present for
 128           copy or renames. If it is not present, this is None.
 129
 130     If the pattern is not matched, None is returned."""
 131
 132     match = diffTreePattern().next().match(entry)
 133     if match:
 134         return {
 135             'src_mode': match.group(1),
 136             'dst_mode': match.group(2),
 137             'src_sha1': match.group(3),
 138             'dst_sha1': match.group(4),
 139             'status': match.group(5),
 140             'status_score': match.group(6),
 141             'src': match.group(7),
 142             'dst': match.group(10)
 143         }
 144     return None
 145
 146 def isModeExec(mode):
 147     # Returns True if the given git mode represents an executable file,
 148     # otherwise False.
 149     return mode[-3:] == "755"
 150
 151 def isModeExecChanged(src_mode, dst_mode):
 152     return isModeExec(src_mode) != isModeExec(dst_mode)
 153
 154 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 155     cmd = "p4 -G %s" % cmd
 156     if verbose:
 157         sys.stderr.write("Opening pipe: %s\n" % cmd)
 158
 159     # Use a temporary file to avoid deadlocks without
 160     # subprocess.communicate(), which would put another copy
 161     # of stdout into memory.
 162     stdin_file = None
 163     if stdin is not None:
 164         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 165         stdin_file.write(stdin)
 166         stdin_file.flush()
 167         stdin_file.seek(0)
 168
 169     p4 = subprocess.Popen(cmd, shell=True,
 170                           stdin=stdin_file,
 171                           stdout=subprocess.PIPE)
 172
 173     result = []
 174     try:
 175         while True:
 176             entry = marshal.load(p4.stdout)
 177             result.append(entry)
 178     except EOFError:
 179         pass
 180     exitCode = p4.wait()
 181     if exitCode != 0:
 182         entry = {}
 183         entry["p4ExitCode"] = exitCode
 184         result.append(entry)
 185
 186     return result
 187
 188 def p4Cmd(cmd):
 189     list = p4CmdList(cmd)
 190     result = {}
 191     for entry in list:
 192         result.update(entry)
 193     return result;
 194
 195 def p4Where(depotPath):
 196     if not depotPath.endswith("/"):
 197         depotPath += "/"
 198     output = p4Cmd("where %s..." % depotPath)
 199     if output["code"] == "error":
 200         return ""
 201     clientPath = ""
 202     if "path" in output:
 203         clientPath = output.get("path")
 204     elif "data" in output:
 205         data = output.get("data")
 206         lastSpace = data.rfind(" ")
 207         clientPath = data[lastSpace + 1:]
 208
 209     if clientPath.endswith("..."):
 210         clientPath = clientPath[:-3]
 211     return clientPath
 212
 213 def currentGitBranch():
 214     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 215
 216 def isValidGitDir(path):
 217     if (os.path.exists(path + "/HEAD")
 218         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 219         return True;
 220     return False
 221
 222 def parseRevision(ref):
 223     return read_pipe("git rev-parse %s" % ref).strip()
 224
 225 def extractLogMessageFromGitCommit(commit):
 226     logMessage = ""
 227
 228     ## fixme: title is first line of commit, not 1st paragraph.
 229     foundTitle = False
 230     for log in read_pipe_lines("git cat-file commit %s" % commit):
 231        if not foundTitle:
 232            if len(log) == 1:
 233                foundTitle = True
 234            continue
 235
 236        logMessage += log
 237     return logMessage
 238
 239 def extractSettingsGitLog(log):
 240     values = {}
 241     for line in log.split("\n"):
 242         line = line.strip()
 243         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 244         if not m:
 245             continue
 246
 247         assignments = m.group(1).split (':')
 248         for a in assignments:
 249             vals = a.split ('=')
 250             key = vals[0].strip()
 251             val = ('='.join (vals[1:])).strip()
 252             if val.endswith ('\"') and val.startswith('"'):
 253                 val = val[1:-1]
 254
 255             values[key] = val
 256
 257     paths = values.get("depot-paths")
 258     if not paths:
 259         paths = values.get("depot-path")
 260     if paths:
 261         values['depot-paths'] = paths.split(',')
 262     return values
 263
 264 def gitBranchExists(branch):
 265     proc = subprocess.Popen(["git", "rev-parse", branch],
 266                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 267     return proc.wait() == 0;
 268
 269 def gitConfig(key):
 270     return read_pipe("git config %s" % key, ignore_error=True).strip()
 271
 272 def p4BranchesInGit(branchesAreInRemotes = True):
 273     branches = {}
 274
 275     cmdline = "git rev-parse --symbolic "
 276     if branchesAreInRemotes:
 277         cmdline += " --remotes"
 278     else:
 279         cmdline += " --branches"
 280
 281     for line in read_pipe_lines(cmdline):
 282         line = line.strip()
 283
 284         ## only import to p4/
 285         if not line.startswith('p4/') or line == "p4/HEAD":
 286             continue
 287         branch = line
 288
 289         # strip off p4
 290         branch = re.sub ("^p4/", "", line)
 291
 292         branches[branch] = parseRevision(line)
 293     return branches
 294
 295 def findUpstreamBranchPoint(head = "HEAD"):
 296     branches = p4BranchesInGit()
 297     # map from depot-path to branch name
 298     branchByDepotPath = {}
 299     for branch in branches.keys():
 300         tip = branches[branch]
 301         log = extractLogMessageFromGitCommit(tip)
 302         settings = extractSettingsGitLog(log)
 303         if settings.has_key("depot-paths"):
 304             paths = ",".join(settings["depot-paths"])
 305             branchByDepotPath[paths] = "remotes/p4/" + branch
 306
 307     settings = None
 308     parent = 0
 309     while parent < 65535:
 310         commit = head + "~%s" % parent
 311         log = extractLogMessageFromGitCommit(commit)
 312         settings = extractSettingsGitLog(log)
 313         if settings.has_key("depot-paths"):
 314             paths = ",".join(settings["depot-paths"])
 315             if branchByDepotPath.has_key(paths):
 316                 return [branchByDepotPath[paths], settings]
 317
 318         parent = parent + 1
 319
 320     return ["", settings]
 321
 322 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 323     if not silent:
 324         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 325                % localRefPrefix)
 326
 327     originPrefix = "origin/p4/"
 328
 329     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 330         line = line.strip()
 331         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 332             continue
 333
 334         headName = line[len(originPrefix):]
 335         remoteHead = localRefPrefix + headName
 336         originHead = line
 337
 338         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 339         if (not original.has_key('depot-paths')
 340             or not original.has_key('change')):
 341             continue
 342
 343         update = False
 344         if not gitBranchExists(remoteHead):
 345             if verbose:
 346                 print "creating %s" % remoteHead
 347             update = True
 348         else:
 349             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 350             if settings.has_key('change') > 0:
 351                 if settings['depot-paths'] == original['depot-paths']:
 352                     originP4Change = int(original['change'])
 353                     p4Change = int(settings['change'])
 354                     if originP4Change > p4Change:
 355                         print ("%s (%s) is newer than %s (%s). "
 356                                "Updating p4 branch from origin."
 357                                % (originHead, originP4Change,
 358                                   remoteHead, p4Change))
 359                         update = True
 360                 else:
 361                     print ("Ignoring: %s was imported from %s while "
 362                            "%s was imported from %s"
 363                            % (originHead, ','.join(original['depot-paths']),
 364                               remoteHead, ','.join(settings['depot-paths'])))
 365
 366         if update:
 367             system("git update-ref %s %s" % (remoteHead, originHead))
 368
 369 def originP4BranchesExist():
 370         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 371
 372 def p4ChangesForPaths(depotPaths, changeRange):
 373     assert depotPaths
 374     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 375                                                         for p in depotPaths]))
 376
 377     changes = []
 378     for line in output:
 379         changeNum = line.split(" ")[1]
 380         changes.append(int(changeNum))
 381
 382     changes.sort()
 383     return changes
 384
 385 class Command:
 386     def __init__(self):
 387         self.usage = "usage: %prog [options]"
 388         self.needsGit = True
 389
 390 class P4Debug(Command):
 391     def __init__(self):
 392         Command.__init__(self)
 393         self.options = [
 394             optparse.make_option("--verbose", dest="verbose", action="store_true",
 395                                  default=False),
 396             ]
 397         self.description = "A tool to debug the output of p4 -G."
 398         self.needsGit = False
 399         self.verbose = False
 400
 401     def run(self, args):
 402         j = 0
 403         for output in p4CmdList(" ".join(args)):
 404             print 'Element: %d' % j
 405             j += 1
 406             print output
 407         return True
 408
 409 class P4RollBack(Command):
 410     def __init__(self):
 411         Command.__init__(self)
 412         self.options = [
 413             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 414             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 415         ]
 416         self.description = "A tool to debug the multi-branch import. Don't use :)"
 417         self.verbose = False
 418         self.rollbackLocalBranches = False
 419
 420     def run(self, args):
 421         if len(args) != 1:
 422             return False
 423         maxChange = int(args[0])
 424
 425         if "p4ExitCode" in p4Cmd("changes -m 1"):
 426             die("Problems executing p4");
 427
 428         if self.rollbackLocalBranches:
 429             refPrefix = "refs/heads/"
 430             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 431         else:
 432             refPrefix = "refs/remotes/"
 433             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 434
 435         for line in lines:
 436             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 437                 line = line.strip()
 438                 ref = refPrefix + line
 439                 log = extractLogMessageFromGitCommit(ref)
 440                 settings = extractSettingsGitLog(log)
 441
 442                 depotPaths = settings['depot-paths']
 443                 change = settings['change']
 444
 445                 changed = False
 446
 447                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 448                                                            for p in depotPaths]))) == 0:
 449                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 450                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 451                     continue
 452
 453                 while change and int(change) > maxChange:
 454                     changed = True
 455                     if self.verbose:
 456                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 457                     system("git update-ref %s \"%s^\"" % (ref, ref))
 458                     log = extractLogMessageFromGitCommit(ref)
 459                     settings =  extractSettingsGitLog(log)
 460
 461
 462                     depotPaths = settings['depot-paths']
 463                     change = settings['change']
 464
 465                 if changed:
 466                     print "%s rewound to %s" % (ref, change)
 467
 468         return True
 469
 470 class P4Submit(Command):
 471     def __init__(self):
 472         Command.__init__(self)
 473         self.options = [
 474                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 475                 optparse.make_option("--origin", dest="origin"),
 476                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 477         ]
 478         self.description = "Submit changes from git to the perforce depot."
 479         self.usage += " [name of git branch to submit into perforce depot]"
 480         self.interactive = True
 481         self.origin = ""
 482         self.detectRename = False
 483         self.verbose = False
 484         self.isWindows = (platform.system() == "Windows")
 485
 486     def check(self):
 487         if len(p4CmdList("opened ...")) > 0:
 488             die("You have files opened with perforce! Close them before starting the sync.")
 489
 490     # replaces everything between 'Description:' and the next P4 submit template field with the
 491     # commit message
 492     def prepareLogMessage(self, template, message):
 493         result = ""
 494
 495         inDescriptionSection = False
 496
 497         for line in template.split("\n"):
 498             if line.startswith("#"):
 499                 result += line + "\n"
 500                 continue
 501
 502             if inDescriptionSection:
 503                 if line.startswith("Files:"):
 504                     inDescriptionSection = False
 505                 else:
 506                     continue
 507             else:
 508                 if line.startswith("Description:"):
 509                     inDescriptionSection = True
 510                     line += "\n"
 511                     for messageLine in message.split("\n"):
 512                         line += "\t" + messageLine + "\n"
 513
 514             result += line + "\n"
 515
 516         return result
 517
 518     def prepareSubmitTemplate(self):
 519         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 520         template = ""
 521         inFilesSection = False
 522         for line in p4_read_pipe_lines("change -o"):
 523             if line.endswith("\r\n"):
 524                 line = line[:-2] + "\n"
 525             if inFilesSection:
 526                 if line.startswith("\t"):
 527                     # path starts and ends with a tab
 528                     path = line[1:]
 529                     lastTab = path.rfind("\t")
 530                     if lastTab != -1:
 531                         path = path[:lastTab]
 532                         if not path.startswith(self.depotPath):
 533                             continue
 534                 else:
 535                     inFilesSection = False
 536             else:
 537                 if line.startswith("Files:"):
 538                     inFilesSection = True
 539
 540             template += line
 541
 542         return template
 543
 544     def applyCommit(self, id):
 545         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 546         diffOpts = ("", "-M")[self.detectRename]
 547         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 548         filesToAdd = set()
 549         filesToDelete = set()
 550         editedFiles = set()
 551         filesToChangeExecBit = {}
 552         for line in diff:
 553             diff = parseDiffTreeEntry(line)
 554             modifier = diff['status']
 555             path = diff['src']
 556             if modifier == "M":
 557                 system("p4 edit \"%s\"" % path)
 558                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 559                     filesToChangeExecBit[path] = diff['dst_mode']
 560                 editedFiles.add(path)
 561             elif modifier == "A":
 562                 filesToAdd.add(path)
 563                 filesToChangeExecBit[path] = diff['dst_mode']
 564                 if path in filesToDelete:
 565                     filesToDelete.remove(path)
 566             elif modifier == "D":
 567                 filesToDelete.add(path)
 568                 if path in filesToAdd:
 569                     filesToAdd.remove(path)
 570             elif modifier == "R":
 571                 src, dest = diff['src'], diff['dst']
 572                 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 573                 system("p4 edit \"%s\"" % (dest))
 574                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 575                     filesToChangeExecBit[dest] = diff['dst_mode']
 576                 os.unlink(dest)
 577                 editedFiles.add(dest)
 578                 filesToDelete.add(src)
 579             else:
 580                 die("unknown modifier %s for %s" % (modifier, path))
 581
 582         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 583         patchcmd = diffcmd + " | git apply "
 584         tryPatchCmd = patchcmd + "--check -"
 585         applyPatchCmd = patchcmd + "--check --apply -"
 586
 587         if os.system(tryPatchCmd) != 0:
 588             print "Unfortunately applying the change failed!"
 589             print "What do you want to do?"
 590             response = "x"
 591             while response != "s" and response != "a" and response != "w":
 592                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 593                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 594             if response == "s":
 595                 print "Skipping! Good luck with the next patches..."
 596                 for f in editedFiles:
 597                     system("p4 revert \"%s\"" % f);
 598                 for f in filesToAdd:
 599                     system("rm %s" %f)
 600                 return
 601             elif response == "a":
 602                 os.system(applyPatchCmd)
 603                 if len(filesToAdd) > 0:
 604                     print "You may also want to call p4 add on the following files:"
 605                     print " ".join(filesToAdd)
 606                 if len(filesToDelete):
 607                     print "The following files should be scheduled for deletion with p4 delete:"
 608                     print " ".join(filesToDelete)
 609                 die("Please resolve and submit the conflict manually and "
 610                     + "continue afterwards with git-p4 submit --continue")
 611             elif response == "w":
 612                 system(diffcmd + " > patch.txt")
 613                 print "Patch saved to patch.txt in %s !" % self.clientPath
 614                 die("Please resolve and submit the conflict manually and "
 615                     "continue afterwards with git-p4 submit --continue")
 616
 617         system(applyPatchCmd)
 618
 619         for f in filesToAdd:
 620             system("p4 add \"%s\"" % f)
 621         for f in filesToDelete:
 622             system("p4 revert \"%s\"" % f)
 623             system("p4 delete \"%s\"" % f)
 624
 625         # Set/clear executable bits
 626         for f in filesToChangeExecBit.keys():
 627             mode = filesToChangeExecBit[f]
 628             setP4ExecBit(f, mode)
 629
 630         logMessage = extractLogMessageFromGitCommit(id)
 631         logMessage = logMessage.strip()
 632
 633         template = self.prepareSubmitTemplate()
 634
 635         if self.interactive:
 636             submitTemplate = self.prepareLogMessage(template, logMessage)
 637             if os.environ.has_key("P4DIFF"):
 638                 del(os.environ["P4DIFF"])
 639             diff = read_pipe("p4 diff -du ...")
 640
 641             newdiff = ""
 642             for newFile in filesToAdd:
 643                 newdiff += "==== new file ====\n"
 644                 newdiff += "--- /dev/null\n"
 645                 newdiff += "+++ %s\n" % newFile
 646                 f = open(newFile, "r")
 647                 for line in f.readlines():
 648                     newdiff += "+" + line
 649                 f.close()
 650
 651             separatorLine = "######## everything below this line is just the diff #######\n"
 652
 653             [handle, fileName] = tempfile.mkstemp()
 654             tmpFile = os.fdopen(handle, "w+")
 655             if self.isWindows:
 656                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 657                 separatorLine = separatorLine.replace("\n", "\r\n")
 658                 newdiff = newdiff.replace("\n", "\r\n")
 659             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 660             tmpFile.close()
 661             defaultEditor = "vi"
 662             if platform.system() == "Windows":
 663                 defaultEditor = "notepad"
 664             if os.environ.has_key("P4EDITOR"):
 665                 editor = os.environ.get("P4EDITOR")
 666             else:
 667                 editor = os.environ.get("EDITOR", defaultEditor);
 668             system(editor + " " + fileName)
 669             tmpFile = open(fileName, "rb")
 670             message = tmpFile.read()
 671             tmpFile.close()
 672             os.remove(fileName)
 673             submitTemplate = message[:message.index(separatorLine)]
 674             if self.isWindows:
 675                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 676
 677             write_pipe("p4 submit -i", submitTemplate)
 678         else:
 679             fileName = "submit.txt"
 680             file = open(fileName, "w+")
 681             file.write(self.prepareLogMessage(template, logMessage))
 682             file.close()
 683             print ("Perforce submit template written as %s. "
 684                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 685                    % (fileName, fileName))
 686
 687     def run(self, args):
 688         if len(args) == 0:
 689             self.master = currentGitBranch()
 690             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 691                 die("Detecting current git branch failed!")
 692         elif len(args) == 1:
 693             self.master = args[0]
 694         else:
 695             return False
 696
 697         allowSubmit = gitConfig("git-p4.allowSubmit")
 698         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 699             die("%s is not in git-p4.allowSubmit" % self.master)
 700
 701         [upstream, settings] = findUpstreamBranchPoint()
 702         self.depotPath = settings['depot-paths'][0]
 703         if len(self.origin) == 0:
 704             self.origin = upstream
 705
 706         if self.verbose:
 707             print "Origin branch is " + self.origin
 708
 709         if len(self.depotPath) == 0:
 710             print "Internal error: cannot locate perforce depot path from existing branches"
 711             sys.exit(128)
 712
 713         self.clientPath = p4Where(self.depotPath)
 714
 715         if len(self.clientPath) == 0:
 716             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 717             sys.exit(128)
 718
 719         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 720         self.oldWorkingDirectory = os.getcwd()
 721
 722         os.chdir(self.clientPath)
 723         print "Syncronizing p4 checkout..."
 724         system("p4 sync ...")
 725
 726         self.check()
 727
 728         commits = []
 729         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 730             commits.append(line.strip())
 731         commits.reverse()
 732
 733         while len(commits) > 0:
 734             commit = commits[0]
 735             commits = commits[1:]
 736             self.applyCommit(commit)
 737             if not self.interactive:
 738                 break
 739
 740         if len(commits) == 0:
 741             print "All changes applied!"
 742             os.chdir(self.oldWorkingDirectory)
 743
 744             sync = P4Sync()
 745             sync.run([])
 746
 747             rebase = P4Rebase()
 748             rebase.rebase()
 749
 750         return True
 751
 752 class P4Sync(Command):
 753     def __init__(self):
 754         Command.__init__(self)
 755         self.options = [
 756                 optparse.make_option("--branch", dest="branch"),
 757                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 758                 optparse.make_option("--changesfile", dest="changesFile"),
 759                 optparse.make_option("--silent", dest="silent", action="store_true"),
 760                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 761                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 762                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 763                                      help="Import into refs/heads/ , not refs/remotes"),
 764                 optparse.make_option("--max-changes", dest="maxChanges"),
 765                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 766                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 767                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 768                                      help="Only sync files that are included in the Perforce Client Spec")
 769         ]
 770         self.description = """Imports from Perforce into a git repository.\n
 771     example:
 772     //depot/my/project/ -- to import the current head
 773     //depot/my/project/@all -- to import everything
 774     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 775
 776     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 777
 778         self.usage += " //depot/path[@revRange]"
 779         self.silent = False
 780         self.createdBranches = Set()
 781         self.committedChanges = Set()
 782         self.branch = ""
 783         self.detectBranches = False
 784         self.detectLabels = False
 785         self.changesFile = ""
 786         self.syncWithOrigin = True
 787         self.verbose = False
 788         self.importIntoRemotes = True
 789         self.maxChanges = ""
 790         self.isWindows = (platform.system() == "Windows")
 791         self.keepRepoPath = False
 792         self.depotPaths = None
 793         self.p4BranchesInGit = []
 794         self.cloneExclude = []
 795         self.useClientSpec = False
 796         self.clientSpecDirs = []
 797
 798         if gitConfig("git-p4.syncFromOrigin") == "false":
 799             self.syncWithOrigin = False
 800
 801     def extractFilesFromCommit(self, commit):
 802         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 803                              for path in self.cloneExclude]
 804         files = []
 805         fnum = 0
 806         while commit.has_key("depotFile%s" % fnum):
 807             path =  commit["depotFile%s" % fnum]
 808
 809             if [p for p in self.cloneExclude
 810                 if path.startswith (p)]:
 811                 found = False
 812             else:
 813                 found = [p for p in self.depotPaths
 814                          if path.startswith (p)]
 815             if not found:
 816                 fnum = fnum + 1
 817                 continue
 818
 819             file = {}
 820             file["path"] = path
 821             file["rev"] = commit["rev%s" % fnum]
 822             file["action"] = commit["action%s" % fnum]
 823             file["type"] = commit["type%s" % fnum]
 824             files.append(file)
 825             fnum = fnum + 1
 826         return files
 827
 828     def stripRepoPath(self, path, prefixes):
 829         if self.keepRepoPath:
 830             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 831
 832         for p in prefixes:
 833             if path.startswith(p):
 834                 path = path[len(p):]
 835
 836         return path
 837
 838     def splitFilesIntoBranches(self, commit):
 839         branches = {}
 840         fnum = 0
 841         while commit.has_key("depotFile%s" % fnum):
 842             path =  commit["depotFile%s" % fnum]
 843             found = [p for p in self.depotPaths
 844                      if path.startswith (p)]
 845             if not found:
 846                 fnum = fnum + 1
 847                 continue
 848
 849             file = {}
 850             file["path"] = path
 851             file["rev"] = commit["rev%s" % fnum]
 852             file["action"] = commit["action%s" % fnum]
 853             file["type"] = commit["type%s" % fnum]
 854             fnum = fnum + 1
 855
 856             relPath = self.stripRepoPath(path, self.depotPaths)
 857
 858             for branch in self.knownBranches.keys():
 859
 860                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 861                 if relPath.startswith(branch + "/"):
 862                     if branch not in branches:
 863                         branches[branch] = []
 864                     branches[branch].append(file)
 865                     break
 866
 867         return branches
 868
 869     ## Should move this out, doesn't use SELF.
 870     def readP4Files(self, files):
 871         filesForCommit = []
 872         filesToRead = []
 873
 874         for f in files:
 875             includeFile = True
 876             for val in self.clientSpecDirs:
 877                 if f['path'].startswith(val[0]):
 878                     if val[1] <= 0:
 879                         includeFile = False
 880                     break
 881
 882             if includeFile:
 883                 filesForCommit.append(f)
 884                 if f['action'] != 'delete':
 885                     filesToRead.append(f)
 886
 887         filedata = []
 888         if len(filesToRead) > 0:
 889             filedata = p4CmdList('-x - print',
 890                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 891                                                   for f in filesToRead]),
 892                                  stdin_mode='w+')
 893
 894             if "p4ExitCode" in filedata[0]:
 895                 die("Problems executing p4. Error: [%d]."
 896                     % (filedata[0]['p4ExitCode']));
 897
 898         j = 0;
 899         contents = {}
 900         while j < len(filedata):
 901             stat = filedata[j]
 902             j += 1
 903             text = [];
 904             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 905                 text.append(filedata[j]['data'])
 906                 j += 1
 907             text = ''.join(text)
 908
 909             if not stat.has_key('depotFile'):
 910                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 911                 continue
 912
 913             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 914                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 915             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 916                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 917
 918             contents[stat['depotFile']] = text
 919
 920         for f in filesForCommit:
 921             path = f['path']
 922             if contents.has_key(path):
 923                 f['data'] = contents[path]
 924
 925         return filesForCommit
 926
 927     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 928         epoch = details["time"]
 929         author = details["user"]
 930
 931         if self.verbose:
 932             print "commit into %s" % branch
 933
 934         # start with reading files; if that fails, we should not
 935         # create a commit.
 936         new_files = []
 937         for f in files:
 938             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 939                 new_files.append (f)
 940             else:
 941                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 942         files = self.readP4Files(new_files)
 943
 944         self.gitStream.write("commit %s\n" % branch)
 945 #        gitStream.write("mark :%s\n" % details["change"])
 946         self.committedChanges.add(int(details["change"]))
 947         committer = ""
 948         if author not in self.users:
 949             self.getUserMapFromPerforceServer()
 950         if author in self.users:
 951             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 952         else:
 953             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 954
 955         self.gitStream.write("committer %s\n" % committer)
 956
 957         self.gitStream.write("data <<EOT\n")
 958         self.gitStream.write(details["desc"])
 959         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 960                              % (','.join (branchPrefixes), details["change"]))
 961         if len(details['options']) > 0:
 962             self.gitStream.write(": options = %s" % details['options'])
 963         self.gitStream.write("]\nEOT\n\n")
 964
 965         if len(parent) > 0:
 966             if self.verbose:
 967                 print "parent %s" % parent
 968             self.gitStream.write("from %s\n" % parent)
 969
 970         for file in files:
 971             if file["type"] == "apple":
 972                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 973                 continue
 974
 975             relPath = self.stripRepoPath(file['path'], branchPrefixes)
 976             if file["action"] == "delete":
 977                 self.gitStream.write("D %s\n" % relPath)
 978             else:
 979                 data = file['data']
 980
 981                 mode = "644"
 982                 if isP4Exec(file["type"]):
 983                     mode = "755"
 984                 elif file["type"] == "symlink":
 985                     mode = "120000"
 986                     # p4 print on a symlink contains "target\n", so strip it off
 987                     data = data[:-1]
 988
 989                 if self.isWindows and file["type"].endswith("text"):
 990                     data = data.replace("\r\n", "\n")
 991
 992                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 993                 self.gitStream.write("data %s\n" % len(data))
 994                 self.gitStream.write(data)
 995                 self.gitStream.write("\n")
 996
 997         self.gitStream.write("\n")
 998
 999         change = int(details["change"])
1000
1001         if self.labels.has_key(change):
1002             label = self.labels[change]
1003             labelDetails = label[0]
1004             labelRevisions = label[1]
1005             if self.verbose:
1006                 print "Change %s is labelled %s" % (change, labelDetails)
1007
1008             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1009                                                     for p in branchPrefixes]))
1010
1011             if len(files) == len(labelRevisions):
1012
1013                 cleanedFiles = {}
1014                 for info in files:
1015                     if info["action"] == "delete":
1016                         continue
1017                     cleanedFiles[info["depotFile"]] = info["rev"]
1018
1019                 if cleanedFiles == labelRevisions:
1020                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1021                     self.gitStream.write("from %s\n" % branch)
1022
1023                     owner = labelDetails["Owner"]
1024                     tagger = ""
1025                     if author in self.users:
1026                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1027                     else:
1028                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1029                     self.gitStream.write("tagger %s\n" % tagger)
1030                     self.gitStream.write("data <<EOT\n")
1031                     self.gitStream.write(labelDetails["Description"])
1032                     self.gitStream.write("EOT\n\n")
1033
1034                 else:
1035                     if not self.silent:
1036                         print ("Tag %s does not match with change %s: files do not match."
1037                                % (labelDetails["label"], change))
1038
1039             else:
1040                 if not self.silent:
1041                     print ("Tag %s does not match with change %s: file count is different."
1042                            % (labelDetails["label"], change))
1043
1044     def getUserCacheFilename(self):
1045         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1046         return home + "/.gitp4-usercache.txt"
1047
1048     def getUserMapFromPerforceServer(self):
1049         if self.userMapFromPerforceServer:
1050             return
1051         self.users = {}
1052
1053         for output in p4CmdList("users"):
1054             if not output.has_key("User"):
1055                 continue
1056             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1057
1058
1059         s = ''
1060         for (key, val) in self.users.items():
1061             s += "%s\t%s\n" % (key, val)
1062
1063         open(self.getUserCacheFilename(), "wb").write(s)
1064         self.userMapFromPerforceServer = True
1065
1066     def loadUserMapFromCache(self):
1067         self.users = {}
1068         self.userMapFromPerforceServer = False
1069         try:
1070             cache = open(self.getUserCacheFilename(), "rb")
1071             lines = cache.readlines()
1072             cache.close()
1073             for line in lines:
1074                 entry = line.strip().split("\t")
1075                 self.users[entry[0]] = entry[1]
1076         except IOError:
1077             self.getUserMapFromPerforceServer()
1078
1079     def getLabels(self):
1080         self.labels = {}
1081
1082         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1083         if len(l) > 0 and not self.silent:
1084             print "Finding files belonging to labels in %s" % `self.depotPaths`
1085
1086         for output in l:
1087             label = output["label"]
1088             revisions = {}
1089             newestChange = 0
1090             if self.verbose:
1091                 print "Querying files for label %s" % label
1092             for file in p4CmdList("files "
1093                                   +  ' '.join (["%s...@%s" % (p, label)
1094                                                 for p in self.depotPaths])):
1095                 revisions[file["depotFile"]] = file["rev"]
1096                 change = int(file["change"])
1097                 if change > newestChange:
1098                     newestChange = change
1099
1100             self.labels[newestChange] = [output, revisions]
1101
1102         if self.verbose:
1103             print "Label changes: %s" % self.labels.keys()
1104
1105     def guessProjectName(self):
1106         for p in self.depotPaths:
1107             if p.endswith("/"):
1108                 p = p[:-1]
1109             p = p[p.strip().rfind("/") + 1:]
1110             if not p.endswith("/"):
1111                p += "/"
1112             return p
1113
1114     def getBranchMapping(self):
1115         lostAndFoundBranches = set()
1116
1117         for info in p4CmdList("branches"):
1118             details = p4Cmd("branch -o %s" % info["branch"])
1119             viewIdx = 0
1120             while details.has_key("View%s" % viewIdx):
1121                 paths = details["View%s" % viewIdx].split(" ")
1122                 viewIdx = viewIdx + 1
1123                 # require standard //depot/foo/... //depot/bar/... mapping
1124                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1125                     continue
1126                 source = paths[0]
1127                 destination = paths[1]
1128                 ## HACK
1129                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1130                     source = source[len(self.depotPaths[0]):-4]
1131                     destination = destination[len(self.depotPaths[0]):-4]
1132
1133                     if destination in self.knownBranches:
1134                         if not self.silent:
1135                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1136                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1137                         continue
1138
1139                     self.knownBranches[destination] = source
1140
1141                     lostAndFoundBranches.discard(destination)
1142
1143                     if source not in self.knownBranches:
1144                         lostAndFoundBranches.add(source)
1145
1146
1147         for branch in lostAndFoundBranches:
1148             self.knownBranches[branch] = branch
1149
1150     def getBranchMappingFromGitBranches(self):
1151         branches = p4BranchesInGit(self.importIntoRemotes)
1152         for branch in branches.keys():
1153             if branch == "master":
1154                 branch = "main"
1155             else:
1156                 branch = branch[len(self.projectName):]
1157             self.knownBranches[branch] = branch
1158
1159     def listExistingP4GitBranches(self):
1160         # branches holds mapping from name to commit
1161         branches = p4BranchesInGit(self.importIntoRemotes)
1162         self.p4BranchesInGit = branches.keys()
1163         for branch in branches.keys():
1164             self.initialParents[self.refPrefix + branch] = branches[branch]
1165
1166     def updateOptionDict(self, d):
1167         option_keys = {}
1168         if self.keepRepoPath:
1169             option_keys['keepRepoPath'] = 1
1170
1171         d["options"] = ' '.join(sorted(option_keys.keys()))
1172
1173     def readOptions(self, d):
1174         self.keepRepoPath = (d.has_key('options')
1175                              and ('keepRepoPath' in d['options']))
1176
1177     def gitRefForBranch(self, branch):
1178         if branch == "main":
1179             return self.refPrefix + "master"
1180
1181         if len(branch) <= 0:
1182             return branch
1183
1184         return self.refPrefix + self.projectName + branch
1185
1186     def gitCommitByP4Change(self, ref, change):
1187         if self.verbose:
1188             print "looking in ref " + ref + " for change %s using bisect..." % change
1189
1190         earliestCommit = ""
1191         latestCommit = parseRevision(ref)
1192
1193         while True:
1194             if self.verbose:
1195                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1196             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1197             if len(next) == 0:
1198                 if self.verbose:
1199                     print "argh"
1200                 return ""
1201             log = extractLogMessageFromGitCommit(next)
1202             settings = extractSettingsGitLog(log)
1203             currentChange = int(settings['change'])
1204             if self.verbose:
1205                 print "current change %s" % currentChange
1206
1207             if currentChange == change:
1208                 if self.verbose:
1209                     print "found %s" % next
1210                 return next
1211
1212             if currentChange < change:
1213                 earliestCommit = "^%s" % next
1214             else:
1215                 latestCommit = "%s" % next
1216
1217         return ""
1218
1219     def importNewBranch(self, branch, maxChange):
1220         # make fast-import flush all changes to disk and update the refs using the checkpoint
1221         # command so that we can try to find the branch parent in the git history
1222         self.gitStream.write("checkpoint\n\n");
1223         self.gitStream.flush();
1224         branchPrefix = self.depotPaths[0] + branch + "/"
1225         range = "@1,%s" % maxChange
1226         #print "prefix" + branchPrefix
1227         changes = p4ChangesForPaths([branchPrefix], range)
1228         if len(changes) <= 0:
1229             return False
1230         firstChange = changes[0]
1231         #print "first change in branch: %s" % firstChange
1232         sourceBranch = self.knownBranches[branch]
1233         sourceDepotPath = self.depotPaths[0] + sourceBranch
1234         sourceRef = self.gitRefForBranch(sourceBranch)
1235         #print "source " + sourceBranch
1236
1237         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1238         #print "branch parent: %s" % branchParentChange
1239         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1240         if len(gitParent) > 0:
1241             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1242             #print "parent git commit: %s" % gitParent
1243
1244         self.importChanges(changes)
1245         return True
1246
1247     def importChanges(self, changes):
1248         cnt = 1
1249         for change in changes:
1250             description = p4Cmd("describe %s" % change)
1251             self.updateOptionDict(description)
1252
1253             if not self.silent:
1254                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1255                 sys.stdout.flush()
1256             cnt = cnt + 1
1257
1258             try:
1259                 if self.detectBranches:
1260                     branches = self.splitFilesIntoBranches(description)
1261                     for branch in branches.keys():
1262                         ## HACK  --hwn
1263                         branchPrefix = self.depotPaths[0] + branch + "/"
1264
1265                         parent = ""
1266
1267                         filesForCommit = branches[branch]
1268
1269                         if self.verbose:
1270                             print "branch is %s" % branch
1271
1272                         self.updatedBranches.add(branch)
1273
1274                         if branch not in self.createdBranches:
1275                             self.createdBranches.add(branch)
1276                             parent = self.knownBranches[branch]
1277                             if parent == branch:
1278                                 parent = ""
1279                             else:
1280                                 fullBranch = self.projectName + branch
1281                                 if fullBranch not in self.p4BranchesInGit:
1282                                     if not self.silent:
1283                                         print("\n    Importing new branch %s" % fullBranch);
1284                                     if self.importNewBranch(branch, change - 1):
1285                                         parent = ""
1286                                         self.p4BranchesInGit.append(fullBranch)
1287                                     if not self.silent:
1288                                         print("\n    Resuming with change %s" % change);
1289
1290                                 if self.verbose:
1291                                     print "parent determined through known branches: %s" % parent
1292
1293                         branch = self.gitRefForBranch(branch)
1294                         parent = self.gitRefForBranch(parent)
1295
1296                         if self.verbose:
1297                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1298
1299                         if len(parent) == 0 and branch in self.initialParents:
1300                             parent = self.initialParents[branch]
1301                             del self.initialParents[branch]
1302
1303                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1304                 else:
1305                     files = self.extractFilesFromCommit(description)
1306                     self.commit(description, files, self.branch, self.depotPaths,
1307                                 self.initialParent)
1308                     self.initialParent = ""
1309             except IOError:
1310                 print self.gitError.read()
1311                 sys.exit(1)
1312
1313     def importHeadRevision(self, revision):
1314         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1315
1316         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1317         details["desc"] = ("Initial import of %s from the state at revision %s"
1318                            % (' '.join(self.depotPaths), revision))
1319         details["change"] = revision
1320         newestRevision = 0
1321
1322         fileCnt = 0
1323         for info in p4CmdList("files "
1324                               +  ' '.join(["%s...%s"
1325                                            % (p, revision)
1326                                            for p in self.depotPaths])):
1327
1328             if info['code'] == 'error':
1329                 sys.stderr.write("p4 returned an error: %s\n"
1330                                  % info['data'])
1331                 sys.exit(1)
1332
1333
1334             change = int(info["change"])
1335             if change > newestRevision:
1336                 newestRevision = change
1337
1338             if info["action"] == "delete":
1339                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1340                 #fileCnt = fileCnt + 1
1341                 continue
1342
1343             for prop in ["depotFile", "rev", "action", "type" ]:
1344                 details["%s%s" % (prop, fileCnt)] = info[prop]
1345
1346             fileCnt = fileCnt + 1
1347
1348         details["change"] = newestRevision
1349         self.updateOptionDict(details)
1350         try:
1351             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1352         except IOError:
1353             print "IO error with git fast-import. Is your git version recent enough?"
1354             print self.gitError.read()
1355
1356
1357     def getClientSpec(self):
1358         specList = p4CmdList( "client -o" )
1359         temp = {}
1360         for entry in specList:
1361             for k,v in entry.iteritems():
1362                 if k.startswith("View"):
1363                     if v.startswith('"'):
1364                         start = 1
1365                     else:
1366                         start = 0
1367                     index = v.find("...")
1368                     v = v[start:index]
1369                     if v.startswith("-"):
1370                         v = v[1:]
1371                         temp[v] = -len(v)
1372                     else:
1373                         temp[v] = len(v)
1374         self.clientSpecDirs = temp.items()
1375         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1376
1377     def run(self, args):
1378         self.depotPaths = []
1379         self.changeRange = ""
1380         self.initialParent = ""
1381         self.previousDepotPaths = []
1382
1383         # map from branch depot path to parent branch
1384         self.knownBranches = {}
1385         self.initialParents = {}
1386         self.hasOrigin = originP4BranchesExist()
1387         if not self.syncWithOrigin:
1388             self.hasOrigin = False
1389
1390         if self.importIntoRemotes:
1391             self.refPrefix = "refs/remotes/p4/"
1392         else:
1393             self.refPrefix = "refs/heads/p4/"
1394
1395         if self.syncWithOrigin and self.hasOrigin:
1396             if not self.silent:
1397                 print "Syncing with origin first by calling git fetch origin"
1398             system("git fetch origin")
1399
1400         if len(self.branch) == 0:
1401             self.branch = self.refPrefix + "master"
1402             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1403                 system("git update-ref %s refs/heads/p4" % self.branch)
1404                 system("git branch -D p4");
1405             # create it /after/ importing, when master exists
1406             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1407                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1408
1409         if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1410             self.getClientSpec()
1411
1412         # TODO: should always look at previous commits,
1413         # merge with previous imports, if possible.
1414         if args == []:
1415             if self.hasOrigin:
1416                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1417             self.listExistingP4GitBranches()
1418
1419             if len(self.p4BranchesInGit) > 1:
1420                 if not self.silent:
1421                     print "Importing from/into multiple branches"
1422                 self.detectBranches = True
1423
1424             if self.verbose:
1425                 print "branches: %s" % self.p4BranchesInGit
1426
1427             p4Change = 0
1428             for branch in self.p4BranchesInGit:
1429                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1430
1431                 settings = extractSettingsGitLog(logMsg)
1432
1433                 self.readOptions(settings)
1434                 if (settings.has_key('depot-paths')
1435                     and settings.has_key ('change')):
1436                     change = int(settings['change']) + 1
1437                     p4Change = max(p4Change, change)
1438
1439                     depotPaths = sorted(settings['depot-paths'])
1440                     if self.previousDepotPaths == []:
1441                         self.previousDepotPaths = depotPaths
1442                     else:
1443                         paths = []
1444                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1445                             for i in range(0, min(len(cur), len(prev))):
1446                                 if cur[i] <> prev[i]:
1447                                     i = i - 1
1448                                     break
1449
1450                             paths.append (cur[:i + 1])
1451
1452                         self.previousDepotPaths = paths
1453
1454             if p4Change > 0:
1455                 self.depotPaths = sorted(self.previousDepotPaths)
1456                 self.changeRange = "@%s,#head" % p4Change
1457                 if not self.detectBranches:
1458                     self.initialParent = parseRevision(self.branch)
1459                 if not self.silent and not self.detectBranches:
1460                     print "Performing incremental import into %s git branch" % self.branch
1461
1462         if not self.branch.startswith("refs/"):
1463             self.branch = "refs/heads/" + self.branch
1464
1465         if len(args) == 0 and self.depotPaths:
1466             if not self.silent:
1467                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1468         else:
1469             if self.depotPaths and self.depotPaths != args:
1470                 print ("previous import used depot path %s and now %s was specified. "
1471                        "This doesn't work!" % (' '.join (self.depotPaths),
1472                                                ' '.join (args)))
1473                 sys.exit(1)
1474
1475             self.depotPaths = sorted(args)
1476
1477         revision = ""
1478         self.users = {}
1479
1480         newPaths = []
1481         for p in self.depotPaths:
1482             if p.find("@") != -1:
1483                 atIdx = p.index("@")
1484                 self.changeRange = p[atIdx:]
1485                 if self.changeRange == "@all":
1486                     self.changeRange = ""
1487                 elif ',' not in self.changeRange:
1488                     revision = self.changeRange
1489                     self.changeRange = ""
1490                 p = p[:atIdx]
1491             elif p.find("#") != -1:
1492                 hashIdx = p.index("#")
1493                 revision = p[hashIdx:]
1494                 p = p[:hashIdx]
1495             elif self.previousDepotPaths == []:
1496                 revision = "#head"
1497
1498             p = re.sub ("\.\.\.$", "", p)
1499             if not p.endswith("/"):
1500                 p += "/"
1501
1502             newPaths.append(p)
1503
1504         self.depotPaths = newPaths
1505
1506
1507         self.loadUserMapFromCache()
1508         self.labels = {}
1509         if self.detectLabels:
1510             self.getLabels();
1511
1512         if self.detectBranches:
1513             ## FIXME - what's a P4 projectName ?
1514             self.projectName = self.guessProjectName()
1515
1516             if self.hasOrigin:
1517                 self.getBranchMappingFromGitBranches()
1518             else:
1519                 self.getBranchMapping()
1520             if self.verbose:
1521                 print "p4-git branches: %s" % self.p4BranchesInGit
1522                 print "initial parents: %s" % self.initialParents
1523             for b in self.p4BranchesInGit:
1524                 if b != "master":
1525
1526                     ## FIXME
1527                     b = b[len(self.projectName):]
1528                 self.createdBranches.add(b)
1529
1530         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1531
1532         importProcess = subprocess.Popen(["git", "fast-import"],
1533                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1534                                          stderr=subprocess.PIPE);
1535         self.gitOutput = importProcess.stdout
1536         self.gitStream = importProcess.stdin
1537         self.gitError = importProcess.stderr
1538
1539         if revision:
1540             self.importHeadRevision(revision)
1541         else:
1542             changes = []
1543
1544             if len(self.changesFile) > 0:
1545                 output = open(self.changesFile).readlines()
1546                 changeSet = Set()
1547                 for line in output:
1548                     changeSet.add(int(line))
1549
1550                 for change in changeSet:
1551                     changes.append(change)
1552
1553                 changes.sort()
1554             else:
1555                 if self.verbose:
1556                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1557                                                               self.changeRange)
1558                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1559
1560                 if len(self.maxChanges) > 0:
1561                     changes = changes[:min(int(self.maxChanges), len(changes))]
1562
1563             if len(changes) == 0:
1564                 if not self.silent:
1565                     print "No changes to import!"
1566                 return True
1567
1568             if not self.silent and not self.detectBranches:
1569                 print "Import destination: %s" % self.branch
1570
1571             self.updatedBranches = set()
1572
1573             self.importChanges(changes)
1574
1575             if not self.silent:
1576                 print ""
1577                 if len(self.updatedBranches) > 0:
1578                     sys.stdout.write("Updated branches: ")
1579                     for b in self.updatedBranches:
1580                         sys.stdout.write("%s " % b)
1581                     sys.stdout.write("\n")
1582
1583         self.gitStream.close()
1584         if importProcess.wait() != 0:
1585             die("fast-import failed: %s" % self.gitError.read())
1586         self.gitOutput.close()
1587         self.gitError.close()
1588
1589         return True
1590
1591 class P4Rebase(Command):
1592     def __init__(self):
1593         Command.__init__(self)
1594         self.options = [ ]
1595         self.description = ("Fetches the latest revision from perforce and "
1596                             + "rebases the current work (branch) against it")
1597         self.verbose = False
1598
1599     def run(self, args):
1600         sync = P4Sync()
1601         sync.run([])
1602
1603         return self.rebase()
1604
1605     def rebase(self):
1606         if os.system("git update-index --refresh") != 0:
1607             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1608         if len(read_pipe("git diff-index HEAD --")) > 0:
1609             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1610
1611         [upstream, settings] = findUpstreamBranchPoint()
1612         if len(upstream) == 0:
1613             die("Cannot find upstream branchpoint for rebase")
1614
1615         # the branchpoint may be p4/foo~3, so strip off the parent
1616         upstream = re.sub("~[0-9]+$", "", upstream)
1617
1618         print "Rebasing the current branch onto %s" % upstream
1619         oldHead = read_pipe("git rev-parse HEAD").strip()
1620         system("git rebase %s" % upstream)
1621         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1622         return True
1623
1624 class P4Clone(P4Sync):
1625     def __init__(self):
1626         P4Sync.__init__(self)
1627         self.description = "Creates a new git repository and imports from Perforce into it"
1628         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1629         self.options += [
1630             optparse.make_option("--destination", dest="cloneDestination",
1631                                  action='store', default=None,
1632                                  help="where to leave result of the clone"),
1633             optparse.make_option("-/", dest="cloneExclude",
1634                                  action="append", type="string",
1635                                  help="exclude depot path")
1636         ]
1637         self.cloneDestination = None
1638         self.needsGit = False
1639
1640     # This is required for the "append" cloneExclude action
1641     def ensure_value(self, attr, value):
1642         if not hasattr(self, attr) or getattr(self, attr) is None:
1643             setattr(self, attr, value)
1644         return getattr(self, attr)
1645
1646     def defaultDestination(self, args):
1647         ## TODO: use common prefix of args?
1648         depotPath = args[0]
1649         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1650         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1651         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1652         depotDir = re.sub(r"/$", "", depotDir)
1653         return os.path.split(depotDir)[1]
1654
1655     def run(self, args):
1656         if len(args) < 1:
1657             return False
1658
1659         if self.keepRepoPath and not self.cloneDestination:
1660             sys.stderr.write("Must specify destination for --keep-path\n")
1661             sys.exit(1)
1662
1663         depotPaths = args
1664
1665         if not self.cloneDestination and len(depotPaths) > 1:
1666             self.cloneDestination = depotPaths[-1]
1667             depotPaths = depotPaths[:-1]
1668
1669         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1670         for p in depotPaths:
1671             if not p.startswith("//"):
1672                 return False
1673
1674         if not self.cloneDestination:
1675             self.cloneDestination = self.defaultDestination(args)
1676
1677         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1678         if not os.path.exists(self.cloneDestination):
1679             os.makedirs(self.cloneDestination)
1680         os.chdir(self.cloneDestination)
1681         system("git init")
1682         self.gitdir = os.getcwd() + "/.git"
1683         if not P4Sync.run(self, depotPaths):
1684             return False
1685         if self.branch != "master":
1686             if gitBranchExists("refs/remotes/p4/master"):
1687                 system("git branch master refs/remotes/p4/master")
1688                 system("git checkout -f")
1689             else:
1690                 print "Could not detect main branch. No checkout/master branch created."
1691
1692         return True
1693
1694 class P4Branches(Command):
1695     def __init__(self):
1696         Command.__init__(self)
1697         self.options = [ ]
1698         self.description = ("Shows the git branches that hold imports and their "
1699                             + "corresponding perforce depot paths")
1700         self.verbose = False
1701
1702     def run(self, args):
1703         if originP4BranchesExist():
1704             createOrUpdateBranchesFromOrigin()
1705
1706         cmdline = "git rev-parse --symbolic "
1707         cmdline += " --remotes"
1708
1709         for line in read_pipe_lines(cmdline):
1710             line = line.strip()
1711
1712             if not line.startswith('p4/') or line == "p4/HEAD":
1713                 continue
1714             branch = line
1715
1716             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1717             settings = extractSettingsGitLog(log)
1718
1719             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1720         return True
1721
1722 class HelpFormatter(optparse.IndentedHelpFormatter):
1723     def __init__(self):
1724         optparse.IndentedHelpFormatter.__init__(self)
1725
1726     def format_description(self, description):
1727         if description:
1728             return description + "\n"
1729         else:
1730             return ""
1731
1732 def printUsage(commands):
1733     print "usage: %s <command> [options]" % sys.argv[0]
1734     print ""
1735     print "valid commands: %s" % ", ".join(commands)
1736     print ""
1737     print "Try %s <command> --help for command specific help." % sys.argv[0]
1738     print ""
1739
1740 commands = {
1741     "debug" : P4Debug,
1742     "submit" : P4Submit,
1743     "commit" : P4Submit,
1744     "sync" : P4Sync,
1745     "rebase" : P4Rebase,
1746     "clone" : P4Clone,
1747     "rollback" : P4RollBack,
1748     "branches" : P4Branches
1749 }
1750
1751
1752 def main():
1753     if len(sys.argv[1:]) == 0:
1754         printUsage(commands.keys())
1755         sys.exit(2)
1756
1757     cmd = ""
1758     cmdName = sys.argv[1]
1759     try:
1760         klass = commands[cmdName]
1761         cmd = klass()
1762     except KeyError:
1763         print "unknown command %s" % cmdName
1764         print ""
1765         printUsage(commands.keys())
1766         sys.exit(2)
1767
1768     options = cmd.options
1769     cmd.gitdir = os.environ.get("GIT_DIR", None)
1770
1771     args = sys.argv[2:]
1772
1773     if len(options) > 0:
1774         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1775
1776         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1777                                        options,
1778                                        description = cmd.description,
1779                                        formatter = HelpFormatter())
1780
1781         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1782     global verbose
1783     verbose = cmd.verbose
1784     if cmd.needsGit:
1785         if cmd.gitdir == None:
1786             cmd.gitdir = os.path.abspath(".git")
1787             if not isValidGitDir(cmd.gitdir):
1788                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1789                 if os.path.exists(cmd.gitdir):
1790                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1791                     if len(cdup) > 0:
1792                         os.chdir(cdup);
1793
1794         if not isValidGitDir(cmd.gitdir):
1795             if isValidGitDir(cmd.gitdir + "/.git"):
1796                 cmd.gitdir += "/.git"
1797             else:
1798                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1799
1800         os.environ["GIT_DIR"] = cmd.gitdir
1801
1802     if not cmd.run(args):
1803         parser.print_help()
1804
1805
1806 if __name__ == '__main__':
1807     main()