contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = "%s " % "p4"
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += "-u %s " % user
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += "-P %s " % password
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += "-p %s " % port
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += "-h %s " % host
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += "-c %s " % client
  46
  47     real_cmd += "%s" % (cmd)
  48     if verbose:
  49         print real_cmd
  50     return real_cmd
  51
  52 def chdir(dir):
  53     if os.name == 'nt':
  54         os.environ['PWD']=dir
  55     os.chdir(dir)
  56
  57 def die(msg):
  58     if verbose:
  59         raise Exception(msg)
  60     else:
  61         sys.stderr.write(msg + "\n")
  62         sys.exit(1)
  63
  64 def write_pipe(c, str):
  65     if verbose:
  66         sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68     pipe = os.popen(c, 'w')
  69     val = pipe.write(str)
  70     if pipe.close():
  71         die('Command failed: %s' % c)
  72
  73     return val
  74
  75 def p4_write_pipe(c, str):
  76     real_cmd = p4_build_cmd(c)
  77     return write_pipe(real_cmd, str)
  78
  79 def read_pipe(c, ignore_error=False):
  80     if verbose:
  81         sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83     pipe = os.popen(c, 'rb')
  84     val = pipe.read()
  85     if pipe.close() and not ignore_error:
  86         die('Command failed: %s' % c)
  87
  88     return val
  89
  90 def p4_read_pipe(c, ignore_error=False):
  91     real_cmd = p4_build_cmd(c)
  92     return read_pipe(real_cmd, ignore_error)
  93
  94 def read_pipe_lines(c):
  95     if verbose:
  96         sys.stderr.write('Reading pipe: %s\n' % c)
  97     ## todo: check return status
  98     pipe = os.popen(c, 'rb')
  99     val = pipe.readlines()
 100     if pipe.close():
 101         die('Command failed: %s' % c)
 102
 103     return val
 104
 105 def p4_read_pipe_lines(c):
 106     """Specifically invoke p4 on the command supplied. """
 107     real_cmd = p4_build_cmd(c)
 108     return read_pipe_lines(real_cmd)
 109
 110 def system(cmd):
 111     if verbose:
 112         sys.stderr.write("executing %s\n" % cmd)
 113     if os.system(cmd) != 0:
 114         die("command failed: %s" % cmd)
 115
 116 def p4_system(cmd):
 117     """Specifically invoke p4 as the system command. """
 118     real_cmd = p4_build_cmd(cmd)
 119     return system(real_cmd)
 120
 121 def isP4Exec(kind):
 122     """Determine if a Perforce 'kind' should have execute permission
 123
 124     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126     a plus sign, it is also executable"""
 127     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129 def setP4ExecBit(file, mode):
 130     # Reopens an already open file and changes the execute bit to match
 131     # the execute bit setting in the passed in mode.
 132
 133     p4Type = "+x"
 134
 135     if not isModeExec(mode):
 136         p4Type = getP4OpenedType(file)
 137         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139         if p4Type[-1] == "+":
 140             p4Type = p4Type[0:-1]
 141
 142     p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144 def getP4OpenedType(file):
 145     # Returns the perforce file type for the given file.
 146
 147     result = p4_read_pipe("opened %s" % file)
 148     match = re.match(".*\((.+)\)\r?$", result)
 149     if match:
 150         return match.group(1)
 151     else:
 152         die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154 def diffTreePattern():
 155     # This is a simple generator for the diff tree regex pattern. This could be
 156     # a class variable if this and parseDiffTreeEntry were a part of a class.
 157     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158     while True:
 159         yield pattern
 160
 161 def parseDiffTreeEntry(entry):
 162     """Parses a single diff tree entry into its component elements.
 163
 164     See git-diff-tree(1) manpage for details about the format of the diff
 165     output. This method returns a dictionary with the following elements:
 166
 167     src_mode - The mode of the source file
 168     dst_mode - The mode of the destination file
 169     src_sha1 - The sha1 for the source file
 170     dst_sha1 - The sha1 fr the destination file
 171     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172     status_score - The score for the status (applicable for 'C' and 'R'
 173                    statuses). This is None if there is no score.
 174     src - The path for the source file.
 175     dst - The path for the destination file. This is only present for
 176           copy or renames. If it is not present, this is None.
 177
 178     If the pattern is not matched, None is returned."""
 179
 180     match = diffTreePattern().next().match(entry)
 181     if match:
 182         return {
 183             'src_mode': match.group(1),
 184             'dst_mode': match.group(2),
 185             'src_sha1': match.group(3),
 186             'dst_sha1': match.group(4),
 187             'status': match.group(5),
 188             'status_score': match.group(6),
 189             'src': match.group(7),
 190             'dst': match.group(10)
 191         }
 192     return None
 193
 194 def isModeExec(mode):
 195     # Returns True if the given git mode represents an executable file,
 196     # otherwise False.
 197     return mode[-3:] == "755"
 198
 199 def isModeExecChanged(src_mode, dst_mode):
 200     return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203     cmd = p4_build_cmd("-G %s" % (cmd))
 204     if verbose:
 205         sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207     # Use a temporary file to avoid deadlocks without
 208     # subprocess.communicate(), which would put another copy
 209     # of stdout into memory.
 210     stdin_file = None
 211     if stdin is not None:
 212         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213         stdin_file.write(stdin)
 214         stdin_file.flush()
 215         stdin_file.seek(0)
 216
 217     p4 = subprocess.Popen(cmd, shell=True,
 218                           stdin=stdin_file,
 219                           stdout=subprocess.PIPE)
 220
 221     result = []
 222     try:
 223         while True:
 224             entry = marshal.load(p4.stdout)
 225             if cb is not None:
 226                 cb(entry)
 227             else:
 228                 result.append(entry)
 229     except EOFError:
 230         pass
 231     exitCode = p4.wait()
 232     if exitCode != 0:
 233         entry = {}
 234         entry["p4ExitCode"] = exitCode
 235         result.append(entry)
 236
 237     return result
 238
 239 def p4Cmd(cmd):
 240     list = p4CmdList(cmd)
 241     result = {}
 242     for entry in list:
 243         result.update(entry)
 244     return result;
 245
 246 def p4Where(depotPath):
 247     if not depotPath.endswith("/"):
 248         depotPath += "/"
 249     depotPath = depotPath + "..."
 250     outputList = p4CmdList("where %s" % depotPath)
 251     output = None
 252     for entry in outputList:
 253         if "depotFile" in entry:
 254             if entry["depotFile"] == depotPath:
 255                 output = entry
 256                 break
 257         elif "data" in entry:
 258             data = entry.get("data")
 259             space = data.find(" ")
 260             if data[:space] == depotPath:
 261                 output = entry
 262                 break
 263     if output == None:
 264         return ""
 265     if output["code"] == "error":
 266         return ""
 267     clientPath = ""
 268     if "path" in output:
 269         clientPath = output.get("path")
 270     elif "data" in output:
 271         data = output.get("data")
 272         lastSpace = data.rfind(" ")
 273         clientPath = data[lastSpace + 1:]
 274
 275     if clientPath.endswith("..."):
 276         clientPath = clientPath[:-3]
 277     return clientPath
 278
 279 def currentGitBranch():
 280     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282 def isValidGitDir(path):
 283     if (os.path.exists(path + "/HEAD")
 284         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285         return True;
 286     return False
 287
 288 def parseRevision(ref):
 289     return read_pipe("git rev-parse %s" % ref).strip()
 290
 291 def extractLogMessageFromGitCommit(commit):
 292     logMessage = ""
 293
 294     ## fixme: title is first line of commit, not 1st paragraph.
 295     foundTitle = False
 296     for log in read_pipe_lines("git cat-file commit %s" % commit):
 297        if not foundTitle:
 298            if len(log) == 1:
 299                foundTitle = True
 300            continue
 301
 302        logMessage += log
 303     return logMessage
 304
 305 def extractSettingsGitLog(log):
 306     values = {}
 307     for line in log.split("\n"):
 308         line = line.strip()
 309         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310         if not m:
 311             continue
 312
 313         assignments = m.group(1).split (':')
 314         for a in assignments:
 315             vals = a.split ('=')
 316             key = vals[0].strip()
 317             val = ('='.join (vals[1:])).strip()
 318             if val.endswith ('\"') and val.startswith('"'):
 319                 val = val[1:-1]
 320
 321             values[key] = val
 322
 323     paths = values.get("depot-paths")
 324     if not paths:
 325         paths = values.get("depot-path")
 326     if paths:
 327         values['depot-paths'] = paths.split(',')
 328     return values
 329
 330 def gitBranchExists(branch):
 331     proc = subprocess.Popen(["git", "rev-parse", branch],
 332                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333     return proc.wait() == 0;
 334
 335 _gitConfig = {}
 336 def gitConfig(key):
 337     if not _gitConfig.has_key(key):
 338         _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339     return _gitConfig[key]
 340
 341 def p4BranchesInGit(branchesAreInRemotes = True):
 342     branches = {}
 343
 344     cmdline = "git rev-parse --symbolic "
 345     if branchesAreInRemotes:
 346         cmdline += " --remotes"
 347     else:
 348         cmdline += " --branches"
 349
 350     for line in read_pipe_lines(cmdline):
 351         line = line.strip()
 352
 353         ## only import to p4/
 354         if not line.startswith('p4/') or line == "p4/HEAD":
 355             continue
 356         branch = line
 357
 358         # strip off p4
 359         branch = re.sub ("^p4/", "", line)
 360
 361         branches[branch] = parseRevision(line)
 362     return branches
 363
 364 def findUpstreamBranchPoint(head = "HEAD"):
 365     branches = p4BranchesInGit()
 366     # map from depot-path to branch name
 367     branchByDepotPath = {}
 368     for branch in branches.keys():
 369         tip = branches[branch]
 370         log = extractLogMessageFromGitCommit(tip)
 371         settings = extractSettingsGitLog(log)
 372         if settings.has_key("depot-paths"):
 373             paths = ",".join(settings["depot-paths"])
 374             branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376     settings = None
 377     parent = 0
 378     while parent < 65535:
 379         commit = head + "~%s" % parent
 380         log = extractLogMessageFromGitCommit(commit)
 381         settings = extractSettingsGitLog(log)
 382         if settings.has_key("depot-paths"):
 383             paths = ",".join(settings["depot-paths"])
 384             if branchByDepotPath.has_key(paths):
 385                 return [branchByDepotPath[paths], settings]
 386
 387         parent = parent + 1
 388
 389     return ["", settings]
 390
 391 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392     if not silent:
 393         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394                % localRefPrefix)
 395
 396     originPrefix = "origin/p4/"
 397
 398     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399         line = line.strip()
 400         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401             continue
 402
 403         headName = line[len(originPrefix):]
 404         remoteHead = localRefPrefix + headName
 405         originHead = line
 406
 407         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408         if (not original.has_key('depot-paths')
 409             or not original.has_key('change')):
 410             continue
 411
 412         update = False
 413         if not gitBranchExists(remoteHead):
 414             if verbose:
 415                 print "creating %s" % remoteHead
 416             update = True
 417         else:
 418             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419             if settings.has_key('change') > 0:
 420                 if settings['depot-paths'] == original['depot-paths']:
 421                     originP4Change = int(original['change'])
 422                     p4Change = int(settings['change'])
 423                     if originP4Change > p4Change:
 424                         print ("%s (%s) is newer than %s (%s). "
 425                                "Updating p4 branch from origin."
 426                                % (originHead, originP4Change,
 427                                   remoteHead, p4Change))
 428                         update = True
 429                 else:
 430                     print ("Ignoring: %s was imported from %s while "
 431                            "%s was imported from %s"
 432                            % (originHead, ','.join(original['depot-paths']),
 433                               remoteHead, ','.join(settings['depot-paths'])))
 434
 435         if update:
 436             system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438 def originP4BranchesExist():
 439         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441 def p4ChangesForPaths(depotPaths, changeRange):
 442     assert depotPaths
 443     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                         for p in depotPaths]))
 445
 446     changes = {}
 447     for line in output:
 448         changeNum = int(line.split(" ")[1])
 449         changes[changeNum] = True
 450
 451     changelist = changes.keys()
 452     changelist.sort()
 453     return changelist
 454
 455 class Command:
 456     def __init__(self):
 457         self.usage = "usage: %prog [options]"
 458         self.needsGit = True
 459
 460 class P4Debug(Command):
 461     def __init__(self):
 462         Command.__init__(self)
 463         self.options = [
 464             optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                  default=False),
 466             ]
 467         self.description = "A tool to debug the output of p4 -G."
 468         self.needsGit = False
 469         self.verbose = False
 470
 471     def run(self, args):
 472         j = 0
 473         for output in p4CmdList(" ".join(args)):
 474             print 'Element: %d' % j
 475             j += 1
 476             print output
 477         return True
 478
 479 class P4RollBack(Command):
 480     def __init__(self):
 481         Command.__init__(self)
 482         self.options = [
 483             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485         ]
 486         self.description = "A tool to debug the multi-branch import. Don't use :)"
 487         self.verbose = False
 488         self.rollbackLocalBranches = False
 489
 490     def run(self, args):
 491         if len(args) != 1:
 492             return False
 493         maxChange = int(args[0])
 494
 495         if "p4ExitCode" in p4Cmd("changes -m 1"):
 496             die("Problems executing p4");
 497
 498         if self.rollbackLocalBranches:
 499             refPrefix = "refs/heads/"
 500             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501         else:
 502             refPrefix = "refs/remotes/"
 503             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505         for line in lines:
 506             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                 line = line.strip()
 508                 ref = refPrefix + line
 509                 log = extractLogMessageFromGitCommit(ref)
 510                 settings = extractSettingsGitLog(log)
 511
 512                 depotPaths = settings['depot-paths']
 513                 change = settings['change']
 514
 515                 changed = False
 516
 517                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                            for p in depotPaths]))) == 0:
 519                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                     continue
 522
 523                 while change and int(change) > maxChange:
 524                     changed = True
 525                     if self.verbose:
 526                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                     system("git update-ref %s \"%s^\"" % (ref, ref))
 528                     log = extractLogMessageFromGitCommit(ref)
 529                     settings =  extractSettingsGitLog(log)
 530
 531
 532                     depotPaths = settings['depot-paths']
 533                     change = settings['change']
 534
 535                 if changed:
 536                     print "%s rewound to %s" % (ref, change)
 537
 538         return True
 539
 540 class P4Submit(Command):
 541     def __init__(self):
 542         Command.__init__(self)
 543         self.options = [
 544                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                 optparse.make_option("--origin", dest="origin"),
 546                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 547         ]
 548         self.description = "Submit changes from git to the perforce depot."
 549         self.usage += " [name of git branch to submit into perforce depot]"
 550         self.interactive = True
 551         self.origin = ""
 552         self.detectRename = False
 553         self.verbose = False
 554         self.isWindows = (platform.system() == "Windows")
 555
 556     def check(self):
 557         if len(p4CmdList("opened ...")) > 0:
 558             die("You have files opened with perforce! Close them before starting the sync.")
 559
 560     # replaces everything between 'Description:' and the next P4 submit template field with the
 561     # commit message
 562     def prepareLogMessage(self, template, message):
 563         result = ""
 564
 565         inDescriptionSection = False
 566
 567         for line in template.split("\n"):
 568             if line.startswith("#"):
 569                 result += line + "\n"
 570                 continue
 571
 572             if inDescriptionSection:
 573                 if line.startswith("Files:"):
 574                     inDescriptionSection = False
 575                 else:
 576                     continue
 577             else:
 578                 if line.startswith("Description:"):
 579                     inDescriptionSection = True
 580                     line += "\n"
 581                     for messageLine in message.split("\n"):
 582                         line += "\t" + messageLine + "\n"
 583
 584             result += line + "\n"
 585
 586         return result
 587
 588     def prepareSubmitTemplate(self):
 589         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590         template = ""
 591         inFilesSection = False
 592         for line in p4_read_pipe_lines("change -o"):
 593             if line.endswith("\r\n"):
 594                 line = line[:-2] + "\n"
 595             if inFilesSection:
 596                 if line.startswith("\t"):
 597                     # path starts and ends with a tab
 598                     path = line[1:]
 599                     lastTab = path.rfind("\t")
 600                     if lastTab != -1:
 601                         path = path[:lastTab]
 602                         if not path.startswith(self.depotPath):
 603                             continue
 604                 else:
 605                     inFilesSection = False
 606             else:
 607                 if line.startswith("Files:"):
 608                     inFilesSection = True
 609
 610             template += line
 611
 612         return template
 613
 614     def applyCommit(self, id):
 615         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616         diffOpts = ("", "-M")[self.detectRename]
 617         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618         filesToAdd = set()
 619         filesToDelete = set()
 620         editedFiles = set()
 621         filesToChangeExecBit = {}
 622         for line in diff:
 623             diff = parseDiffTreeEntry(line)
 624             modifier = diff['status']
 625             path = diff['src']
 626             if modifier == "M":
 627                 p4_system("edit \"%s\"" % path)
 628                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                     filesToChangeExecBit[path] = diff['dst_mode']
 630                 editedFiles.add(path)
 631             elif modifier == "A":
 632                 filesToAdd.add(path)
 633                 filesToChangeExecBit[path] = diff['dst_mode']
 634                 if path in filesToDelete:
 635                     filesToDelete.remove(path)
 636             elif modifier == "D":
 637                 filesToDelete.add(path)
 638                 if path in filesToAdd:
 639                     filesToAdd.remove(path)
 640             elif modifier == "R":
 641                 src, dest = diff['src'], diff['dst']
 642                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                 p4_system("edit \"%s\"" % (dest))
 644                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                     filesToChangeExecBit[dest] = diff['dst_mode']
 646                 os.unlink(dest)
 647                 editedFiles.add(dest)
 648                 filesToDelete.add(src)
 649             else:
 650                 die("unknown modifier %s for %s" % (modifier, path))
 651
 652         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653         patchcmd = diffcmd + " | git apply "
 654         tryPatchCmd = patchcmd + "--check -"
 655         applyPatchCmd = patchcmd + "--check --apply -"
 656
 657         if os.system(tryPatchCmd) != 0:
 658             print "Unfortunately applying the change failed!"
 659             print "What do you want to do?"
 660             response = "x"
 661             while response != "s" and response != "a" and response != "w":
 662                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664             if response == "s":
 665                 print "Skipping! Good luck with the next patches..."
 666                 for f in editedFiles:
 667                     p4_system("revert \"%s\"" % f);
 668                 for f in filesToAdd:
 669                     system("rm %s" %f)
 670                 return
 671             elif response == "a":
 672                 os.system(applyPatchCmd)
 673                 if len(filesToAdd) > 0:
 674                     print "You may also want to call p4 add on the following files:"
 675                     print " ".join(filesToAdd)
 676                 if len(filesToDelete):
 677                     print "The following files should be scheduled for deletion with p4 delete:"
 678                     print " ".join(filesToDelete)
 679                 die("Please resolve and submit the conflict manually and "
 680                     + "continue afterwards with git-p4 submit --continue")
 681             elif response == "w":
 682                 system(diffcmd + " > patch.txt")
 683                 print "Patch saved to patch.txt in %s !" % self.clientPath
 684                 die("Please resolve and submit the conflict manually and "
 685                     "continue afterwards with git-p4 submit --continue")
 686
 687         system(applyPatchCmd)
 688
 689         for f in filesToAdd:
 690             p4_system("add \"%s\"" % f)
 691         for f in filesToDelete:
 692             p4_system("revert \"%s\"" % f)
 693             p4_system("delete \"%s\"" % f)
 694
 695         # Set/clear executable bits
 696         for f in filesToChangeExecBit.keys():
 697             mode = filesToChangeExecBit[f]
 698             setP4ExecBit(f, mode)
 699
 700         logMessage = extractLogMessageFromGitCommit(id)
 701         logMessage = logMessage.strip()
 702
 703         template = self.prepareSubmitTemplate()
 704
 705         if self.interactive:
 706             submitTemplate = self.prepareLogMessage(template, logMessage)
 707             if os.environ.has_key("P4DIFF"):
 708                 del(os.environ["P4DIFF"])
 709             diff = ""
 710             for editedFile in editedFiles:
 711                 diff += p4_read_pipe("diff -du %r" % editedFile)
 712
 713             newdiff = ""
 714             for newFile in filesToAdd:
 715                 newdiff += "==== new file ====\n"
 716                 newdiff += "--- /dev/null\n"
 717                 newdiff += "+++ %s\n" % newFile
 718                 f = open(newFile, "r")
 719                 for line in f.readlines():
 720                     newdiff += "+" + line
 721                 f.close()
 722
 723             separatorLine = "######## everything below this line is just the diff #######\n"
 724
 725             [handle, fileName] = tempfile.mkstemp()
 726             tmpFile = os.fdopen(handle, "w+")
 727             if self.isWindows:
 728                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 729                 separatorLine = separatorLine.replace("\n", "\r\n")
 730                 newdiff = newdiff.replace("\n", "\r\n")
 731             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 732             tmpFile.close()
 733             mtime = os.stat(fileName).st_mtime
 734             if os.environ.has_key("P4EDITOR"):
 735                 editor = os.environ.get("P4EDITOR")
 736             else:
 737                 editor = read_pipe("git var GIT_EDITOR").strip()
 738             system(editor + " " + fileName)
 739
 740             response = "y"
 741             if os.stat(fileName).st_mtime <= mtime:
 742                 response = "x"
 743                 while response != "y" and response != "n":
 744                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 745
 746             if response == "y":
 747                 tmpFile = open(fileName, "rb")
 748                 message = tmpFile.read()
 749                 tmpFile.close()
 750                 submitTemplate = message[:message.index(separatorLine)]
 751                 if self.isWindows:
 752                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 753                 p4_write_pipe("submit -i", submitTemplate)
 754             else:
 755                 for f in editedFiles:
 756                     p4_system("revert \"%s\"" % f);
 757                 for f in filesToAdd:
 758                     p4_system("revert \"%s\"" % f);
 759                     system("rm %s" %f)
 760
 761             os.remove(fileName)
 762         else:
 763             fileName = "submit.txt"
 764             file = open(fileName, "w+")
 765             file.write(self.prepareLogMessage(template, logMessage))
 766             file.close()
 767             print ("Perforce submit template written as %s. "
 768                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 769                    % (fileName, fileName))
 770
 771     def run(self, args):
 772         if len(args) == 0:
 773             self.master = currentGitBranch()
 774             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 775                 die("Detecting current git branch failed!")
 776         elif len(args) == 1:
 777             self.master = args[0]
 778         else:
 779             return False
 780
 781         allowSubmit = gitConfig("git-p4.allowSubmit")
 782         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 783             die("%s is not in git-p4.allowSubmit" % self.master)
 784
 785         [upstream, settings] = findUpstreamBranchPoint()
 786         self.depotPath = settings['depot-paths'][0]
 787         if len(self.origin) == 0:
 788             self.origin = upstream
 789
 790         if self.verbose:
 791             print "Origin branch is " + self.origin
 792
 793         if len(self.depotPath) == 0:
 794             print "Internal error: cannot locate perforce depot path from existing branches"
 795             sys.exit(128)
 796
 797         self.clientPath = p4Where(self.depotPath)
 798
 799         if len(self.clientPath) == 0:
 800             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 801             sys.exit(128)
 802
 803         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 804         self.oldWorkingDirectory = os.getcwd()
 805
 806         chdir(self.clientPath)
 807         print "Synchronizing p4 checkout..."
 808         p4_system("sync ...")
 809
 810         self.check()
 811
 812         commits = []
 813         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 814             commits.append(line.strip())
 815         commits.reverse()
 816
 817         while len(commits) > 0:
 818             commit = commits[0]
 819             commits = commits[1:]
 820             self.applyCommit(commit)
 821             if not self.interactive:
 822                 break
 823
 824         if len(commits) == 0:
 825             print "All changes applied!"
 826             chdir(self.oldWorkingDirectory)
 827
 828             sync = P4Sync()
 829             sync.run([])
 830
 831             rebase = P4Rebase()
 832             rebase.rebase()
 833
 834         return True
 835
 836 class P4Sync(Command):
 837     def __init__(self):
 838         Command.__init__(self)
 839         self.options = [
 840                 optparse.make_option("--branch", dest="branch"),
 841                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 842                 optparse.make_option("--changesfile", dest="changesFile"),
 843                 optparse.make_option("--silent", dest="silent", action="store_true"),
 844                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 845                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 846                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 847                                      help="Import into refs/heads/ , not refs/remotes"),
 848                 optparse.make_option("--max-changes", dest="maxChanges"),
 849                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 850                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 851                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 852                                      help="Only sync files that are included in the Perforce Client Spec")
 853         ]
 854         self.description = """Imports from Perforce into a git repository.\n
 855     example:
 856     //depot/my/project/ -- to import the current head
 857     //depot/my/project/@all -- to import everything
 858     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 859
 860     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 861
 862         self.usage += " //depot/path[@revRange]"
 863         self.silent = False
 864         self.createdBranches = set()
 865         self.committedChanges = set()
 866         self.branch = ""
 867         self.detectBranches = False
 868         self.detectLabels = False
 869         self.changesFile = ""
 870         self.syncWithOrigin = True
 871         self.verbose = False
 872         self.importIntoRemotes = True
 873         self.maxChanges = ""
 874         self.isWindows = (platform.system() == "Windows")
 875         self.keepRepoPath = False
 876         self.depotPaths = None
 877         self.p4BranchesInGit = []
 878         self.cloneExclude = []
 879         self.useClientSpec = False
 880         self.clientSpecDirs = []
 881
 882         if gitConfig("git-p4.syncFromOrigin") == "false":
 883             self.syncWithOrigin = False
 884
 885     def extractFilesFromCommit(self, commit):
 886         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 887                              for path in self.cloneExclude]
 888         files = []
 889         fnum = 0
 890         while commit.has_key("depotFile%s" % fnum):
 891             path =  commit["depotFile%s" % fnum]
 892
 893             if [p for p in self.cloneExclude
 894                 if path.startswith (p)]:
 895                 found = False
 896             else:
 897                 found = [p for p in self.depotPaths
 898                          if path.startswith (p)]
 899             if not found:
 900                 fnum = fnum + 1
 901                 continue
 902
 903             file = {}
 904             file["path"] = path
 905             file["rev"] = commit["rev%s" % fnum]
 906             file["action"] = commit["action%s" % fnum]
 907             file["type"] = commit["type%s" % fnum]
 908             files.append(file)
 909             fnum = fnum + 1
 910         return files
 911
 912     def stripRepoPath(self, path, prefixes):
 913         if self.useClientSpec:
 914
 915             # if using the client spec, we use the output directory
 916             # specified in the client.  For example, a view
 917             #   //depot/foo/branch/... //client/branch/foo/...
 918             # will end up putting all foo/branch files into
 919             #  branch/foo/
 920             for val in self.clientSpecDirs:
 921                 if path.startswith(val[0]):
 922                     # replace the depot path with the client path
 923                     path = path.replace(val[0], val[1][1])
 924                     # now strip out the client (//client/...)
 925                     path = re.sub("^(//[^/]+/)", '', path)
 926                     # the rest is all path
 927                     return path
 928
 929         if self.keepRepoPath:
 930             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 931
 932         for p in prefixes:
 933             if path.startswith(p):
 934                 path = path[len(p):]
 935
 936         return path
 937
 938     def splitFilesIntoBranches(self, commit):
 939         branches = {}
 940         fnum = 0
 941         while commit.has_key("depotFile%s" % fnum):
 942             path =  commit["depotFile%s" % fnum]
 943             found = [p for p in self.depotPaths
 944                      if path.startswith (p)]
 945             if not found:
 946                 fnum = fnum + 1
 947                 continue
 948
 949             file = {}
 950             file["path"] = path
 951             file["rev"] = commit["rev%s" % fnum]
 952             file["action"] = commit["action%s" % fnum]
 953             file["type"] = commit["type%s" % fnum]
 954             fnum = fnum + 1
 955
 956             relPath = self.stripRepoPath(path, self.depotPaths)
 957
 958             for branch in self.knownBranches.keys():
 959
 960                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 961                 if relPath.startswith(branch + "/"):
 962                     if branch not in branches:
 963                         branches[branch] = []
 964                     branches[branch].append(file)
 965                     break
 966
 967         return branches
 968
 969     # output one file from the P4 stream
 970     # - helper for streamP4Files
 971
 972     def streamOneP4File(self, file, contents):
 973         if file["type"] == "apple":
 974             print "\nfile %s is a strange apple file that forks. Ignoring" % \
 975                 file['depotFile']
 976             return
 977
 978         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 979         if verbose:
 980             sys.stderr.write("%s\n" % relPath)
 981
 982         mode = "644"
 983         if isP4Exec(file["type"]):
 984             mode = "755"
 985         elif file["type"] == "symlink":
 986             mode = "120000"
 987             # p4 print on a symlink contains "target\n", so strip it off
 988             data = ''.join(contents)
 989             contents = [data[:-1]]
 990
 991         if self.isWindows and file["type"].endswith("text"):
 992             mangled = []
 993             for data in contents:
 994                 data = data.replace("\r\n", "\n")
 995                 mangled.append(data)
 996             contents = mangled
 997
 998         if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 999             contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
1000         elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1001             contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
1002
1003         self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1004
1005         # total length...
1006         length = 0
1007         for d in contents:
1008             length = length + len(d)
1009
1010         self.gitStream.write("data %d\n" % length)
1011         for d in contents:
1012             self.gitStream.write(d)
1013         self.gitStream.write("\n")
1014
1015     def streamOneP4Deletion(self, file):
1016         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1017         if verbose:
1018             sys.stderr.write("delete %s\n" % relPath)
1019         self.gitStream.write("D %s\n" % relPath)
1020
1021     # handle another chunk of streaming data
1022     def streamP4FilesCb(self, marshalled):
1023
1024         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1025             # start of a new file - output the old one first
1026             self.streamOneP4File(self.stream_file, self.stream_contents)
1027             self.stream_file = {}
1028             self.stream_contents = []
1029             self.stream_have_file_info = False
1030
1031         # pick up the new file information... for the
1032         # 'data' field we need to append to our array
1033         for k in marshalled.keys():
1034             if k == 'data':
1035                 self.stream_contents.append(marshalled['data'])
1036             else:
1037                 self.stream_file[k] = marshalled[k]
1038
1039         self.stream_have_file_info = True
1040
1041     # Stream directly from "p4 files" into "git fast-import"
1042     def streamP4Files(self, files):
1043         filesForCommit = []
1044         filesToRead = []
1045         filesToDelete = []
1046
1047         for f in files:
1048             includeFile = True
1049             for val in self.clientSpecDirs:
1050                 if f['path'].startswith(val[0]):
1051                     if val[1][0] <= 0:
1052                         includeFile = False
1053                     break
1054
1055             if includeFile:
1056                 filesForCommit.append(f)
1057                 if f['action'] not in ('delete', 'move/delete', 'purge'):
1058                     filesToRead.append(f)
1059                 else:
1060                     filesToDelete.append(f)
1061
1062         # deleted files...
1063         for f in filesToDelete:
1064             self.streamOneP4Deletion(f)
1065
1066         if len(filesToRead) > 0:
1067             self.stream_file = {}
1068             self.stream_contents = []
1069             self.stream_have_file_info = False
1070
1071             # curry self argument
1072             def streamP4FilesCbSelf(entry):
1073                 self.streamP4FilesCb(entry)
1074
1075             p4CmdList("-x - print",
1076                 '\n'.join(['%s#%s' % (f['path'], f['rev'])
1077                                                   for f in filesToRead]),
1078                 cb=streamP4FilesCbSelf)
1079
1080             # do the last chunk
1081             if self.stream_file.has_key('depotFile'):
1082                 self.streamOneP4File(self.stream_file, self.stream_contents)
1083
1084     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1085         epoch = details["time"]
1086         author = details["user"]
1087         self.branchPrefixes = branchPrefixes
1088
1089         if self.verbose:
1090             print "commit into %s" % branch
1091
1092         # start with reading files; if that fails, we should not
1093         # create a commit.
1094         new_files = []
1095         for f in files:
1096             if [p for p in branchPrefixes if f['path'].startswith(p)]:
1097                 new_files.append (f)
1098             else:
1099                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1100
1101         self.gitStream.write("commit %s\n" % branch)
1102 #        gitStream.write("mark :%s\n" % details["change"])
1103         self.committedChanges.add(int(details["change"]))
1104         committer = ""
1105         if author not in self.users:
1106             self.getUserMapFromPerforceServer()
1107         if author in self.users:
1108             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1109         else:
1110             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1111
1112         self.gitStream.write("committer %s\n" % committer)
1113
1114         self.gitStream.write("data <<EOT\n")
1115         self.gitStream.write(details["desc"])
1116         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1117                              % (','.join (branchPrefixes), details["change"]))
1118         if len(details['options']) > 0:
1119             self.gitStream.write(": options = %s" % details['options'])
1120         self.gitStream.write("]\nEOT\n\n")
1121
1122         if len(parent) > 0:
1123             if self.verbose:
1124                 print "parent %s" % parent
1125             self.gitStream.write("from %s\n" % parent)
1126
1127         self.streamP4Files(new_files)
1128         self.gitStream.write("\n")
1129
1130         change = int(details["change"])
1131
1132         if self.labels.has_key(change):
1133             label = self.labels[change]
1134             labelDetails = label[0]
1135             labelRevisions = label[1]
1136             if self.verbose:
1137                 print "Change %s is labelled %s" % (change, labelDetails)
1138
1139             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1140                                                     for p in branchPrefixes]))
1141
1142             if len(files) == len(labelRevisions):
1143
1144                 cleanedFiles = {}
1145                 for info in files:
1146                     if info["action"] in ("delete", "purge"):
1147                         continue
1148                     cleanedFiles[info["depotFile"]] = info["rev"]
1149
1150                 if cleanedFiles == labelRevisions:
1151                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1152                     self.gitStream.write("from %s\n" % branch)
1153
1154                     owner = labelDetails["Owner"]
1155                     tagger = ""
1156                     if author in self.users:
1157                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1158                     else:
1159                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1160                     self.gitStream.write("tagger %s\n" % tagger)
1161                     self.gitStream.write("data <<EOT\n")
1162                     self.gitStream.write(labelDetails["Description"])
1163                     self.gitStream.write("EOT\n\n")
1164
1165                 else:
1166                     if not self.silent:
1167                         print ("Tag %s does not match with change %s: files do not match."
1168                                % (labelDetails["label"], change))
1169
1170             else:
1171                 if not self.silent:
1172                     print ("Tag %s does not match with change %s: file count is different."
1173                            % (labelDetails["label"], change))
1174
1175     def getUserCacheFilename(self):
1176         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1177         return home + "/.gitp4-usercache.txt"
1178
1179     def getUserMapFromPerforceServer(self):
1180         if self.userMapFromPerforceServer:
1181             return
1182         self.users = {}
1183
1184         for output in p4CmdList("users"):
1185             if not output.has_key("User"):
1186                 continue
1187             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1188
1189
1190         s = ''
1191         for (key, val) in self.users.items():
1192             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1193
1194         open(self.getUserCacheFilename(), "wb").write(s)
1195         self.userMapFromPerforceServer = True
1196
1197     def loadUserMapFromCache(self):
1198         self.users = {}
1199         self.userMapFromPerforceServer = False
1200         try:
1201             cache = open(self.getUserCacheFilename(), "rb")
1202             lines = cache.readlines()
1203             cache.close()
1204             for line in lines:
1205                 entry = line.strip().split("\t")
1206                 self.users[entry[0]] = entry[1]
1207         except IOError:
1208             self.getUserMapFromPerforceServer()
1209
1210     def getLabels(self):
1211         self.labels = {}
1212
1213         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1214         if len(l) > 0 and not self.silent:
1215             print "Finding files belonging to labels in %s" % `self.depotPaths`
1216
1217         for output in l:
1218             label = output["label"]
1219             revisions = {}
1220             newestChange = 0
1221             if self.verbose:
1222                 print "Querying files for label %s" % label
1223             for file in p4CmdList("files "
1224                                   +  ' '.join (["%s...@%s" % (p, label)
1225                                                 for p in self.depotPaths])):
1226                 revisions[file["depotFile"]] = file["rev"]
1227                 change = int(file["change"])
1228                 if change > newestChange:
1229                     newestChange = change
1230
1231             self.labels[newestChange] = [output, revisions]
1232
1233         if self.verbose:
1234             print "Label changes: %s" % self.labels.keys()
1235
1236     def guessProjectName(self):
1237         for p in self.depotPaths:
1238             if p.endswith("/"):
1239                 p = p[:-1]
1240             p = p[p.strip().rfind("/") + 1:]
1241             if not p.endswith("/"):
1242                p += "/"
1243             return p
1244
1245     def getBranchMapping(self):
1246         lostAndFoundBranches = set()
1247
1248         for info in p4CmdList("branches"):
1249             details = p4Cmd("branch -o %s" % info["branch"])
1250             viewIdx = 0
1251             while details.has_key("View%s" % viewIdx):
1252                 paths = details["View%s" % viewIdx].split(" ")
1253                 viewIdx = viewIdx + 1
1254                 # require standard //depot/foo/... //depot/bar/... mapping
1255                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1256                     continue
1257                 source = paths[0]
1258                 destination = paths[1]
1259                 ## HACK
1260                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1261                     source = source[len(self.depotPaths[0]):-4]
1262                     destination = destination[len(self.depotPaths[0]):-4]
1263
1264                     if destination in self.knownBranches:
1265                         if not self.silent:
1266                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1267                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1268                         continue
1269
1270                     self.knownBranches[destination] = source
1271
1272                     lostAndFoundBranches.discard(destination)
1273
1274                     if source not in self.knownBranches:
1275                         lostAndFoundBranches.add(source)
1276
1277
1278         for branch in lostAndFoundBranches:
1279             self.knownBranches[branch] = branch
1280
1281     def getBranchMappingFromGitBranches(self):
1282         branches = p4BranchesInGit(self.importIntoRemotes)
1283         for branch in branches.keys():
1284             if branch == "master":
1285                 branch = "main"
1286             else:
1287                 branch = branch[len(self.projectName):]
1288             self.knownBranches[branch] = branch
1289
1290     def listExistingP4GitBranches(self):
1291         # branches holds mapping from name to commit
1292         branches = p4BranchesInGit(self.importIntoRemotes)
1293         self.p4BranchesInGit = branches.keys()
1294         for branch in branches.keys():
1295             self.initialParents[self.refPrefix + branch] = branches[branch]
1296
1297     def updateOptionDict(self, d):
1298         option_keys = {}
1299         if self.keepRepoPath:
1300             option_keys['keepRepoPath'] = 1
1301
1302         d["options"] = ' '.join(sorted(option_keys.keys()))
1303
1304     def readOptions(self, d):
1305         self.keepRepoPath = (d.has_key('options')
1306                              and ('keepRepoPath' in d['options']))
1307
1308     def gitRefForBranch(self, branch):
1309         if branch == "main":
1310             return self.refPrefix + "master"
1311
1312         if len(branch) <= 0:
1313             return branch
1314
1315         return self.refPrefix + self.projectName + branch
1316
1317     def gitCommitByP4Change(self, ref, change):
1318         if self.verbose:
1319             print "looking in ref " + ref + " for change %s using bisect..." % change
1320
1321         earliestCommit = ""
1322         latestCommit = parseRevision(ref)
1323
1324         while True:
1325             if self.verbose:
1326                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1327             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1328             if len(next) == 0:
1329                 if self.verbose:
1330                     print "argh"
1331                 return ""
1332             log = extractLogMessageFromGitCommit(next)
1333             settings = extractSettingsGitLog(log)
1334             currentChange = int(settings['change'])
1335             if self.verbose:
1336                 print "current change %s" % currentChange
1337
1338             if currentChange == change:
1339                 if self.verbose:
1340                     print "found %s" % next
1341                 return next
1342
1343             if currentChange < change:
1344                 earliestCommit = "^%s" % next
1345             else:
1346                 latestCommit = "%s" % next
1347
1348         return ""
1349
1350     def importNewBranch(self, branch, maxChange):
1351         # make fast-import flush all changes to disk and update the refs using the checkpoint
1352         # command so that we can try to find the branch parent in the git history
1353         self.gitStream.write("checkpoint\n\n");
1354         self.gitStream.flush();
1355         branchPrefix = self.depotPaths[0] + branch + "/"
1356         range = "@1,%s" % maxChange
1357         #print "prefix" + branchPrefix
1358         changes = p4ChangesForPaths([branchPrefix], range)
1359         if len(changes) <= 0:
1360             return False
1361         firstChange = changes[0]
1362         #print "first change in branch: %s" % firstChange
1363         sourceBranch = self.knownBranches[branch]
1364         sourceDepotPath = self.depotPaths[0] + sourceBranch
1365         sourceRef = self.gitRefForBranch(sourceBranch)
1366         #print "source " + sourceBranch
1367
1368         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1369         #print "branch parent: %s" % branchParentChange
1370         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1371         if len(gitParent) > 0:
1372             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1373             #print "parent git commit: %s" % gitParent
1374
1375         self.importChanges(changes)
1376         return True
1377
1378     def importChanges(self, changes):
1379         cnt = 1
1380         for change in changes:
1381             description = p4Cmd("describe %s" % change)
1382             self.updateOptionDict(description)
1383
1384             if not self.silent:
1385                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1386                 sys.stdout.flush()
1387             cnt = cnt + 1
1388
1389             try:
1390                 if self.detectBranches:
1391                     branches = self.splitFilesIntoBranches(description)
1392                     for branch in branches.keys():
1393                         ## HACK  --hwn
1394                         branchPrefix = self.depotPaths[0] + branch + "/"
1395
1396                         parent = ""
1397
1398                         filesForCommit = branches[branch]
1399
1400                         if self.verbose:
1401                             print "branch is %s" % branch
1402
1403                         self.updatedBranches.add(branch)
1404
1405                         if branch not in self.createdBranches:
1406                             self.createdBranches.add(branch)
1407                             parent = self.knownBranches[branch]
1408                             if parent == branch:
1409                                 parent = ""
1410                             else:
1411                                 fullBranch = self.projectName + branch
1412                                 if fullBranch not in self.p4BranchesInGit:
1413                                     if not self.silent:
1414                                         print("\n    Importing new branch %s" % fullBranch);
1415                                     if self.importNewBranch(branch, change - 1):
1416                                         parent = ""
1417                                         self.p4BranchesInGit.append(fullBranch)
1418                                     if not self.silent:
1419                                         print("\n    Resuming with change %s" % change);
1420
1421                                 if self.verbose:
1422                                     print "parent determined through known branches: %s" % parent
1423
1424                         branch = self.gitRefForBranch(branch)
1425                         parent = self.gitRefForBranch(parent)
1426
1427                         if self.verbose:
1428                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1429
1430                         if len(parent) == 0 and branch in self.initialParents:
1431                             parent = self.initialParents[branch]
1432                             del self.initialParents[branch]
1433
1434                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1435                 else:
1436                     files = self.extractFilesFromCommit(description)
1437                     self.commit(description, files, self.branch, self.depotPaths,
1438                                 self.initialParent)
1439                     self.initialParent = ""
1440             except IOError:
1441                 print self.gitError.read()
1442                 sys.exit(1)
1443
1444     def importHeadRevision(self, revision):
1445         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1446
1447         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1448         details["desc"] = ("Initial import of %s from the state at revision %s"
1449                            % (' '.join(self.depotPaths), revision))
1450         details["change"] = revision
1451         newestRevision = 0
1452
1453         fileCnt = 0
1454         for info in p4CmdList("files "
1455                               +  ' '.join(["%s...%s"
1456                                            % (p, revision)
1457                                            for p in self.depotPaths])):
1458
1459             if info['code'] == 'error':
1460                 sys.stderr.write("p4 returned an error: %s\n"
1461                                  % info['data'])
1462                 sys.exit(1)
1463
1464
1465             change = int(info["change"])
1466             if change > newestRevision:
1467                 newestRevision = change
1468
1469             if info["action"] in ("delete", "purge"):
1470                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1471                 #fileCnt = fileCnt + 1
1472                 continue
1473
1474             for prop in ["depotFile", "rev", "action", "type" ]:
1475                 details["%s%s" % (prop, fileCnt)] = info[prop]
1476
1477             fileCnt = fileCnt + 1
1478
1479         details["change"] = newestRevision
1480         self.updateOptionDict(details)
1481         try:
1482             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1483         except IOError:
1484             print "IO error with git fast-import. Is your git version recent enough?"
1485             print self.gitError.read()
1486
1487
1488     def getClientSpec(self):
1489         specList = p4CmdList( "client -o" )
1490         temp = {}
1491         for entry in specList:
1492             for k,v in entry.iteritems():
1493                 if k.startswith("View"):
1494
1495                     # p4 has these %%1 to %%9 arguments in specs to
1496                     # reorder paths; which we can't handle (yet :)
1497                     if re.match('%%\d', v) != None:
1498                         print "Sorry, can't handle %%n arguments in client specs"
1499                         sys.exit(1)
1500
1501                     if v.startswith('"'):
1502                         start = 1
1503                     else:
1504                         start = 0
1505                     index = v.find("...")
1506
1507                     # save the "client view"; i.e the RHS of the view
1508                     # line that tells the client where to put the
1509                     # files for this view.
1510                     cv = v[index+3:].strip() # +3 to remove previous '...'
1511
1512                     # if the client view doesn't end with a
1513                     # ... wildcard, then we're going to mess up the
1514                     # output directory, so fail gracefully.
1515                     if not cv.endswith('...'):
1516                         print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1517                         sys.exit(1)
1518                     cv=cv[:-3]
1519
1520                     # now save the view; +index means included, -index
1521                     # means it should be filtered out.
1522                     v = v[start:index]
1523                     if v.startswith("-"):
1524                         v = v[1:]
1525                         include = -len(v)
1526                     else:
1527                         include = len(v)
1528
1529                     temp[v] = (include, cv)
1530
1531         self.clientSpecDirs = temp.items()
1532         self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
1533
1534     def run(self, args):
1535         self.depotPaths = []
1536         self.changeRange = ""
1537         self.initialParent = ""
1538         self.previousDepotPaths = []
1539
1540         # map from branch depot path to parent branch
1541         self.knownBranches = {}
1542         self.initialParents = {}
1543         self.hasOrigin = originP4BranchesExist()
1544         if not self.syncWithOrigin:
1545             self.hasOrigin = False
1546
1547         if self.importIntoRemotes:
1548             self.refPrefix = "refs/remotes/p4/"
1549         else:
1550             self.refPrefix = "refs/heads/p4/"
1551
1552         if self.syncWithOrigin and self.hasOrigin:
1553             if not self.silent:
1554                 print "Syncing with origin first by calling git fetch origin"
1555             system("git fetch origin")
1556
1557         if len(self.branch) == 0:
1558             self.branch = self.refPrefix + "master"
1559             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1560                 system("git update-ref %s refs/heads/p4" % self.branch)
1561                 system("git branch -D p4");
1562             # create it /after/ importing, when master exists
1563             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1564                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1565
1566         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1567             self.getClientSpec()
1568
1569         # TODO: should always look at previous commits,
1570         # merge with previous imports, if possible.
1571         if args == []:
1572             if self.hasOrigin:
1573                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1574             self.listExistingP4GitBranches()
1575
1576             if len(self.p4BranchesInGit) > 1:
1577                 if not self.silent:
1578                     print "Importing from/into multiple branches"
1579                 self.detectBranches = True
1580
1581             if self.verbose:
1582                 print "branches: %s" % self.p4BranchesInGit
1583
1584             p4Change = 0
1585             for branch in self.p4BranchesInGit:
1586                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1587
1588                 settings = extractSettingsGitLog(logMsg)
1589
1590                 self.readOptions(settings)
1591                 if (settings.has_key('depot-paths')
1592                     and settings.has_key ('change')):
1593                     change = int(settings['change']) + 1
1594                     p4Change = max(p4Change, change)
1595
1596                     depotPaths = sorted(settings['depot-paths'])
1597                     if self.previousDepotPaths == []:
1598                         self.previousDepotPaths = depotPaths
1599                     else:
1600                         paths = []
1601                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1602                             for i in range(0, min(len(cur), len(prev))):
1603                                 if cur[i] <> prev[i]:
1604                                     i = i - 1
1605                                     break
1606
1607                             paths.append (cur[:i + 1])
1608
1609                         self.previousDepotPaths = paths
1610
1611             if p4Change > 0:
1612                 self.depotPaths = sorted(self.previousDepotPaths)
1613                 self.changeRange = "@%s,#head" % p4Change
1614                 if not self.detectBranches:
1615                     self.initialParent = parseRevision(self.branch)
1616                 if not self.silent and not self.detectBranches:
1617                     print "Performing incremental import into %s git branch" % self.branch
1618
1619         if not self.branch.startswith("refs/"):
1620             self.branch = "refs/heads/" + self.branch
1621
1622         if len(args) == 0 and self.depotPaths:
1623             if not self.silent:
1624                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1625         else:
1626             if self.depotPaths and self.depotPaths != args:
1627                 print ("previous import used depot path %s and now %s was specified. "
1628                        "This doesn't work!" % (' '.join (self.depotPaths),
1629                                                ' '.join (args)))
1630                 sys.exit(1)
1631
1632             self.depotPaths = sorted(args)
1633
1634         revision = ""
1635         self.users = {}
1636
1637         newPaths = []
1638         for p in self.depotPaths:
1639             if p.find("@") != -1:
1640                 atIdx = p.index("@")
1641                 self.changeRange = p[atIdx:]
1642                 if self.changeRange == "@all":
1643                     self.changeRange = ""
1644                 elif ',' not in self.changeRange:
1645                     revision = self.changeRange
1646                     self.changeRange = ""
1647                 p = p[:atIdx]
1648             elif p.find("#") != -1:
1649                 hashIdx = p.index("#")
1650                 revision = p[hashIdx:]
1651                 p = p[:hashIdx]
1652             elif self.previousDepotPaths == []:
1653                 revision = "#head"
1654
1655             p = re.sub ("\.\.\.$", "", p)
1656             if not p.endswith("/"):
1657                 p += "/"
1658
1659             newPaths.append(p)
1660
1661         self.depotPaths = newPaths
1662
1663
1664         self.loadUserMapFromCache()
1665         self.labels = {}
1666         if self.detectLabels:
1667             self.getLabels();
1668
1669         if self.detectBranches:
1670             ## FIXME - what's a P4 projectName ?
1671             self.projectName = self.guessProjectName()
1672
1673             if self.hasOrigin:
1674                 self.getBranchMappingFromGitBranches()
1675             else:
1676                 self.getBranchMapping()
1677             if self.verbose:
1678                 print "p4-git branches: %s" % self.p4BranchesInGit
1679                 print "initial parents: %s" % self.initialParents
1680             for b in self.p4BranchesInGit:
1681                 if b != "master":
1682
1683                     ## FIXME
1684                     b = b[len(self.projectName):]
1685                 self.createdBranches.add(b)
1686
1687         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1688
1689         importProcess = subprocess.Popen(["git", "fast-import"],
1690                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1691                                          stderr=subprocess.PIPE);
1692         self.gitOutput = importProcess.stdout
1693         self.gitStream = importProcess.stdin
1694         self.gitError = importProcess.stderr
1695
1696         if revision:
1697             self.importHeadRevision(revision)
1698         else:
1699             changes = []
1700
1701             if len(self.changesFile) > 0:
1702                 output = open(self.changesFile).readlines()
1703                 changeSet = set()
1704                 for line in output:
1705                     changeSet.add(int(line))
1706
1707                 for change in changeSet:
1708                     changes.append(change)
1709
1710                 changes.sort()
1711             else:
1712                 if self.verbose:
1713                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1714                                                               self.changeRange)
1715                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1716
1717                 if len(self.maxChanges) > 0:
1718                     changes = changes[:min(int(self.maxChanges), len(changes))]
1719
1720             if len(changes) == 0:
1721                 if not self.silent:
1722                     print "No changes to import!"
1723                 return True
1724
1725             if not self.silent and not self.detectBranches:
1726                 print "Import destination: %s" % self.branch
1727
1728             self.updatedBranches = set()
1729
1730             self.importChanges(changes)
1731
1732             if not self.silent:
1733                 print ""
1734                 if len(self.updatedBranches) > 0:
1735                     sys.stdout.write("Updated branches: ")
1736                     for b in self.updatedBranches:
1737                         sys.stdout.write("%s " % b)
1738                     sys.stdout.write("\n")
1739
1740         self.gitStream.close()
1741         if importProcess.wait() != 0:
1742             die("fast-import failed: %s" % self.gitError.read())
1743         self.gitOutput.close()
1744         self.gitError.close()
1745
1746         return True
1747
1748 class P4Rebase(Command):
1749     def __init__(self):
1750         Command.__init__(self)
1751         self.options = [ ]
1752         self.description = ("Fetches the latest revision from perforce and "
1753                             + "rebases the current work (branch) against it")
1754         self.verbose = False
1755
1756     def run(self, args):
1757         sync = P4Sync()
1758         sync.run([])
1759
1760         return self.rebase()
1761
1762     def rebase(self):
1763         if os.system("git update-index --refresh") != 0:
1764             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1765         if len(read_pipe("git diff-index HEAD --")) > 0:
1766             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1767
1768         [upstream, settings] = findUpstreamBranchPoint()
1769         if len(upstream) == 0:
1770             die("Cannot find upstream branchpoint for rebase")
1771
1772         # the branchpoint may be p4/foo~3, so strip off the parent
1773         upstream = re.sub("~[0-9]+$", "", upstream)
1774
1775         print "Rebasing the current branch onto %s" % upstream
1776         oldHead = read_pipe("git rev-parse HEAD").strip()
1777         system("git rebase %s" % upstream)
1778         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1779         return True
1780
1781 class P4Clone(P4Sync):
1782     def __init__(self):
1783         P4Sync.__init__(self)
1784         self.description = "Creates a new git repository and imports from Perforce into it"
1785         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1786         self.options += [
1787             optparse.make_option("--destination", dest="cloneDestination",
1788                                  action='store', default=None,
1789                                  help="where to leave result of the clone"),
1790             optparse.make_option("-/", dest="cloneExclude",
1791                                  action="append", type="string",
1792                                  help="exclude depot path")
1793         ]
1794         self.cloneDestination = None
1795         self.needsGit = False
1796
1797     # This is required for the "append" cloneExclude action
1798     def ensure_value(self, attr, value):
1799         if not hasattr(self, attr) or getattr(self, attr) is None:
1800             setattr(self, attr, value)
1801         return getattr(self, attr)
1802
1803     def defaultDestination(self, args):
1804         ## TODO: use common prefix of args?
1805         depotPath = args[0]
1806         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1807         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1808         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1809         depotDir = re.sub(r"/$", "", depotDir)
1810         return os.path.split(depotDir)[1]
1811
1812     def run(self, args):
1813         if len(args) < 1:
1814             return False
1815
1816         if self.keepRepoPath and not self.cloneDestination:
1817             sys.stderr.write("Must specify destination for --keep-path\n")
1818             sys.exit(1)
1819
1820         depotPaths = args
1821
1822         if not self.cloneDestination and len(depotPaths) > 1:
1823             self.cloneDestination = depotPaths[-1]
1824             depotPaths = depotPaths[:-1]
1825
1826         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1827         for p in depotPaths:
1828             if not p.startswith("//"):
1829                 return False
1830
1831         if not self.cloneDestination:
1832             self.cloneDestination = self.defaultDestination(args)
1833
1834         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1835         if not os.path.exists(self.cloneDestination):
1836             os.makedirs(self.cloneDestination)
1837         chdir(self.cloneDestination)
1838         system("git init")
1839         self.gitdir = os.getcwd() + "/.git"
1840         if not P4Sync.run(self, depotPaths):
1841             return False
1842         if self.branch != "master":
1843             if self.importIntoRemotes:
1844                 masterbranch = "refs/remotes/p4/master"
1845             else:
1846                 masterbranch = "refs/heads/p4/master"
1847             if gitBranchExists(masterbranch):
1848                 system("git branch master %s" % masterbranch)
1849                 system("git checkout -f")
1850             else:
1851                 print "Could not detect main branch. No checkout/master branch created."
1852
1853         return True
1854
1855 class P4Branches(Command):
1856     def __init__(self):
1857         Command.__init__(self)
1858         self.options = [ ]
1859         self.description = ("Shows the git branches that hold imports and their "
1860                             + "corresponding perforce depot paths")
1861         self.verbose = False
1862
1863     def run(self, args):
1864         if originP4BranchesExist():
1865             createOrUpdateBranchesFromOrigin()
1866
1867         cmdline = "git rev-parse --symbolic "
1868         cmdline += " --remotes"
1869
1870         for line in read_pipe_lines(cmdline):
1871             line = line.strip()
1872
1873             if not line.startswith('p4/') or line == "p4/HEAD":
1874                 continue
1875             branch = line
1876
1877             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1878             settings = extractSettingsGitLog(log)
1879
1880             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1881         return True
1882
1883 class HelpFormatter(optparse.IndentedHelpFormatter):
1884     def __init__(self):
1885         optparse.IndentedHelpFormatter.__init__(self)
1886
1887     def format_description(self, description):
1888         if description:
1889             return description + "\n"
1890         else:
1891             return ""
1892
1893 def printUsage(commands):
1894     print "usage: %s <command> [options]" % sys.argv[0]
1895     print ""
1896     print "valid commands: %s" % ", ".join(commands)
1897     print ""
1898     print "Try %s <command> --help for command specific help." % sys.argv[0]
1899     print ""
1900
1901 commands = {
1902     "debug" : P4Debug,
1903     "submit" : P4Submit,
1904     "commit" : P4Submit,
1905     "sync" : P4Sync,
1906     "rebase" : P4Rebase,
1907     "clone" : P4Clone,
1908     "rollback" : P4RollBack,
1909     "branches" : P4Branches
1910 }
1911
1912
1913 def main():
1914     if len(sys.argv[1:]) == 0:
1915         printUsage(commands.keys())
1916         sys.exit(2)
1917
1918     cmd = ""
1919     cmdName = sys.argv[1]
1920     try:
1921         klass = commands[cmdName]
1922         cmd = klass()
1923     except KeyError:
1924         print "unknown command %s" % cmdName
1925         print ""
1926         printUsage(commands.keys())
1927         sys.exit(2)
1928
1929     options = cmd.options
1930     cmd.gitdir = os.environ.get("GIT_DIR", None)
1931
1932     args = sys.argv[2:]
1933
1934     if len(options) > 0:
1935         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1936
1937         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1938                                        options,
1939                                        description = cmd.description,
1940                                        formatter = HelpFormatter())
1941
1942         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1943     global verbose
1944     verbose = cmd.verbose
1945     if cmd.needsGit:
1946         if cmd.gitdir == None:
1947             cmd.gitdir = os.path.abspath(".git")
1948             if not isValidGitDir(cmd.gitdir):
1949                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1950                 if os.path.exists(cmd.gitdir):
1951                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1952                     if len(cdup) > 0:
1953                         chdir(cdup);
1954
1955         if not isValidGitDir(cmd.gitdir):
1956             if isValidGitDir(cmd.gitdir + "/.git"):
1957                 cmd.gitdir += "/.git"
1958             else:
1959                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1960
1961         os.environ["GIT_DIR"] = cmd.gitdir
1962
1963     if not cmd.run(args):
1964         parser.print_help()
1965
1966
1967 if __name__ == '__main__':
1968     main()