#!/usr/bin/env python # # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. # # Author: Simon Hausmann # Copyright: 2007 Simon Hausmann # 2007 Trolltech ASA # License: MIT # # TODO: * Consider making --with-origin the default, assuming that the git # protocol is always more efficient. (needs manual testing first :) # import optparse, sys, os, marshal, popen2, subprocess, shelve import tempfile, getopt, sha, os.path, time, platform from sets import Set; gitdir = os.environ.get("GIT_DIR", "") def mypopen(command): return os.popen(command, "rb"); def p4CmdList(cmd): cmd = "p4 -G %s" % cmd pipe = os.popen(cmd, "rb") result = [] try: while True: entry = marshal.load(pipe) result.append(entry) except EOFError: pass exitCode = pipe.close() if exitCode != None: entry = {} entry["p4ExitCode"] = exitCode result.append(entry) return result def p4Cmd(cmd): list = p4CmdList(cmd) result = {} for entry in list: result.update(entry) return result; def p4Where(depotPath): if not depotPath.endswith("/"): depotPath += "/" output = p4Cmd("where %s..." % depotPath) if output["code"] == "error": return "" clientPath = "" if "path" in output: clientPath = output.get("path") elif "data" in output: data = output.get("data") lastSpace = data.rfind(" ") clientPath = data[lastSpace + 1:] if clientPath.endswith("..."): clientPath = clientPath[:-3] return clientPath def die(msg): sys.stderr.write(msg + "\n") sys.exit(1) def currentGitBranch(): return mypopen("git name-rev HEAD").read().split(" ")[1][:-1] def isValidGitDir(path): if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"): return True; return False def parseRevision(ref): return mypopen("git rev-parse %s" % ref).read()[:-1] def system(cmd): if os.system(cmd) != 0: die("command failed: %s" % cmd) def extractLogMessageFromGitCommit(commit): logMessage = "" foundTitle = False for log in mypopen("git cat-file commit %s" % commit).readlines(): if not foundTitle: if len(log) == 1: foundTitle = True continue logMessage += log return logMessage def extractDepotPathAndChangeFromGitLog(log): values = {} for line in log.split("\n"): line = line.strip() if line.startswith("[git-p4:") and line.endswith("]"): line = line[8:-1].strip() for assignment in line.split(":"): variable = assignment.strip() value = "" equalPos = assignment.find("=") if equalPos != -1: variable = assignment[:equalPos].strip() value = assignment[equalPos + 1:].strip() if value.startswith("\"") and value.endswith("\""): value = value[1:-1] values[variable] = value return values.get("depot-path"), values.get("change") def gitBranchExists(branch): proc = subprocess.Popen(["git", "rev-parse", branch], stderr=subprocess.PIPE, stdout=subprocess.PIPE); return proc.wait() == 0; class Command: def __init__(self): self.usage = "usage: %prog [options]" self.needsGit = True class P4Debug(Command): def __init__(self): Command.__init__(self) self.options = [ ] self.description = "A tool to debug the output of p4 -G." self.needsGit = False def run(self, args): for output in p4CmdList(" ".join(args)): print output return True class P4RollBack(Command): def __init__(self): Command.__init__(self) self.options = [ optparse.make_option("--verbose", dest="verbose", action="store_true"), optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true") ] self.description = "A tool to debug the multi-branch import. Don't use :)" self.verbose = False self.rollbackLocalBranches = False def run(self, args): if len(args) != 1: return False maxChange = int(args[0]) if "p4ExitCode" in p4Cmd("changes -m 1"): die("Problems executing p4"); if self.rollbackLocalBranches: refPrefix = "refs/heads/" lines = mypopen("git rev-parse --symbolic --branches").readlines() else: refPrefix = "refs/remotes/" lines = mypopen("git rev-parse --symbolic --remotes").readlines() for line in lines: if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"): ref = refPrefix + line[:-1] log = extractLogMessageFromGitCommit(ref) depotPath, change = extractDepotPathAndChangeFromGitLog(log) changed = False if len(p4Cmd("changes -m 1 %s...@%s" % (depotPath, maxChange))) == 0: print "Branch %s did not exist at change %s, deleting." % (ref, maxChange) system("git update-ref -d %s `git rev-parse %s`" % (ref, ref)) continue while len(change) > 0 and int(change) > maxChange: changed = True if self.verbose: print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange) system("git update-ref %s \"%s^\"" % (ref, ref)) log = extractLogMessageFromGitCommit(ref) depotPath, change = extractDepotPathAndChangeFromGitLog(log) if changed: print "%s rewound to %s" % (ref, change) return True class P4Submit(Command): def __init__(self): Command.__init__(self) self.options = [ optparse.make_option("--continue", action="store_false", dest="firstTime"), optparse.make_option("--origin", dest="origin"), optparse.make_option("--reset", action="store_true", dest="reset"), optparse.make_option("--log-substitutions", dest="substFile"), optparse.make_option("--noninteractive", action="store_false"), optparse.make_option("--dry-run", action="store_true"), optparse.make_option("--direct", dest="directSubmit", action="store_true"), ] self.description = "Submit changes from git to the perforce depot." self.usage += " [name of git branch to submit into perforce depot]" self.firstTime = True self.reset = False self.interactive = True self.dryRun = False self.substFile = "" self.firstTime = True self.origin = "" self.directSubmit = False self.logSubstitutions = {} self.logSubstitutions[""] = "%log%" self.logSubstitutions["\tDetails:"] = "\tDetails: %log%" def check(self): if len(p4CmdList("opened ...")) > 0: die("You have files opened with perforce! Close them before starting the sync.") def start(self): if len(self.config) > 0 and not self.reset: die("Cannot start sync. Previous sync config found at %s\nIf you want to start submitting again from scratch maybe you want to call git-p4 submit --reset" % self.configFile) commits = [] if self.directSubmit: commits.append("0") else: for line in mypopen("git rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines(): commits.append(line[:-1]) commits.reverse() self.config["commits"] = commits def prepareLogMessage(self, template, message): result = "" for line in template.split("\n"): if line.startswith("#"): result += line + "\n" continue substituted = False for key in self.logSubstitutions.keys(): if line.find(key) != -1: value = self.logSubstitutions[key] value = value.replace("%log%", message) if value != "@remove@": result += line.replace(key, value) + "\n" substituted = True break if not substituted: result += line + "\n" return result def apply(self, id): if self.directSubmit: print "Applying local change in working directory/index" diff = self.diffStatus else: print "Applying %s" % (mypopen("git log --max-count=1 --pretty=oneline %s" % id).read()) diff = mypopen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines() filesToAdd = set() filesToDelete = set() editedFiles = set() for line in diff: modifier = line[0] path = line[1:].strip() if modifier == "M": system("p4 edit \"%s\"" % path) editedFiles.add(path) elif modifier == "A": filesToAdd.add(path) if path in filesToDelete: filesToDelete.remove(path) elif modifier == "D": filesToDelete.add(path) if path in filesToAdd: filesToAdd.remove(path) else: die("unknown modifier %s for %s" % (modifier, path)) if self.directSubmit: diffcmd = "cat \"%s\"" % self.diffFile else: diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id) patchcmd = diffcmd + " | git apply " tryPatchCmd = patchcmd + "--check -" applyPatchCmd = patchcmd + "--check --apply -" if os.system(tryPatchCmd) != 0: print "Unfortunately applying the change failed!" print "What do you want to do?" response = "x" while response != "s" and response != "a" and response != "w": response = raw_input("[s]kip this patch / [a]pply the patch forcibly and with .rej files / [w]rite the patch to a file (patch.txt) ") if response == "s": print "Skipping! Good luck with the next patches..." return elif response == "a": os.system(applyPatchCmd) if len(filesToAdd) > 0: print "You may also want to call p4 add on the following files:" print " ".join(filesToAdd) if len(filesToDelete): print "The following files should be scheduled for deletion with p4 delete:" print " ".join(filesToDelete) die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue") elif response == "w": system(diffcmd + " > patch.txt") print "Patch saved to patch.txt in %s !" % self.clientPath die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue") system(applyPatchCmd) for f in filesToAdd: system("p4 add %s" % f) for f in filesToDelete: system("p4 revert %s" % f) system("p4 delete %s" % f) logMessage = "" if not self.directSubmit: logMessage = extractLogMessageFromGitCommit(id) logMessage = logMessage.replace("\n", "\n\t") logMessage = logMessage[:-1] template = mypopen("p4 change -o").read() if self.interactive: submitTemplate = self.prepareLogMessage(template, logMessage) diff = mypopen("p4 diff -du ...").read() for newFile in filesToAdd: diff += "==== new file ====\n" diff += "--- /dev/null\n" diff += "+++ %s\n" % newFile f = open(newFile, "r") for line in f.readlines(): diff += "+" + line f.close() separatorLine = "######## everything below this line is just the diff #######" if platform.system() == "Windows": separatorLine += "\r" separatorLine += "\n" response = "e" firstIteration = True while response == "e": if not firstIteration: response = raw_input("Do you want to submit this change? [y]es/[e]dit/[n]o/[s]kip ") firstIteration = False if response == "e": [handle, fileName] = tempfile.mkstemp() tmpFile = os.fdopen(handle, "w+") tmpFile.write(submitTemplate + separatorLine + diff) tmpFile.close() defaultEditor = "vi" if platform.system() == "Windows": defaultEditor = "notepad" editor = os.environ.get("EDITOR", defaultEditor); system(editor + " " + fileName) tmpFile = open(fileName, "rb") message = tmpFile.read() tmpFile.close() os.remove(fileName) submitTemplate = message[:message.index(separatorLine)] if response == "y" or response == "yes": if self.dryRun: print submitTemplate raw_input("Press return to continue...") else: if self.directSubmit: print "Submitting to git first" os.chdir(self.oldWorkingDirectory) pipe = os.popen("git commit -a -F -", "wb") pipe.write(submitTemplate) pipe.close() os.chdir(self.clientPath) pipe = os.popen("p4 submit -i", "wb") pipe.write(submitTemplate) pipe.close() elif response == "s": for f in editedFiles: system("p4 revert \"%s\"" % f); for f in filesToAdd: system("p4 revert \"%s\"" % f); system("rm %s" %f) for f in filesToDelete: system("p4 delete \"%s\"" % f); return else: print "Not submitting!" self.interactive = False else: fileName = "submit.txt" file = open(fileName, "w+") file.write(self.prepareLogMessage(template, logMessage)) file.close() print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName) def run(self, args): global gitdir # make gitdir absolute so we can cd out into the perforce checkout gitdir = os.path.abspath(gitdir) os.environ["GIT_DIR"] = gitdir if len(args) == 0: self.master = currentGitBranch() if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)): die("Detecting current git branch failed!") elif len(args) == 1: self.master = args[0] else: return False depotPath = "" if gitBranchExists("p4"): [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("p4")) if len(depotPath) == 0 and gitBranchExists("origin"): [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("origin")) if len(depotPath) == 0: print "Internal error: cannot locate perforce depot path from existing branches" sys.exit(128) self.clientPath = p4Where(depotPath) if len(self.clientPath) == 0: print "Error: Cannot locate perforce checkout of %s in client view" % depotPath sys.exit(128) print "Perforce checkout for depot path %s located at %s" % (depotPath, self.clientPath) self.oldWorkingDirectory = os.getcwd() if self.directSubmit: self.diffStatus = mypopen("git diff -r --name-status HEAD").readlines() if len(self.diffStatus) == 0: print "No changes in working directory to submit." return True patch = mypopen("git diff -p --binary --diff-filter=ACMRTUXB HEAD").read() self.diffFile = gitdir + "/p4-git-diff" f = open(self.diffFile, "wb") f.write(patch) f.close(); os.chdir(self.clientPath) response = raw_input("Do you want to sync %s with p4 sync? [y]es/[n]o " % self.clientPath) if response == "y" or response == "yes": system("p4 sync ...") if len(self.origin) == 0: if gitBranchExists("p4"): self.origin = "p4" else: self.origin = "origin" if self.reset: self.firstTime = True if len(self.substFile) > 0: for line in open(self.substFile, "r").readlines(): tokens = line[:-1].split("=") self.logSubstitutions[tokens[0]] = tokens[1] self.check() self.configFile = gitdir + "/p4-git-sync.cfg" self.config = shelve.open(self.configFile, writeback=True) if self.firstTime: self.start() commits = self.config.get("commits", []) while len(commits) > 0: self.firstTime = False commit = commits[0] commits = commits[1:] self.config["commits"] = commits self.apply(commit) if not self.interactive: break self.config.close() if self.directSubmit: os.remove(self.diffFile) if len(commits) == 0: if self.firstTime: print "No changes found to apply between %s and current HEAD" % self.origin else: print "All changes applied!" os.chdir(self.oldWorkingDirectory) response = raw_input("Do you want to sync from Perforce now using git-p4 rebase? [y]es/[n]o ") if response == "y" or response == "yes": rebase = P4Rebase() rebase.run([]) os.remove(self.configFile) return True class P4Sync(Command): def __init__(self): Command.__init__(self) self.options = [ optparse.make_option("--branch", dest="branch"), optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"), optparse.make_option("--changesfile", dest="changesFile"), optparse.make_option("--silent", dest="silent", action="store_true"), optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"), optparse.make_option("--with-origin", dest="syncWithOrigin", action="store_true"), optparse.make_option("--verbose", dest="verbose", action="store_true"), optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false"), optparse.make_option("--max-changes", dest="maxChanges") ] self.description = """Imports from Perforce into a git repository.\n example: //depot/my/project/ -- to import the current head //depot/my/project/@all -- to import everything //depot/my/project/@1,6 -- to import only from revision 1 to 6 (a ... is not needed in the path p4 specification, it's added implicitly)""" self.usage += " //depot/path[@revRange]" self.silent = False self.createdBranches = Set() self.committedChanges = Set() self.branch = "" self.detectBranches = False self.detectLabels = False self.changesFile = "" self.syncWithOrigin = False self.verbose = False self.importIntoRemotes = True self.maxChanges = "" self.isWindows = (platform.system() == "Windows") def p4File(self, depotPath): return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() def extractFilesFromCommit(self, commit): files = [] fnum = 0 while commit.has_key("depotFile%s" % fnum): path = commit["depotFile%s" % fnum] if not path.startswith(self.depotPath): # if not self.silent: # print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.depotPath, change) fnum = fnum + 1 continue file = {} file["path"] = path file["rev"] = commit["rev%s" % fnum] file["action"] = commit["action%s" % fnum] file["type"] = commit["type%s" % fnum] files.append(file) fnum = fnum + 1 return files def splitFilesIntoBranches(self, commit): branches = {} fnum = 0 while commit.has_key("depotFile%s" % fnum): path = commit["depotFile%s" % fnum] if not path.startswith(self.depotPath): # if not self.silent: # print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.depotPath, change) fnum = fnum + 1 continue file = {} file["path"] = path file["rev"] = commit["rev%s" % fnum] file["action"] = commit["action%s" % fnum] file["type"] = commit["type%s" % fnum] fnum = fnum + 1 relPath = path[len(self.depotPath):] for branch in self.knownBranches.keys(): if relPath.startswith(branch + "/"): # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2 if branch not in branches: branches[branch] = [] branches[branch].append(file) return branches def commit(self, details, files, branch, branchPrefix, parent = ""): epoch = details["time"] author = details["user"] if self.verbose: print "commit into %s" % branch self.gitStream.write("commit %s\n" % branch) # gitStream.write("mark :%s\n" % details["change"]) self.committedChanges.add(int(details["change"])) committer = "" if author not in self.users: self.getUserMapFromPerforceServer() if author in self.users: committer = "%s %s %s" % (self.users[author], epoch, self.tz) else: committer = "%s %s %s" % (author, epoch, self.tz) self.gitStream.write("committer %s\n" % committer) self.gitStream.write("data < 0: if self.verbose: print "parent %s" % parent self.gitStream.write("from %s\n" % parent) for file in files: path = file["path"] if not path.startswith(branchPrefix): # if not silent: # print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"]) continue rev = file["rev"] depotPath = path + "#" + rev relPath = path[len(branchPrefix):] action = file["action"] if file["type"] == "apple": print "\nfile %s is a strange apple file that forks. Ignoring!" % path continue if action == "delete": self.gitStream.write("D %s\n" % relPath) else: mode = 644 if file["type"].startswith("x"): mode = 755 data = self.p4File(depotPath) if self.isWindows and file["type"].endswith("text"): data = data.replace("\r\n", "\n") self.gitStream.write("M %s inline %s\n" % (mode, relPath)) self.gitStream.write("data %s\n" % len(data)) self.gitStream.write(data) self.gitStream.write("\n") self.gitStream.write("\n") change = int(details["change"]) if self.labels.has_key(change): label = self.labels[change] labelDetails = label[0] labelRevisions = label[1] if self.verbose: print "Change %s is labelled %s" % (change, labelDetails) files = p4CmdList("files %s...@%s" % (branchPrefix, change)) if len(files) == len(labelRevisions): cleanedFiles = {} for info in files: if info["action"] == "delete": continue cleanedFiles[info["depotFile"]] = info["rev"] if cleanedFiles == labelRevisions: self.gitStream.write("tag tag_%s\n" % labelDetails["label"]) self.gitStream.write("from %s\n" % branch) owner = labelDetails["Owner"] tagger = "" if author in self.users: tagger = "%s %s %s" % (self.users[owner], epoch, self.tz) else: tagger = "%s %s %s" % (owner, epoch, self.tz) self.gitStream.write("tagger %s\n" % tagger) self.gitStream.write("data <" cache = open(gitdir + "/p4-usercache.txt", "wb") for user in self.users.keys(): cache.write("%s\t%s\n" % (user, self.users[user])) cache.close(); self.userMapFromPerforceServer = True def loadUserMapFromCache(self): self.users = {} self.userMapFromPerforceServer = False try: cache = open(gitdir + "/p4-usercache.txt", "rb") lines = cache.readlines() cache.close() for line in lines: entry = line[:-1].split("\t") self.users[entry[0]] = entry[1] except IOError: self.getUserMapFromPerforceServer() def getLabels(self): self.labels = {} l = p4CmdList("labels %s..." % self.depotPath) if len(l) > 0 and not self.silent: print "Finding files belonging to labels in %s" % self.depotPath for output in l: label = output["label"] revisions = {} newestChange = 0 if self.verbose: print "Querying files for label %s" % label for file in p4CmdList("files %s...@%s" % (self.depotPath, label)): revisions[file["depotFile"]] = file["rev"] change = int(file["change"]) if change > newestChange: newestChange = change self.labels[newestChange] = [output, revisions] if self.verbose: print "Label changes: %s" % self.labels.keys() def getBranchMapping(self): self.projectName = self.depotPath[self.depotPath[:-1].rfind("/") + 1:] for info in p4CmdList("branches"): details = p4Cmd("branch -o %s" % info["branch"]) viewIdx = 0 while details.has_key("View%s" % viewIdx): paths = details["View%s" % viewIdx].split(" ") viewIdx = viewIdx + 1 # require standard //depot/foo/... //depot/bar/... mapping if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."): continue source = paths[0] destination = paths[1] if source.startswith(self.depotPath) and destination.startswith(self.depotPath): source = source[len(self.depotPath):-4] destination = destination[len(self.depotPath):-4] if destination not in self.knownBranches: self.knownBranches[destination] = source if source not in self.knownBranches: self.knownBranches[source] = source def listExistingP4GitBranches(self): self.p4BranchesInGit = [] cmdline = "git rev-parse --symbolic " if self.importIntoRemotes: cmdline += " --remotes" else: cmdline += " --branches" for line in mypopen(cmdline).readlines(): if self.importIntoRemotes and ((not line.startswith("p4/")) or line == "p4/HEAD\n"): continue if self.importIntoRemotes: # strip off p4 branch = line[3:-1] else: branch = line[:-1] self.p4BranchesInGit.append(branch) self.initialParents[self.refPrefix + branch] = parseRevision(line[:-1]) def createOrUpdateBranchesFromOrigin(self): if not self.silent: print "Creating/updating branch(es) in %s based on origin branch(es)" % self.refPrefix for line in mypopen("git rev-parse --symbolic --remotes"): if (not line.startswith("origin/")) or line.endswith("HEAD\n"): continue headName = line[len("origin/"):-1] remoteHead = self.refPrefix + headName originHead = "origin/" + headName update = False if not os.path.exists(gitdir + "/" + remoteHead): if self.verbose: print "creating %s" % remoteHead update = True else: [originPreviousDepotPath, originP4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(originHead)) [p4PreviousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(remoteHead)) if len(originPreviousDepotPath) > 0 and len(originP4Change) > 0 and len(p4Change) > 0: if originPreviousDepotPath == p4PreviousDepotPath: originP4Change = int(originP4Change) p4Change = int(p4Change) if originP4Change > p4Change: print "%s (%s) is newer than %s (%s). Updating p4 branch from origin." % (originHead, originP4Change, remoteHead, p4Change) update = True else: print "Ignoring: %s was imported from %s while %s was imported from %s" % (originHead, originPreviousDepotPath, remoteHead, p4PreviousDepotPath) if update: system("git update-ref %s %s" % (remoteHead, originHead)) def run(self, args): self.depotPath = "" self.changeRange = "" self.initialParent = "" self.previousDepotPath = "" # map from branch depot path to parent branch self.knownBranches = {} self.initialParents = {} if self.importIntoRemotes: self.refPrefix = "refs/remotes/p4/" else: self.refPrefix = "refs/heads/" createP4HeadRef = False; if len(self.branch) == 0: self.branch = self.refPrefix + "master" if gitBranchExists("refs/heads/p4") and self.importIntoRemotes: system("git update-ref %s refs/heads/p4" % self.branch) system("git branch -D p4"); # create it /after/ importing, when master exists if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes: createP4HeadRef = True if len(args) == 0: self.createOrUpdateBranchesFromOrigin() self.listExistingP4GitBranches() if len(self.p4BranchesInGit) > 1: if not self.silent: print "Importing from/into multiple branches" self.detectBranches = True if self.verbose: print "branches: %s" % self.p4BranchesInGit p4Change = 0 for branch in self.p4BranchesInGit: depotPath, change = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.refPrefix + branch)) if self.verbose: print "path %s change %s" % (depotPath, change) if len(depotPath) > 0 and len(change) > 0: change = int(change) + 1 p4Change = max(p4Change, change) if len(self.previousDepotPath) == 0: self.previousDepotPath = depotPath else: i = 0 l = min(len(self.previousDepotPath), len(depotPath)) while i < l and self.previousDepotPath[i] == depotPath[i]: i = i + 1 self.previousDepotPath = self.previousDepotPath[:i] if p4Change > 0: self.depotPath = self.previousDepotPath self.changeRange = "@%s,#head" % p4Change self.initialParent = parseRevision(self.branch) if not self.silent and not self.detectBranches: print "Performing incremental import into %s git branch" % self.branch if not self.branch.startswith("refs/"): self.branch = "refs/heads/" + self.branch if len(self.depotPath) != 0: self.depotPath = self.depotPath[:-1] if len(args) == 0 and len(self.depotPath) != 0: if not self.silent: print "Depot path: %s" % self.depotPath elif len(args) != 1: return False else: if len(self.depotPath) != 0 and self.depotPath != args[0]: print "previous import used depot path %s and now %s was specified. this doesn't work!" % (self.depotPath, args[0]) sys.exit(1) self.depotPath = args[0] self.revision = "" self.users = {} if self.depotPath.find("@") != -1: atIdx = self.depotPath.index("@") self.changeRange = self.depotPath[atIdx:] if self.changeRange == "@all": self.changeRange = "" elif self.changeRange.find(",") == -1: self.revision = self.changeRange self.changeRange = "" self.depotPath = self.depotPath[0:atIdx] elif self.depotPath.find("#") != -1: hashIdx = self.depotPath.index("#") self.revision = self.depotPath[hashIdx:] self.depotPath = self.depotPath[0:hashIdx] elif len(self.previousDepotPath) == 0: self.revision = "#head" if self.depotPath.endswith("..."): self.depotPath = self.depotPath[:-3] if not self.depotPath.endswith("/"): self.depotPath += "/" self.loadUserMapFromCache() self.labels = {} if self.detectLabels: self.getLabels(); if self.detectBranches: self.getBranchMapping(); if self.verbose: print "p4-git branches: %s" % self.p4BranchesInGit print "initial parents: %s" % self.initialParents for b in self.p4BranchesInGit: if b != "master": b = b[len(self.projectName):] self.createdBranches.add(b) self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60)) importProcess = subprocess.Popen(["git", "fast-import"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE); self.gitOutput = importProcess.stdout self.gitStream = importProcess.stdin self.gitError = importProcess.stderr if len(self.revision) > 0: print "Doing initial import of %s from revision %s" % (self.depotPath, self.revision) details = { "user" : "git perforce import user", "time" : int(time.time()) } details["desc"] = "Initial import of %s from the state at revision %s" % (self.depotPath, self.revision) details["change"] = self.revision newestRevision = 0 fileCnt = 0 for info in p4CmdList("files %s...%s" % (self.depotPath, self.revision)): change = int(info["change"]) if change > newestRevision: newestRevision = change if info["action"] == "delete": # don't increase the file cnt, otherwise details["depotFile123"] will have gaps! #fileCnt = fileCnt + 1 continue for prop in [ "depotFile", "rev", "action", "type" ]: details["%s%s" % (prop, fileCnt)] = info[prop] fileCnt = fileCnt + 1 details["change"] = newestRevision try: self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPath) except IOError: print "IO error with git fast-import. Is your git version recent enough?" print self.gitError.read() else: changes = [] if len(self.changesFile) > 0: output = open(self.changesFile).readlines() changeSet = Set() for line in output: changeSet.add(int(line)) for change in changeSet: changes.append(change) changes.sort() else: if self.verbose: print "Getting p4 changes for %s...%s" % (self.depotPath, self.changeRange) output = mypopen("p4 changes %s...%s" % (self.depotPath, self.changeRange)).readlines() for line in output: changeNum = line.split(" ")[1] changes.append(changeNum) changes.reverse() if len(self.maxChanges) > 0: changes = changes[0:min(int(self.maxChanges), len(changes))] if len(changes) == 0: if not self.silent: print "No changes to import!" return True self.updatedBranches = set() cnt = 1 for change in changes: description = p4Cmd("describe %s" % change) if not self.silent: sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) sys.stdout.flush() cnt = cnt + 1 try: if self.detectBranches: branches = self.splitFilesIntoBranches(description) for branch in branches.keys(): branchPrefix = self.depotPath + branch + "/" parent = "" filesForCommit = branches[branch] if self.verbose: print "branch is %s" % branch self.updatedBranches.add(branch) if branch not in self.createdBranches: self.createdBranches.add(branch) parent = self.knownBranches[branch] if parent == branch: parent = "" elif self.verbose: print "parent determined through known branches: %s" % parent # main branch? use master if branch == "main": branch = "master" else: branch = self.projectName + branch if parent == "main": parent = "master" elif len(parent) > 0: parent = self.projectName + parent branch = self.refPrefix + branch if len(parent) > 0: parent = self.refPrefix + parent if self.verbose: print "looking for initial parent for %s; current parent is %s" % (branch, parent) if len(parent) == 0 and branch in self.initialParents: parent = self.initialParents[branch] del self.initialParents[branch] self.commit(description, filesForCommit, branch, branchPrefix, parent) else: files = self.extractFilesFromCommit(description) self.commit(description, files, self.branch, self.depotPath, self.initialParent) self.initialParent = "" except IOError: print self.gitError.read() sys.exit(1) if not self.silent: print "" if len(self.updatedBranches) > 0: sys.stdout.write("Updated branches: ") for b in self.updatedBranches: sys.stdout.write("%s " % b) sys.stdout.write("\n") self.gitStream.close() if importProcess.wait() != 0: die("fast-import failed: %s" % self.gitError.read()) self.gitOutput.close() self.gitError.close() if createP4HeadRef: system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch)) return True class P4Rebase(Command): def __init__(self): Command.__init__(self) self.options = [ optparse.make_option("--with-origin", dest="syncWithOrigin", action="store_true") ] self.description = "Fetches the latest revision from perforce and rebases the current work (branch) against it" self.syncWithOrigin = False def run(self, args): sync = P4Sync() sync.syncWithOrigin = self.syncWithOrigin sync.run([]) print "Rebasing the current branch" oldHead = mypopen("git rev-parse HEAD").read()[:-1] system("git rebase p4") system("git diff-tree --stat --summary -M %s HEAD" % oldHead) return True class P4Clone(P4Sync): def __init__(self): P4Sync.__init__(self) self.description = "Creates a new git repository and imports from Perforce into it" self.usage = "usage: %prog [options] //depot/path[@revRange] [directory]" self.needsGit = False def run(self, args): global gitdir if len(args) < 1: return False depotPath = args[0] dir = "" if len(args) == 2: dir = args[1] elif len(args) > 2: return False if not depotPath.startswith("//"): return False if len(dir) == 0: dir = depotPath atPos = dir.rfind("@") if atPos != -1: dir = dir[0:atPos] hashPos = dir.rfind("#") if hashPos != -1: dir = dir[0:hashPos] if dir.endswith("..."): dir = dir[:-3] if dir.endswith("/"): dir = dir[:-1] slashPos = dir.rfind("/") if slashPos != -1: dir = dir[slashPos + 1:] print "Importing from %s into %s" % (depotPath, dir) os.makedirs(dir) os.chdir(dir) system("git init") gitdir = os.getcwd() + "/.git" if not P4Sync.run(self, [depotPath]): return False if self.branch != "master": if gitBranchExists("refs/remotes/p4/master"): system("git branch master refs/remotes/p4/master") system("git checkout -f") else: print "Could not detect main branch. No checkout/master branch created." return True class HelpFormatter(optparse.IndentedHelpFormatter): def __init__(self): optparse.IndentedHelpFormatter.__init__(self) def format_description(self, description): if description: return description + "\n" else: return "" def printUsage(commands): print "usage: %s [options]" % sys.argv[0] print "" print "valid commands: %s" % ", ".join(commands) print "" print "Try %s --help for command specific help." % sys.argv[0] print "" commands = { "debug" : P4Debug(), "submit" : P4Submit(), "sync" : P4Sync(), "rebase" : P4Rebase(), "clone" : P4Clone(), "rollback" : P4RollBack() } if len(sys.argv[1:]) == 0: printUsage(commands.keys()) sys.exit(2) cmd = "" cmdName = sys.argv[1] try: cmd = commands[cmdName] except KeyError: print "unknown command %s" % cmdName print "" printUsage(commands.keys()) sys.exit(2) options = cmd.options cmd.gitdir = gitdir args = sys.argv[2:] if len(options) > 0: options.append(optparse.make_option("--git-dir", dest="gitdir")) parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName), options, description = cmd.description, formatter = HelpFormatter()) (cmd, args) = parser.parse_args(sys.argv[2:], cmd); if cmd.needsGit: gitdir = cmd.gitdir if len(gitdir) == 0: gitdir = ".git" if not isValidGitDir(gitdir): gitdir = mypopen("git rev-parse --git-dir").read()[:-1] if os.path.exists(gitdir): cdup = mypopen("git rev-parse --show-cdup").read()[:-1]; if len(cdup) > 0: os.chdir(cdup); if not isValidGitDir(gitdir): if isValidGitDir(gitdir + "/.git"): gitdir += "/.git" else: die("fatal: cannot locate git repository at %s" % gitdir) os.environ["GIT_DIR"] = gitdir if not cmd.run(args): parser.print_help()