[thirdparty/git.git] / contrib / fast-import / git-p4.py

#!/usr/bin/env python
#
# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
#
# Author: Simon Hausmann <hausmann@kde.org>
# License: MIT <http://www.opensource.org/licenses/mit-license.php>
#

import optparse, sys, os, marshal, popen2, shelve
import tempfile

gitdir = os.environ.get("GIT_DIR", "")

def p4CmdList(cmd):
    cmd = "p4 -G %s" % cmd
    pipe = os.popen(cmd, "rb")

    result = []
    try:
        while True:
            entry = marshal.load(pipe)
            result.append(entry)
    except EOFError:
        pass
    pipe.close()

    return result

def p4Cmd(cmd):
    list = p4CmdList(cmd)
    result = {}
    for entry in list:
        result.update(entry)
    return result;

def die(msg):
    sys.stderr.write(msg + "\n")
    sys.exit(1)

def currentGitBranch():
    return os.popen("git-name-rev HEAD").read().split(" ")[1][:-1]

def isValidGitDir(path):
    if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"):
        return True;
    return False

def system(cmd):
    if os.system(cmd) != 0:
        die("command failed: %s" % cmd)

class P4Debug:
    def __init__(self):
        self.options = [
        ]
        self.description = "A tool to debug the output of p4 -G."

    def run(self, args):
        for output in p4CmdList(" ".join(args)):
            print output

class P4CleanTags:
    def __init__(self):
        self.options = [
#                optparse.make_option("--branch", dest="branch", default="refs/heads/master")
        ]
        self.description = "A tool to remove stale unused tags from incremental perforce imports."
    def run(self, args):
        branch = currentGitBranch()
        print "Cleaning out stale p4 import tags..."
        sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch)
        output = sout.read()
        try:
            tagIdx = output.index(" tags/p4/")
        except:
            print "Cannot find any p4/* tag. Nothing to do."
            sys.exit(0)

        try:
            caretIdx = output.index("^")
        except:
            caretIdx = len(output) - 1
        rev = int(output[tagIdx + 9 : caretIdx])

        allTags = os.popen("git tag -l p4/").readlines()
        for i in range(len(allTags)):
            allTags[i] = int(allTags[i][3:-1])

        allTags.sort()

        allTags.remove(rev)

        for rev in allTags:
            print os.popen("git tag -d p4/%s" % rev).read()

        print "%s tags removed." % len(allTags)

class P4Sync:
    def __init__(self):
        self.options = [
                optparse.make_option("--continue", action="store_false", dest="firstTime"),
                optparse.make_option("--origin", dest="origin"),
                optparse.make_option("--reset", action="store_true", dest="reset"),
                optparse.make_option("--master", dest="master"),
                optparse.make_option("--log-substitutions", dest="substFile"),
                optparse.make_option("--noninteractive", action="store_false"),
                optparse.make_option("--dry-run", action="store_true")
        ]
        self.description = "Submit changes from git to the perforce depot."
        self.firstTime = True
        self.reset = False
        self.interactive = True
        self.dryRun = False
        self.substFile = ""
        self.firstTime = True
        self.origin = "origin"
        self.master = ""

        self.logSubstitutions = {}
        self.logSubstitutions["<enter description here>"] = "%log%"
        self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"

    def check(self):
        if len(p4CmdList("opened ...")) > 0:
            die("You have files opened with perforce! Close them before starting the sync.")

    def start(self):
        if len(self.config) > 0 and not self.reset:
            die("Cannot start sync. Previous sync config found at %s" % self.configFile)

        commits = []
        for line in os.popen("git-rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines():
            commits.append(line[:-1])
        commits.reverse()

        self.config["commits"] = commits

        print "Creating temporary p4-sync branch from %s ..." % self.origin
        system("git checkout -f -b p4-sync %s" % self.origin)

    def prepareLogMessage(self, template, message):
        result = ""

        for line in template.split("\n"):
            if line.startswith("#"):
                result += line + "\n"
                continue

            substituted = False
            for key in self.logSubstitutions.keys():
                if line.find(key) != -1:
                    value = self.logSubstitutions[key]
                    value = value.replace("%log%", message)
                    if value != "@remove@":
                        result += line.replace(key, value) + "\n"
                    substituted = True
                    break

            if not substituted:
                result += line + "\n"

        return result

    def apply(self, id):
        print "Applying %s" % (os.popen("git-log --max-count=1 --pretty=oneline %s" % id).read())
        diff = os.popen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines()
        filesToAdd = set()
        filesToDelete = set()
        for line in diff:
            modifier = line[0]
            path = line[1:].strip()
            if modifier == "M":
                system("p4 edit %s" % path)
            elif modifier == "A":
                filesToAdd.add(path)
                if path in filesToDelete:
                    filesToDelete.remove(path)
            elif modifier == "D":
                filesToDelete.add(path)
                if path in filesToAdd:
                    filesToAdd.remove(path)
            else:
                die("unknown modifier %s for %s" % (modifier, path))

        system("git-diff-files --name-only -z | git-update-index --remove -z --stdin")
        system("git cherry-pick --no-commit \"%s\"" % id)

        for f in filesToAdd:
            system("p4 add %s" % f)
        for f in filesToDelete:
            system("p4 revert %s" % f)
            system("p4 delete %s" % f)

        logMessage = ""
        foundTitle = False
        for log in os.popen("git-cat-file commit %s" % id).readlines():
            if not foundTitle:
                if len(log) == 1:
                    foundTitle = 1
                continue

            if len(logMessage) > 0:
                logMessage += "\t"
            logMessage += log

        template = os.popen("p4 change -o").read()

        if self.interactive:
            submitTemplate = self.prepareLogMessage(template, logMessage)
            diff = os.popen("p4 diff -du ...").read()

            for newFile in filesToAdd:
                diff += "==== new file ====\n"
                diff += "--- /dev/null\n"
                diff += "+++ %s\n" % newFile
                f = open(newFile, "r")
                for line in f.readlines():
                    diff += "+" + line
                f.close()

            pipe = os.popen("less", "w")
            pipe.write(submitTemplate + diff)
            pipe.close()

            response = "e"
            while response == "e":
                response = raw_input("Do you want to submit this change (y/e/n)? ")
                if response == "e":
                    [handle, fileName] = tempfile.mkstemp()
                    tmpFile = os.fdopen(handle, "w+")
                    tmpFile.write(submitTemplate)
                    tmpFile.close()
                    editor = os.environ.get("EDITOR", "vi")
                    system(editor + " " + fileName)
                    tmpFile = open(fileName, "r")
                    submitTemplate = tmpFile.read()
                    tmpFile.close()
                    os.remove(fileName)

            if response == "y" or response == "yes":
               if self.dryRun:
                   print submitTemplate
                   raw_input("Press return to continue...")
               else:
                    pipe = os.popen("p4 submit -i", "w")
                    pipe.write(submitTemplate)
                    pipe.close()
            else:
                print "Not submitting!"
                self.interactive = False
        else:
            fileName = "submit.txt"
            file = open(fileName, "w+")
            file.write(self.prepareLogMessage(template, logMessage))
            file.close()
            print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName)

    def run(self, args):
        if self.reset:
            self.firstTime = True

        if len(self.substFile) > 0:
            for line in open(self.substFile, "r").readlines():
                tokens = line[:-1].split("=")
                self.logSubstitutions[tokens[0]] = tokens[1]

        if len(self.master) == 0:
            self.master = currentGitBranch()
            if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)):
                die("Detecting current git branch failed!")

        self.check()
        self.configFile = gitdir + "/p4-git-sync.cfg"
        self.config = shelve.open(self.configFile, writeback=True)

        if self.firstTime:
            self.start()

        commits = self.config.get("commits", [])

        while len(commits) > 0:
            self.firstTime = False
            commit = commits[0]
            commits = commits[1:]
            self.config["commits"] = commits
            self.apply(commit)
            if not self.interactive:
                break

        self.config.close()

        if len(commits) == 0:
            if self.firstTime:
                print "No changes found to apply between %s and current HEAD" % self.origin
            else:
                print "All changes applied!"
                print "Deleting temporary p4-sync branch and going back to %s" % self.master
                system("git checkout %s" % self.master)
                system("git branch -D p4-sync")
                print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert ..."
                system("p4 edit ... >/dev/null")
                system("p4 revert ... >/dev/null")
            os.remove(self.configFile)


def printUsage(commands):
    print "usage: %s <command> [options]" % sys.argv[0]
    print ""
    print "valid commands: %s" % ", ".join(commands)
    print ""
    print "Try %s <command> --help for command specific help." % sys.argv[0]
    print ""

commands = {
    "debug" : P4Debug(),
    "clean-tags" : P4CleanTags(),
    "sync-to-perforce" : P4Sync()
}

if len(sys.argv[1:]) == 0:
    printUsage(commands.keys())
    sys.exit(2)

cmd = ""
cmdName = sys.argv[1]
try:
    cmd = commands[cmdName]
except KeyError:
    print "unknown command %s" % cmdName
    print ""
    printUsage(commands.keys())
    sys.exit(2)

options = cmd.options
cmd.gitdir = gitdir
options.append(optparse.make_option("--git-dir", dest="gitdir"))

parser = optparse.OptionParser("usage: %prog " + cmdName + " [options]", options,
                               description = cmd.description)

(cmd, args) = parser.parse_args(sys.argv[2:], cmd);

gitdir = cmd.gitdir
if len(gitdir) == 0:
    gitdir = ".git"

if not isValidGitDir(gitdir):
    if isValidGitDir(gitdir + "/.git"):
        gitdir += "/.git"
    else:
        dir("fatal: cannot locate git repository at %s" % gitdir)

os.environ["GIT_DIR"] = gitdir

cmd.run(args)
Commit	Line	Data
86949eef SH	1	#!/usr/bin/env python
	2	#
	3	# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
	4	#
	5	# Author: Simon Hausmann <hausmann@kde.org>
	6	# License: MIT <http://www.opensource.org/licenses/mit-license.php>
	7	#
	8
4f5cf76a SH	9	import optparse, sys, os, marshal, popen2, shelve
	10	import tempfile
	11
	12	gitdir = os.environ.get("GIT_DIR", "")
86949eef SH	13
	14	def p4CmdList(cmd):
	15	cmd = "p4 -G %s" % cmd
	16	pipe = os.popen(cmd, "rb")
	17
	18	result = []
	19	try:
	20	while True:
	21	entry = marshal.load(pipe)
	22	result.append(entry)
	23	except EOFError:
	24	pass
	25	pipe.close()
	26
	27	return result
	28
	29	def p4Cmd(cmd):
	30	list = p4CmdList(cmd)
	31	result = {}
	32	for entry in list:
	33	result.update(entry)
	34	return result;
	35
	36	def die(msg):
	37	sys.stderr.write(msg + "\n")
	38	sys.exit(1)
	39
	40	def currentGitBranch():
	41	return os.popen("git-name-rev HEAD").read().split(" ")[1][:-1]
	42
4f5cf76a SH	43	def isValidGitDir(path):
	44	if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"):
	45	return True;
	46	return False
	47
	48	def system(cmd):
	49	if os.system(cmd) != 0:
	50	die("command failed: %s" % cmd)
	51
86949eef SH	52	class P4Debug:
	53	def __init__(self):
	54	self.options = [
	55	]
c8c39116	56	self.description = "A tool to debug the output of p4 -G."
86949eef SH	57
	58	def run(self, args):
	59	for output in p4CmdList(" ".join(args)):
	60	print output
	61
	62	class P4CleanTags:
	63	def __init__(self):
	64	self.options = [
	65	# optparse.make_option("--branch", dest="branch", default="refs/heads/master")
	66	]
c8c39116	67	self.description = "A tool to remove stale unused tags from incremental perforce imports."
86949eef SH	68	def run(self, args):
	69	branch = currentGitBranch()
	70	print "Cleaning out stale p4 import tags..."
	71	sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch)
	72	output = sout.read()
	73	try:
	74	tagIdx = output.index(" tags/p4/")
	75	except:
	76	print "Cannot find any p4/* tag. Nothing to do."
	77	sys.exit(0)
	78
	79	try:
	80	caretIdx = output.index("^")
	81	except:
	82	caretIdx = len(output) - 1
	83	rev = int(output[tagIdx + 9 : caretIdx])
	84
	85	allTags = os.popen("git tag -l p4/").readlines()
	86	for i in range(len(allTags)):
	87	allTags[i] = int(allTags[i][3:-1])
	88
	89	allTags.sort()
	90
	91	allTags.remove(rev)
	92
	93	for rev in allTags:
	94	print os.popen("git tag -d p4/%s" % rev).read()
	95
	96	print "%s tags removed." % len(allTags)
	97
4f5cf76a SH	98	class P4Sync:
	99	def __init__(self):
	100	self.options = [
	101	optparse.make_option("--continue", action="store_false", dest="firstTime"),
	102	optparse.make_option("--origin", dest="origin"),
	103	optparse.make_option("--reset", action="store_true", dest="reset"),
	104	optparse.make_option("--master", dest="master"),
	105	optparse.make_option("--log-substitutions", dest="substFile"),
	106	optparse.make_option("--noninteractive", action="store_false"),
	107	optparse.make_option("--dry-run", action="store_true")
	108	]
	109	self.description = "Submit changes from git to the perforce depot."
	110	self.firstTime = True
	111	self.reset = False
	112	self.interactive = True
	113	self.dryRun = False
	114	self.substFile = ""
	115	self.firstTime = True
	116	self.origin = "origin"
	117	self.master = ""
	118
	119	self.logSubstitutions = {}
	120	self.logSubstitutions["<enter description here>"] = "%log%"
	121	self.logSubstitutions["\tDetails:"] = "\tDetails: %log%"
	122
	123	def check(self):
	124	if len(p4CmdList("opened ...")) > 0:
	125	die("You have files opened with perforce! Close them before starting the sync.")
	126
	127	def start(self):
	128	if len(self.config) > 0 and not self.reset:
	129	die("Cannot start sync. Previous sync config found at %s" % self.configFile)
	130
	131	commits = []
	132	for line in os.popen("git-rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines():
	133	commits.append(line[:-1])
	134	commits.reverse()
	135
	136	self.config["commits"] = commits
	137
	138	print "Creating temporary p4-sync branch from %s ..." % self.origin
	139	system("git checkout -f -b p4-sync %s" % self.origin)
	140
	141	def prepareLogMessage(self, template, message):
	142	result = ""
	143
	144	for line in template.split("\n"):
	145	if line.startswith("#"):
	146	result += line + "\n"
	147	continue
	148
	149	substituted = False
	150	for key in self.logSubstitutions.keys():
	151	if line.find(key) != -1:
	152	value = self.logSubstitutions[key]
	153	value = value.replace("%log%", message)
	154	if value != "@remove@":
	155	result += line.replace(key, value) + "\n"
	156	substituted = True
	157	break
	158
	159	if not substituted:
	160	result += line + "\n"
	161
162	return result
163
164	def apply(self, id):
165	print "Applying %s" % (os.popen("git-log --max-count=1 --pretty=oneline %s" % id).read())
166	diff = os.popen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines()
167	filesToAdd = set()
168	filesToDelete = set()
169	for line in diff:
170	modifier = line[0]
171	path = line[1:].strip()
172	if modifier == "M":
173	system("p4 edit %s" % path)
174	elif modifier == "A":
175	filesToAdd.add(path)
176	if path in filesToDelete:
177	filesToDelete.remove(path)
178	elif modifier == "D":
179	filesToDelete.add(path)
180	if path in filesToAdd:
181	filesToAdd.remove(path)
182	else:
183	die("unknown modifier %s for %s" % (modifier, path))
184
185	system("git-diff-files --name-only -z \| git-update-index --remove -z --stdin")
186	system("git cherry-pick --no-commit \"%s\"" % id)
187
188	for f in filesToAdd:
189	system("p4 add %s" % f)
190	for f in filesToDelete:
191	system("p4 revert %s" % f)
192	system("p4 delete %s" % f)
193
194	logMessage = ""
195	foundTitle = False
196	for log in os.popen("git-cat-file commit %s" % id).readlines():
197	if not foundTitle:
198	if len(log) == 1:
199	foundTitle = 1
200	continue
201
202	if len(logMessage) > 0:
203	logMessage += "\t"
204	logMessage += log
205
206	template = os.popen("p4 change -o").read()
207
208	if self.interactive:
209	submitTemplate = self.prepareLogMessage(template, logMessage)
210	diff = os.popen("p4 diff -du ...").read()
211
212	for newFile in filesToAdd:
213	diff += "==== new file ====\n"
214	diff += "--- /dev/null\n"
215	diff += "+++ %s\n" % newFile
216	f = open(newFile, "r")
217	for line in f.readlines():
218	diff += "+" + line
219	f.close()
220
221	pipe = os.popen("less", "w")
222	pipe.write(submitTemplate + diff)
223	pipe.close()
224
225	response = "e"
226	while response == "e":
227	response = raw_input("Do you want to submit this change (y/e/n)? ")
228	if response == "e":
229	[handle, fileName] = tempfile.mkstemp()
230	tmpFile = os.fdopen(handle, "w+")
231	tmpFile.write(submitTemplate)
232	tmpFile.close()
233	editor = os.environ.get("EDITOR", "vi")
234	system(editor + " " + fileName)
235	tmpFile = open(fileName, "r")
236	submitTemplate = tmpFile.read()
237	tmpFile.close()
238	os.remove(fileName)
239
240	if response == "y" or response == "yes":
241	if self.dryRun:
242	print submitTemplate
243	raw_input("Press return to continue...")
244	else:
245	pipe = os.popen("p4 submit -i", "w")
246	pipe.write(submitTemplate)
247	pipe.close()
248	else:
249	print "Not submitting!"
250	self.interactive = False
251	else:
252	fileName = "submit.txt"
253	file = open(fileName, "w+")
254	file.write(self.prepareLogMessage(template, logMessage))
255	file.close()
256	print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName)
257
258	def run(self, args):
259	if self.reset:
260	self.firstTime = True
261
262	if len(self.substFile) > 0:
263	for line in open(self.substFile, "r").readlines():
264	tokens = line[:-1].split("=")
265	self.logSubstitutions[tokens[0]] = tokens[1]
266
267	if len(self.master) == 0:
268	self.master = currentGitBranch()
269	if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)):
270	die("Detecting current git branch failed!")
271
272	self.check()
273	self.configFile = gitdir + "/p4-git-sync.cfg"
274	self.config = shelve.open(self.configFile, writeback=True)
275
276	if self.firstTime:
277	self.start()
278
279	commits = self.config.get("commits", [])
280
281	while len(commits) > 0:
282	self.firstTime = False
283	commit = commits[0]
284	commits = commits[1:]
285	self.config["commits"] = commits
286	self.apply(commit)
287	if not self.interactive:
288	break
289
290	self.config.close()
291
292	if len(commits) == 0:
293	if self.firstTime:
294	print "No changes found to apply between %s and current HEAD" % self.origin
295	else:
296	print "All changes applied!"
297	print "Deleting temporary p4-sync branch and going back to %s" % self.master
298	system("git checkout %s" % self.master)
299	system("git branch -D p4-sync")
300	print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert ..."
301	system("p4 edit ... >/dev/null")
302	system("p4 revert ... >/dev/null")
303	os.remove(self.configFile)
304
305
86949eef SH	306	def printUsage(commands):
	307	print "usage: %s <command> [options]" % sys.argv[0]
	308	print ""
	309	print "valid commands: %s" % ", ".join(commands)
	310	print ""
	311	print "Try %s <command> --help for command specific help." % sys.argv[0]
	312	print ""
	313
	314	commands = {
	315	"debug" : P4Debug(),
4f5cf76a SH	316	"clean-tags" : P4CleanTags(),
4f5cf76a SH	317	"sync-to-perforce" : P4Sync()
86949eef SH	318	}
	319
	320	if len(sys.argv[1:]) == 0:
	321	printUsage(commands.keys())
	322	sys.exit(2)
	323
	324	cmd = ""
	325	cmdName = sys.argv[1]
	326	try:
	327	cmd = commands[cmdName]
	328	except KeyError:
	329	print "unknown command %s" % cmdName
	330	print ""
	331	printUsage(commands.keys())
	332	sys.exit(2)
	333
4f5cf76a SH	334	options = cmd.options
	335	cmd.gitdir = gitdir
	336	options.append(optparse.make_option("--git-dir", dest="gitdir"))
	337
	338	parser = optparse.OptionParser("usage: %prog " + cmdName + " [options]", options,
c8c39116	339	description = cmd.description)
86949eef SH	340
	341	(cmd, args) = parser.parse_args(sys.argv[2:], cmd);
	342
4f5cf76a SH	343	gitdir = cmd.gitdir
	344	if len(gitdir) == 0:
	345	gitdir = ".git"
	346
	347	if not isValidGitDir(gitdir):
	348	if isValidGitDir(gitdir + "/.git"):
	349	gitdir += "/.git"
	350	else:
	351	dir("fatal: cannot locate git repository at %s" % gitdir)
	352
	353	os.environ["GIT_DIR"] = gitdir
	354
86949eef	355	cmd.run(args)