3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 import optparse
, sys
, os
, marshal
, subprocess
, shelve
12 import tempfile
, getopt
, os
.path
, time
, platform
17 # Only labels/tags matching this will be imported/exported
18 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
20 def p4_build_cmd(cmd
):
21 """Build a suitable p4 command line.
23 This consolidates building and returning a p4 command line into one
24 location. It means that hooking into the environment, or other configuration
25 can be done more easily.
29 user
= gitConfig("git-p4.user")
31 real_cmd
+= ["-u",user
]
33 password
= gitConfig("git-p4.password")
35 real_cmd
+= ["-P", password
]
37 port
= gitConfig("git-p4.port")
39 real_cmd
+= ["-p", port
]
41 host
= gitConfig("git-p4.host")
43 real_cmd
+= ["-H", host
]
45 client
= gitConfig("git-p4.client")
47 real_cmd
+= ["-c", client
]
50 if isinstance(cmd
,basestring
):
51 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
57 # P4 uses the PWD environment variable rather than getcwd(). Since we're
58 # not using the shell, we have to set it ourselves. This path could
59 # be relative, so go there first, then figure out where we ended up.
61 os
.environ
['PWD'] = os
.getcwd()
67 sys
.stderr
.write(msg
+ "\n")
70 def write_pipe(c
, stdin
):
72 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
74 expand
= isinstance(c
,basestring
)
75 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
77 val
= pipe
.write(stdin
)
80 die('Command failed: %s' % str(c
))
84 def p4_write_pipe(c
, stdin
):
85 real_cmd
= p4_build_cmd(c
)
86 return write_pipe(real_cmd
, stdin
)
88 def read_pipe(c
, ignore_error
=False):
90 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
92 expand
= isinstance(c
,basestring
)
93 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
96 if p
.wait() and not ignore_error
:
97 die('Command failed: %s' % str(c
))
101 def p4_read_pipe(c
, ignore_error
=False):
102 real_cmd
= p4_build_cmd(c
)
103 return read_pipe(real_cmd
, ignore_error
)
105 def read_pipe_lines(c
):
107 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
109 expand
= isinstance(c
, basestring
)
110 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
112 val
= pipe
.readlines()
113 if pipe
.close() or p
.wait():
114 die('Command failed: %s' % str(c
))
118 def p4_read_pipe_lines(c
):
119 """Specifically invoke p4 on the command supplied. """
120 real_cmd
= p4_build_cmd(c
)
121 return read_pipe_lines(real_cmd
)
123 def p4_has_command(cmd
):
124 """Ask p4 for help on this command. If it returns an error, the
125 command does not exist in this version of p4."""
126 real_cmd
= p4_build_cmd(["help", cmd
])
127 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
128 stderr
=subprocess
.PIPE
)
130 return p
.returncode
== 0
133 expand
= isinstance(cmd
,basestring
)
135 sys
.stderr
.write("executing %s\n" % str(cmd
))
136 subprocess
.check_call(cmd
, shell
=expand
)
139 """Specifically invoke p4 as the system command. """
140 real_cmd
= p4_build_cmd(cmd
)
141 expand
= isinstance(real_cmd
, basestring
)
142 subprocess
.check_call(real_cmd
, shell
=expand
)
144 def p4_integrate(src
, dest
):
145 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
147 def p4_sync(f
, *options
):
148 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
151 # forcibly add file names with wildcards
152 if wildcard_present(f
):
153 p4_system(["add", "-f", f
])
155 p4_system(["add", f
])
158 p4_system(["delete", wildcard_encode(f
)])
161 p4_system(["edit", wildcard_encode(f
)])
164 p4_system(["revert", wildcard_encode(f
)])
166 def p4_reopen(type, f
):
167 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
169 def p4_move(src
, dest
):
170 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
173 # Canonicalize the p4 type and return a tuple of the
174 # base type, plus any modifiers. See "p4 help filetypes"
175 # for a list and explanation.
177 def split_p4_type(p4type
):
179 p4_filetypes_historical
= {
180 "ctempobj": "binary+Sw",
186 "tempobj": "binary+FSw",
187 "ubinary": "binary+F",
188 "uresource": "resource+F",
189 "uxbinary": "binary+Fx",
190 "xbinary": "binary+x",
192 "xtempobj": "binary+Swx",
194 "xunicode": "unicode+x",
197 if p4type
in p4_filetypes_historical
:
198 p4type
= p4_filetypes_historical
[p4type
]
200 s
= p4type
.split("+")
208 # return the raw p4 type of a file (text, text+ko, etc)
211 results
= p4CmdList(["fstat", "-T", "headType", file])
212 return results
[0]['headType']
215 # Given a type base and modifier, return a regexp matching
216 # the keywords that can be expanded in the file
218 def p4_keywords_regexp_for_type(base
, type_mods
):
219 if base
in ("text", "unicode", "binary"):
221 if "ko" in type_mods
:
223 elif "k" in type_mods
:
224 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
228 \$ # Starts with a dollar, followed by...
229 (%s) # one of the keywords, followed by...
230 (:[^$]+)? # possibly an old expansion, followed by...
238 # Given a file, return a regexp matching the possible
239 # RCS keywords that will be expanded, or None for files
240 # with kw expansion turned off.
242 def p4_keywords_regexp_for_file(file):
243 if not os
.path
.exists(file):
246 (type_base
, type_mods
) = split_p4_type(p4_type(file))
247 return p4_keywords_regexp_for_type(type_base
, type_mods
)
249 def setP4ExecBit(file, mode
):
250 # Reopens an already open file and changes the execute bit to match
251 # the execute bit setting in the passed in mode.
255 if not isModeExec(mode
):
256 p4Type
= getP4OpenedType(file)
257 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
258 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
259 if p4Type
[-1] == "+":
260 p4Type
= p4Type
[0:-1]
262 p4_reopen(p4Type
, file)
264 def getP4OpenedType(file):
265 # Returns the perforce file type for the given file.
267 result
= p4_read_pipe(["opened", wildcard_encode(file)])
268 match
= re
.match(".*\((.+)\)\r?$", result
)
270 return match
.group(1)
272 die("Could not determine file type for %s (result: '%s')" % (file, result
))
274 # Return the set of all p4 labels
275 def getP4Labels(depotPaths
):
277 if isinstance(depotPaths
,basestring
):
278 depotPaths
= [depotPaths
]
280 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
286 # Return the set of all git tags
289 for line
in read_pipe_lines(["git", "tag"]):
294 def diffTreePattern():
295 # This is a simple generator for the diff tree regex pattern. This could be
296 # a class variable if this and parseDiffTreeEntry were a part of a class.
297 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
301 def parseDiffTreeEntry(entry
):
302 """Parses a single diff tree entry into its component elements.
304 See git-diff-tree(1) manpage for details about the format of the diff
305 output. This method returns a dictionary with the following elements:
307 src_mode - The mode of the source file
308 dst_mode - The mode of the destination file
309 src_sha1 - The sha1 for the source file
310 dst_sha1 - The sha1 fr the destination file
311 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
312 status_score - The score for the status (applicable for 'C' and 'R'
313 statuses). This is None if there is no score.
314 src - The path for the source file.
315 dst - The path for the destination file. This is only present for
316 copy or renames. If it is not present, this is None.
318 If the pattern is not matched, None is returned."""
320 match
= diffTreePattern().next().match(entry
)
323 'src_mode': match
.group(1),
324 'dst_mode': match
.group(2),
325 'src_sha1': match
.group(3),
326 'dst_sha1': match
.group(4),
327 'status': match
.group(5),
328 'status_score': match
.group(6),
329 'src': match
.group(7),
330 'dst': match
.group(10)
334 def isModeExec(mode
):
335 # Returns True if the given git mode represents an executable file,
337 return mode
[-3:] == "755"
339 def isModeExecChanged(src_mode
, dst_mode
):
340 return isModeExec(src_mode
) != isModeExec(dst_mode
)
342 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
344 if isinstance(cmd
,basestring
):
351 cmd
= p4_build_cmd(cmd
)
353 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
355 # Use a temporary file to avoid deadlocks without
356 # subprocess.communicate(), which would put another copy
357 # of stdout into memory.
359 if stdin
is not None:
360 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
361 if isinstance(stdin
,basestring
):
362 stdin_file
.write(stdin
)
365 stdin_file
.write(i
+ '\n')
369 p4
= subprocess
.Popen(cmd
,
372 stdout
=subprocess
.PIPE
)
377 entry
= marshal
.load(p4
.stdout
)
387 entry
["p4ExitCode"] = exitCode
393 list = p4CmdList(cmd
)
399 def p4Where(depotPath
):
400 if not depotPath
.endswith("/"):
402 depotPath
= depotPath
+ "..."
403 outputList
= p4CmdList(["where", depotPath
])
405 for entry
in outputList
:
406 if "depotFile" in entry
:
407 if entry
["depotFile"] == depotPath
:
410 elif "data" in entry
:
411 data
= entry
.get("data")
412 space
= data
.find(" ")
413 if data
[:space
] == depotPath
:
418 if output
["code"] == "error":
422 clientPath
= output
.get("path")
423 elif "data" in output
:
424 data
= output
.get("data")
425 lastSpace
= data
.rfind(" ")
426 clientPath
= data
[lastSpace
+ 1:]
428 if clientPath
.endswith("..."):
429 clientPath
= clientPath
[:-3]
432 def currentGitBranch():
433 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
435 def isValidGitDir(path
):
436 if (os
.path
.exists(path
+ "/HEAD")
437 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
441 def parseRevision(ref
):
442 return read_pipe("git rev-parse %s" % ref
).strip()
444 def branchExists(ref
):
445 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
449 def extractLogMessageFromGitCommit(commit
):
452 ## fixme: title is first line of commit, not 1st paragraph.
454 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
463 def extractSettingsGitLog(log
):
465 for line
in log
.split("\n"):
467 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
471 assignments
= m
.group(1).split (':')
472 for a
in assignments
:
474 key
= vals
[0].strip()
475 val
= ('='.join (vals
[1:])).strip()
476 if val
.endswith ('\"') and val
.startswith('"'):
481 paths
= values
.get("depot-paths")
483 paths
= values
.get("depot-path")
485 values
['depot-paths'] = paths
.split(',')
488 def gitBranchExists(branch
):
489 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
490 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
491 return proc
.wait() == 0;
494 def gitConfig(key
, args
= None): # set args to "--bool", for instance
495 if not _gitConfig
.has_key(key
):
498 argsFilter
= "%s " % args
499 cmd
= "git config %s%s" % (argsFilter
, key
)
500 _gitConfig
[key
] = read_pipe(cmd
, ignore_error
=True).strip()
501 return _gitConfig
[key
]
503 def gitConfigList(key
):
504 if not _gitConfig
.has_key(key
):
505 _gitConfig
[key
] = read_pipe("git config --get-all %s" % key
, ignore_error
=True).strip().split(os
.linesep
)
506 return _gitConfig
[key
]
508 def p4BranchesInGit(branchesAreInRemotes
= True):
511 cmdline
= "git rev-parse --symbolic "
512 if branchesAreInRemotes
:
513 cmdline
+= " --remotes"
515 cmdline
+= " --branches"
517 for line
in read_pipe_lines(cmdline
):
520 ## only import to p4/
521 if not line
.startswith('p4/') or line
== "p4/HEAD":
526 branch
= re
.sub ("^p4/", "", line
)
528 branches
[branch
] = parseRevision(line
)
531 def findUpstreamBranchPoint(head
= "HEAD"):
532 branches
= p4BranchesInGit()
533 # map from depot-path to branch name
534 branchByDepotPath
= {}
535 for branch
in branches
.keys():
536 tip
= branches
[branch
]
537 log
= extractLogMessageFromGitCommit(tip
)
538 settings
= extractSettingsGitLog(log
)
539 if settings
.has_key("depot-paths"):
540 paths
= ",".join(settings
["depot-paths"])
541 branchByDepotPath
[paths
] = "remotes/p4/" + branch
545 while parent
< 65535:
546 commit
= head
+ "~%s" % parent
547 log
= extractLogMessageFromGitCommit(commit
)
548 settings
= extractSettingsGitLog(log
)
549 if settings
.has_key("depot-paths"):
550 paths
= ",".join(settings
["depot-paths"])
551 if branchByDepotPath
.has_key(paths
):
552 return [branchByDepotPath
[paths
], settings
]
556 return ["", settings
]
558 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
560 print ("Creating/updating branch(es) in %s based on origin branch(es)"
563 originPrefix
= "origin/p4/"
565 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
567 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
570 headName
= line
[len(originPrefix
):]
571 remoteHead
= localRefPrefix
+ headName
574 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
575 if (not original
.has_key('depot-paths')
576 or not original
.has_key('change')):
580 if not gitBranchExists(remoteHead
):
582 print "creating %s" % remoteHead
585 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
586 if settings
.has_key('change') > 0:
587 if settings
['depot-paths'] == original
['depot-paths']:
588 originP4Change
= int(original
['change'])
589 p4Change
= int(settings
['change'])
590 if originP4Change
> p4Change
:
591 print ("%s (%s) is newer than %s (%s). "
592 "Updating p4 branch from origin."
593 % (originHead
, originP4Change
,
594 remoteHead
, p4Change
))
597 print ("Ignoring: %s was imported from %s while "
598 "%s was imported from %s"
599 % (originHead
, ','.join(original
['depot-paths']),
600 remoteHead
, ','.join(settings
['depot-paths'])))
603 system("git update-ref %s %s" % (remoteHead
, originHead
))
605 def originP4BranchesExist():
606 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
608 def p4ChangesForPaths(depotPaths
, changeRange
):
612 cmd
+= ["%s...%s" % (p
, changeRange
)]
613 output
= p4_read_pipe_lines(cmd
)
617 changeNum
= int(line
.split(" ")[1])
618 changes
[changeNum
] = True
620 changelist
= changes
.keys()
624 def p4PathStartsWith(path
, prefix
):
625 # This method tries to remedy a potential mixed-case issue:
627 # If UserA adds //depot/DirA/file1
628 # and UserB adds //depot/dira/file2
630 # we may or may not have a problem. If you have core.ignorecase=true,
631 # we treat DirA and dira as the same directory
632 ignorecase
= gitConfig("core.ignorecase", "--bool") == "true"
634 return path
.lower().startswith(prefix
.lower())
635 return path
.startswith(prefix
)
638 """Look at the p4 client spec, create a View() object that contains
639 all the mappings, and return it."""
641 specList
= p4CmdList("client -o")
642 if len(specList
) != 1:
643 die('Output from "client -o" is %d lines, expecting 1' %
646 # dictionary of all client parameters
649 # just the keys that start with "View"
650 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
655 # append the lines, in order, to the view
656 for view_num
in range(len(view_keys
)):
657 k
= "View%d" % view_num
658 if k
not in view_keys
:
659 die("Expected view key %s missing" % k
)
660 view
.append(entry
[k
])
665 """Grab the client directory."""
667 output
= p4CmdList("client -o")
669 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
672 if "Root" not in entry
:
673 die('Client has no "Root"')
678 # P4 wildcards are not allowed in filenames. P4 complains
679 # if you simply add them, but you can force it with "-f", in
680 # which case it translates them into %xx encoding internally.
682 def wildcard_decode(path
):
683 # Search for and fix just these four characters. Do % last so
684 # that fixing it does not inadvertently create new %-escapes.
685 # Cannot have * in a filename in windows; untested as to
686 # what p4 would do in such a case.
687 if not platform
.system() == "Windows":
688 path
= path
.replace("%2A", "*")
689 path
= path
.replace("%23", "#") \
690 .replace("%40", "@") \
694 def wildcard_encode(path
):
695 # do % first to avoid double-encoding the %s introduced here
696 path
= path
.replace("%", "%25") \
697 .replace("*", "%2A") \
698 .replace("#", "%23") \
702 def wildcard_present(path
):
703 return path
.translate(None, "*#@%") != path
707 self
.usage
= "usage: %prog [options]"
713 self
.userMapFromPerforceServer
= False
714 self
.myP4UserId
= None
718 return self
.myP4UserId
720 results
= p4CmdList("user -o")
722 if r
.has_key('User'):
723 self
.myP4UserId
= r
['User']
725 die("Could not find your p4 user id")
727 def p4UserIsMe(self
, p4User
):
728 # return True if the given p4 user is actually me
730 if not p4User
or p4User
!= me
:
735 def getUserCacheFilename(self
):
736 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
737 return home
+ "/.gitp4-usercache.txt"
739 def getUserMapFromPerforceServer(self
):
740 if self
.userMapFromPerforceServer
:
745 for output
in p4CmdList("users"):
746 if not output
.has_key("User"):
748 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
749 self
.emails
[output
["Email"]] = output
["User"]
753 for (key
, val
) in self
.users
.items():
754 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
756 open(self
.getUserCacheFilename(), "wb").write(s
)
757 self
.userMapFromPerforceServer
= True
759 def loadUserMapFromCache(self
):
761 self
.userMapFromPerforceServer
= False
763 cache
= open(self
.getUserCacheFilename(), "rb")
764 lines
= cache
.readlines()
767 entry
= line
.strip().split("\t")
768 self
.users
[entry
[0]] = entry
[1]
770 self
.getUserMapFromPerforceServer()
772 class P4Debug(Command
):
774 Command
.__init
__(self
)
776 self
.description
= "A tool to debug the output of p4 -G."
777 self
.needsGit
= False
781 for output
in p4CmdList(args
):
782 print 'Element: %d' % j
787 class P4RollBack(Command
):
789 Command
.__init
__(self
)
791 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
793 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
794 self
.rollbackLocalBranches
= False
799 maxChange
= int(args
[0])
801 if "p4ExitCode" in p4Cmd("changes -m 1"):
802 die("Problems executing p4");
804 if self
.rollbackLocalBranches
:
805 refPrefix
= "refs/heads/"
806 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
808 refPrefix
= "refs/remotes/"
809 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
812 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
814 ref
= refPrefix
+ line
815 log
= extractLogMessageFromGitCommit(ref
)
816 settings
= extractSettingsGitLog(log
)
818 depotPaths
= settings
['depot-paths']
819 change
= settings
['change']
823 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
824 for p
in depotPaths
]))) == 0:
825 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
826 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
829 while change
and int(change
) > maxChange
:
832 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
833 system("git update-ref %s \"%s^\"" % (ref
, ref
))
834 log
= extractLogMessageFromGitCommit(ref
)
835 settings
= extractSettingsGitLog(log
)
838 depotPaths
= settings
['depot-paths']
839 change
= settings
['change']
842 print "%s rewound to %s" % (ref
, change
)
846 class P4Submit(Command
, P4UserMap
):
848 Command
.__init
__(self
)
849 P4UserMap
.__init
__(self
)
851 optparse
.make_option("--origin", dest
="origin"),
852 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
853 # preserve the user, requires relevant p4 permissions
854 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
855 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
856 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
857 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
859 self
.description
= "Submit changes from git to the perforce depot."
860 self
.usage
+= " [name of git branch to submit into perforce depot]"
862 self
.detectRenames
= False
863 self
.preserveUser
= gitConfig("git-p4.preserveUser").lower() == "true"
865 self
.prepare_p4_only
= False
866 self
.isWindows
= (platform
.system() == "Windows")
867 self
.exportLabels
= False
868 self
.p4HasMoveCommand
= p4_has_command("move")
871 if len(p4CmdList("opened ...")) > 0:
872 die("You have files opened with perforce! Close them before starting the sync.")
874 def separate_jobs_from_description(self
, message
):
875 """Extract and return a possible Jobs field in the commit
876 message. It goes into a separate section in the p4 change
879 A jobs line starts with "Jobs:" and looks like a new field
880 in a form. Values are white-space separated on the same
881 line or on following lines that start with a tab.
883 This does not parse and extract the full git commit message
884 like a p4 form. It just sees the Jobs: line as a marker
885 to pass everything from then on directly into the p4 form,
886 but outside the description section.
888 Return a tuple (stripped log message, jobs string)."""
890 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
892 return (message
, None)
894 jobtext
= message
[m
.start():]
895 stripped_message
= message
[:m
.start()].rstrip()
896 return (stripped_message
, jobtext
)
898 def prepareLogMessage(self
, template
, message
, jobs
):
899 """Edits the template returned from "p4 change -o" to insert
900 the message in the Description field, and the jobs text in
904 inDescriptionSection
= False
906 for line
in template
.split("\n"):
907 if line
.startswith("#"):
908 result
+= line
+ "\n"
911 if inDescriptionSection
:
912 if line
.startswith("Files:") or line
.startswith("Jobs:"):
913 inDescriptionSection
= False
914 # insert Jobs section
916 result
+= jobs
+ "\n"
920 if line
.startswith("Description:"):
921 inDescriptionSection
= True
923 for messageLine
in message
.split("\n"):
924 line
+= "\t" + messageLine
+ "\n"
926 result
+= line
+ "\n"
930 def patchRCSKeywords(self
, file, pattern
):
931 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
932 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
934 outFile
= os
.fdopen(handle
, "w+")
935 inFile
= open(file, "r")
936 regexp
= re
.compile(pattern
, re
.VERBOSE
)
937 for line
in inFile
.readlines():
938 line
= regexp
.sub(r
'$\1$', line
)
942 # Forcibly overwrite the original file
944 shutil
.move(outFileName
, file)
946 # cleanup our temporary file
947 os
.unlink(outFileName
)
948 print "Failed to strip RCS keywords in %s" % file
951 print "Patched up RCS keywords in %s" % file
953 def p4UserForCommit(self
,id):
954 # Return the tuple (perforce user,git email) for a given git commit id
955 self
.getUserMapFromPerforceServer()
956 gitEmail
= read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
957 gitEmail
= gitEmail
.strip()
958 if not self
.emails
.has_key(gitEmail
):
959 return (None,gitEmail
)
961 return (self
.emails
[gitEmail
],gitEmail
)
963 def checkValidP4Users(self
,commits
):
964 # check if any git authors cannot be mapped to p4 users
966 (user
,email
) = self
.p4UserForCommit(id)
968 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
969 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
972 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
974 def lastP4Changelist(self
):
975 # Get back the last changelist number submitted in this client spec. This
976 # then gets used to patch up the username in the change. If the same
977 # client spec is being used by multiple processes then this might go
979 results
= p4CmdList("client -o") # find the current client
982 if r
.has_key('Client'):
986 die("could not get client spec")
987 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
989 if r
.has_key('change'):
991 die("Could not get changelist number for last submit - cannot patch up user details")
993 def modifyChangelistUser(self
, changelist
, newUser
):
994 # fixup the user field of a changelist after it has been submitted.
995 changes
= p4CmdList("change -o %s" % changelist
)
996 if len(changes
) != 1:
997 die("Bad output from p4 change modifying %s to user %s" %
998 (changelist
, newUser
))
1001 if c
['User'] == newUser
: return # nothing to do
1003 input = marshal
.dumps(c
)
1005 result
= p4CmdList("change -f -i", stdin
=input)
1007 if r
.has_key('code'):
1008 if r
['code'] == 'error':
1009 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1010 if r
.has_key('data'):
1011 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1013 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1015 def canChangeChangelists(self
):
1016 # check to see if we have p4 admin or super-user permissions, either of
1017 # which are required to modify changelists.
1018 results
= p4CmdList(["protects", self
.depotPath
])
1020 if r
.has_key('perm'):
1021 if r
['perm'] == 'admin':
1023 if r
['perm'] == 'super':
1027 def prepareSubmitTemplate(self
):
1028 """Run "p4 change -o" to grab a change specification template.
1029 This does not use "p4 -G", as it is nice to keep the submission
1030 template in original order, since a human might edit it.
1032 Remove lines in the Files section that show changes to files
1033 outside the depot path we're committing into."""
1036 inFilesSection
= False
1037 for line
in p4_read_pipe_lines(['change', '-o']):
1038 if line
.endswith("\r\n"):
1039 line
= line
[:-2] + "\n"
1041 if line
.startswith("\t"):
1042 # path starts and ends with a tab
1044 lastTab
= path
.rfind("\t")
1046 path
= path
[:lastTab
]
1047 if not p4PathStartsWith(path
, self
.depotPath
):
1050 inFilesSection
= False
1052 if line
.startswith("Files:"):
1053 inFilesSection
= True
1059 def edit_template(self
, template_file
):
1060 """Invoke the editor to let the user change the submission
1061 message. Return true if okay to continue with the submit."""
1063 # if configured to skip the editing part, just submit
1064 if gitConfig("git-p4.skipSubmitEdit") == "true":
1067 # look at the modification time, to check later if the user saved
1069 mtime
= os
.stat(template_file
).st_mtime
1072 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1073 editor
= os
.environ
.get("P4EDITOR")
1075 editor
= read_pipe("git var GIT_EDITOR").strip()
1076 system(editor
+ " " + template_file
)
1078 # If the file was not saved, prompt to see if this patch should
1079 # be skipped. But skip this verification step if configured so.
1080 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1083 # modification time updated means user saved the file
1084 if os
.stat(template_file
).st_mtime
> mtime
:
1088 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1094 def applyCommit(self
, id):
1095 """Apply one commit, return True if it succeeded."""
1097 print "Applying", read_pipe(["git", "show", "-s",
1098 "--format=format:%h %s", id])
1100 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1102 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1104 filesToDelete
= set()
1106 pureRenameCopy
= set()
1107 filesToChangeExecBit
= {}
1110 diff
= parseDiffTreeEntry(line
)
1111 modifier
= diff
['status']
1115 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1116 filesToChangeExecBit
[path
] = diff
['dst_mode']
1117 editedFiles
.add(path
)
1118 elif modifier
== "A":
1119 filesToAdd
.add(path
)
1120 filesToChangeExecBit
[path
] = diff
['dst_mode']
1121 if path
in filesToDelete
:
1122 filesToDelete
.remove(path
)
1123 elif modifier
== "D":
1124 filesToDelete
.add(path
)
1125 if path
in filesToAdd
:
1126 filesToAdd
.remove(path
)
1127 elif modifier
== "C":
1128 src
, dest
= diff
['src'], diff
['dst']
1129 p4_integrate(src
, dest
)
1130 pureRenameCopy
.add(dest
)
1131 if diff
['src_sha1'] != diff
['dst_sha1']:
1133 pureRenameCopy
.discard(dest
)
1134 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1136 pureRenameCopy
.discard(dest
)
1137 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1139 editedFiles
.add(dest
)
1140 elif modifier
== "R":
1141 src
, dest
= diff
['src'], diff
['dst']
1142 if self
.p4HasMoveCommand
:
1143 p4_edit(src
) # src must be open before move
1144 p4_move(src
, dest
) # opens for (move/delete, move/add)
1146 p4_integrate(src
, dest
)
1147 if diff
['src_sha1'] != diff
['dst_sha1']:
1150 pureRenameCopy
.add(dest
)
1151 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1152 if not self
.p4HasMoveCommand
:
1153 p4_edit(dest
) # with move: already open, writable
1154 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1155 if not self
.p4HasMoveCommand
:
1157 filesToDelete
.add(src
)
1158 editedFiles
.add(dest
)
1160 die("unknown modifier %s for %s" % (modifier
, path
))
1162 diffcmd
= "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1163 patchcmd
= diffcmd
+ " | git apply "
1164 tryPatchCmd
= patchcmd
+ "--check -"
1165 applyPatchCmd
= patchcmd
+ "--check --apply -"
1166 patch_succeeded
= True
1168 if os
.system(tryPatchCmd
) != 0:
1169 fixed_rcs_keywords
= False
1170 patch_succeeded
= False
1171 print "Unfortunately applying the change failed!"
1173 # Patch failed, maybe it's just RCS keyword woes. Look through
1174 # the patch to see if that's possible.
1175 if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1179 for file in editedFiles | filesToDelete
:
1180 # did this file's delta contain RCS keywords?
1181 pattern
= p4_keywords_regexp_for_file(file)
1184 # this file is a possibility...look for RCS keywords.
1185 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1186 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1187 if regexp
.search(line
):
1189 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1190 kwfiles
[file] = pattern
1193 for file in kwfiles
:
1195 print "zapping %s with %s" % (line
,pattern
)
1196 self
.patchRCSKeywords(file, kwfiles
[file])
1197 fixed_rcs_keywords
= True
1199 if fixed_rcs_keywords
:
1200 print "Retrying the patch with RCS keywords cleaned up"
1201 if os
.system(tryPatchCmd
) == 0:
1202 patch_succeeded
= True
1204 if not patch_succeeded
:
1205 for f
in editedFiles
:
1210 # Apply the patch for real, and do add/delete/+x handling.
1212 system(applyPatchCmd
)
1214 for f
in filesToAdd
:
1216 for f
in filesToDelete
:
1220 # Set/clear executable bits
1221 for f
in filesToChangeExecBit
.keys():
1222 mode
= filesToChangeExecBit
[f
]
1223 setP4ExecBit(f
, mode
)
1226 # Build p4 change description, starting with the contents
1227 # of the git commit message.
1229 logMessage
= extractLogMessageFromGitCommit(id)
1230 logMessage
= logMessage
.strip()
1231 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1233 template
= self
.prepareSubmitTemplate()
1234 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1236 if self
.preserveUser
:
1237 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
1239 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1240 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1241 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1242 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1244 separatorLine
= "######## everything below this line is just the diff #######\n"
1247 if os
.environ
.has_key("P4DIFF"):
1248 del(os
.environ
["P4DIFF"])
1250 for editedFile
in editedFiles
:
1251 diff
+= p4_read_pipe(['diff', '-du',
1252 wildcard_encode(editedFile
)])
1256 for newFile
in filesToAdd
:
1257 newdiff
+= "==== new file ====\n"
1258 newdiff
+= "--- /dev/null\n"
1259 newdiff
+= "+++ %s\n" % newFile
1260 f
= open(newFile
, "r")
1261 for line
in f
.readlines():
1262 newdiff
+= "+" + line
1265 # change description file: submitTemplate, separatorLine, diff, newdiff
1266 (handle
, fileName
) = tempfile
.mkstemp()
1267 tmpFile
= os
.fdopen(handle
, "w+")
1269 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1270 separatorLine
= separatorLine
.replace("\n", "\r\n")
1271 newdiff
= newdiff
.replace("\n", "\r\n")
1272 tmpFile
.write(submitTemplate
+ separatorLine
+ diff
+ newdiff
)
1275 if self
.prepare_p4_only
:
1277 # Leave the p4 tree prepared, and the submit template around
1278 # and let the user decide what to do next
1281 print "P4 workspace prepared for submission."
1282 print "To submit or revert, go to client workspace"
1283 print " " + self
.clientPath
1285 print "To submit, use \"p4 submit\" to write a new description,"
1286 print "or \"p4 submit -i %s\" to use the one prepared by" \
1287 " \"git p4\"." % fileName
1288 print "You can delete the file \"%s\" when finished." % fileName
1290 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
1291 print "To preserve change ownership by user %s, you must\n" \
1292 "do \"p4 change -f <change>\" after submitting and\n" \
1293 "edit the User field."
1295 print "After submitting, renamed files must be re-synced."
1296 print "Invoke \"p4 sync -f\" on each of these files:"
1297 for f
in pureRenameCopy
:
1301 print "To revert the changes, use \"p4 revert ...\", and delete"
1302 print "the submit template file \"%s\"" % fileName
1304 print "Since the commit adds new files, they must be deleted:"
1305 for f
in filesToAdd
:
1311 # Let the user edit the change description, then submit it.
1313 if self
.edit_template(fileName
):
1314 # read the edited message and submit
1316 tmpFile
= open(fileName
, "rb")
1317 message
= tmpFile
.read()
1319 submitTemplate
= message
[:message
.index(separatorLine
)]
1321 submitTemplate
= submitTemplate
.replace("\r\n", "\n")
1322 p4_write_pipe(['submit', '-i'], submitTemplate
)
1324 if self
.preserveUser
:
1326 # Get last changelist number. Cannot easily get it from
1327 # the submit command output as the output is
1329 changelist
= self
.lastP4Changelist()
1330 self
.modifyChangelistUser(changelist
, p4User
)
1332 # The rename/copy happened by applying a patch that created a
1333 # new file. This leaves it writable, which confuses p4.
1334 for f
in pureRenameCopy
:
1340 print "Submission cancelled, undoing p4 changes."
1341 for f
in editedFiles
:
1343 for f
in filesToAdd
:
1346 for f
in filesToDelete
:
1352 # Export git tags as p4 labels. Create a p4 label and then tag
1354 def exportGitTags(self
, gitTags
):
1355 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1356 if len(validLabelRegexp
) == 0:
1357 validLabelRegexp
= defaultLabelRegexp
1358 m
= re
.compile(validLabelRegexp
)
1360 for name
in gitTags
:
1362 if not m
.match(name
):
1364 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1367 # Get the p4 commit this corresponds to
1368 logMessage
= extractLogMessageFromGitCommit(name
)
1369 values
= extractSettingsGitLog(logMessage
)
1371 if not values
.has_key('change'):
1372 # a tag pointing to something not sent to p4; ignore
1374 print "git tag %s does not give a p4 commit" % name
1377 changelist
= values
['change']
1379 # Get the tag details.
1383 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1386 if re
.match(r
'tag\s+', l
):
1388 elif re
.match(r
'\s*$', l
):
1395 body
= ["lightweight tag imported by git p4\n"]
1397 # Create the label - use the same view as the client spec we are using
1398 clientSpec
= getClientSpec()
1400 labelTemplate
= "Label: %s\n" % name
1401 labelTemplate
+= "Description:\n"
1403 labelTemplate
+= "\t" + b
+ "\n"
1404 labelTemplate
+= "View:\n"
1405 for mapping
in clientSpec
.mappings
:
1406 labelTemplate
+= "\t%s\n" % mapping
.depot_side
.path
1409 print "Would create p4 label %s for tag" % name
1410 elif self
.prepare_p4_only
:
1411 print "Not creating p4 label %s for tag due to option" \
1412 " --prepare-p4-only" % name
1414 p4_write_pipe(["label", "-i"], labelTemplate
)
1417 p4_system(["tag", "-l", name
] +
1418 ["%s@%s" % (mapping
.depot_side
.path
, changelist
) for mapping
in clientSpec
.mappings
])
1421 print "created p4 label for tag %s" % name
1423 def run(self
, args
):
1425 self
.master
= currentGitBranch()
1426 if len(self
.master
) == 0 or not gitBranchExists("refs/heads/%s" % self
.master
):
1427 die("Detecting current git branch failed!")
1428 elif len(args
) == 1:
1429 self
.master
= args
[0]
1430 if not branchExists(self
.master
):
1431 die("Branch %s does not exist" % self
.master
)
1435 allowSubmit
= gitConfig("git-p4.allowSubmit")
1436 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1437 die("%s is not in git-p4.allowSubmit" % self
.master
)
1439 [upstream
, settings
] = findUpstreamBranchPoint()
1440 self
.depotPath
= settings
['depot-paths'][0]
1441 if len(self
.origin
) == 0:
1442 self
.origin
= upstream
1444 if self
.preserveUser
:
1445 if not self
.canChangeChangelists():
1446 die("Cannot preserve user names without p4 super-user or admin permissions")
1449 print "Origin branch is " + self
.origin
1451 if len(self
.depotPath
) == 0:
1452 print "Internal error: cannot locate perforce depot path from existing branches"
1455 self
.useClientSpec
= False
1456 if gitConfig("git-p4.useclientspec", "--bool") == "true":
1457 self
.useClientSpec
= True
1458 if self
.useClientSpec
:
1459 self
.clientSpecDirs
= getClientSpec()
1461 if self
.useClientSpec
:
1462 # all files are relative to the client spec
1463 self
.clientPath
= getClientRoot()
1465 self
.clientPath
= p4Where(self
.depotPath
)
1467 if self
.clientPath
== "":
1468 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1470 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1471 self
.oldWorkingDirectory
= os
.getcwd()
1473 # ensure the clientPath exists
1474 new_client_dir
= False
1475 if not os
.path
.exists(self
.clientPath
):
1476 new_client_dir
= True
1477 os
.makedirs(self
.clientPath
)
1479 chdir(self
.clientPath
)
1481 print "Would synchronize p4 checkout in %s" % self
.clientPath
1483 print "Synchronizing p4 checkout..."
1485 # old one was destroyed, and maybe nobody told p4
1486 p4_sync("...", "-f")
1492 for line
in read_pipe_lines("git rev-list --no-merges %s..%s" % (self
.origin
, self
.master
)):
1493 commits
.append(line
.strip())
1496 if self
.preserveUser
or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1497 self
.checkAuthorship
= False
1499 self
.checkAuthorship
= True
1501 if self
.preserveUser
:
1502 self
.checkValidP4Users(commits
)
1505 # Build up a set of options to be passed to diff when
1506 # submitting each commit to p4.
1508 if self
.detectRenames
:
1509 # command-line -M arg
1510 self
.diffOpts
= "-M"
1512 # If not explicitly set check the config variable
1513 detectRenames
= gitConfig("git-p4.detectRenames")
1515 if detectRenames
.lower() == "false" or detectRenames
== "":
1517 elif detectRenames
.lower() == "true":
1518 self
.diffOpts
= "-M"
1520 self
.diffOpts
= "-M%s" % detectRenames
1522 # no command-line arg for -C or --find-copies-harder, just
1524 detectCopies
= gitConfig("git-p4.detectCopies")
1525 if detectCopies
.lower() == "false" or detectCopies
== "":
1527 elif detectCopies
.lower() == "true":
1528 self
.diffOpts
+= " -C"
1530 self
.diffOpts
+= " -C%s" % detectCopies
1532 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1533 self
.diffOpts
+= " --find-copies-harder"
1536 # Apply the commits, one at a time. On failure, ask if should
1537 # continue to try the rest of the patches, or quit.
1542 last
= len(commits
) - 1
1543 for i
, commit
in enumerate(commits
):
1545 print " ", read_pipe(["git", "show", "-s",
1546 "--format=format:%h %s", commit
])
1549 ok
= self
.applyCommit(commit
)
1551 applied
.append(commit
)
1553 if self
.prepare_p4_only
and i
< last
:
1554 print "Processing only the first commit due to option" \
1555 " --prepare-p4-only"
1560 print "What do you want to do?"
1561 response
= raw_input("[s]kip this commit but apply"
1562 " the rest, or [q]uit? ")
1565 if response
[0] == "s":
1566 print "Skipping this commit, but applying the rest"
1568 if response
[0] == "q":
1575 chdir(self
.oldWorkingDirectory
)
1579 elif self
.prepare_p4_only
:
1581 elif len(commits
) == len(applied
):
1582 print "All commits applied!"
1591 if len(applied
) == 0:
1592 print "No commits applied."
1594 print "Applied only the commits marked with '*':"
1600 print star
, read_pipe(["git", "show", "-s",
1601 "--format=format:%h %s", c
])
1602 print "You will have to do 'git p4 sync' and rebase."
1604 if gitConfig("git-p4.exportLabels", "--bool") == "true":
1605 self
.exportLabels
= True
1607 if self
.exportLabels
:
1608 p4Labels
= getP4Labels(self
.depotPath
)
1609 gitTags
= getGitTags()
1611 missingGitTags
= gitTags
- p4Labels
1612 self
.exportGitTags(missingGitTags
)
1614 # exit with error unless everything applied perfecly
1615 if len(commits
) != len(applied
):
1621 """Represent a p4 view ("p4 help views"), and map files in a
1622 repo according to the view."""
1625 """A depot or client path, possibly containing wildcards.
1626 The only one supported is ... at the end, currently.
1627 Initialize with the full path, with //depot or //client."""
1629 def __init__(self
, path
, is_depot
):
1631 self
.is_depot
= is_depot
1632 self
.find_wildcards()
1633 # remember the prefix bit, useful for relative mappings
1634 m
= re
.match("(//[^/]+/)", self
.path
)
1636 die("Path %s does not start with //prefix/" % self
.path
)
1638 if not self
.is_depot
:
1639 # strip //client/ on client paths
1640 self
.path
= self
.path
[len(prefix
):]
1642 def find_wildcards(self
):
1643 """Make sure wildcards are valid, and set up internal
1646 self
.ends_triple_dot
= False
1647 # There are three wildcards allowed in p4 views
1648 # (see "p4 help views"). This code knows how to
1649 # handle "..." (only at the end), but cannot deal with
1650 # "%%n" or "*". Only check the depot_side, as p4 should
1651 # validate that the client_side matches too.
1652 if re
.search(r
'%%[1-9]', self
.path
):
1653 die("Can't handle %%n wildcards in view: %s" % self
.path
)
1654 if self
.path
.find("*") >= 0:
1655 die("Can't handle * wildcards in view: %s" % self
.path
)
1656 triple_dot_index
= self
.path
.find("...")
1657 if triple_dot_index
>= 0:
1658 if triple_dot_index
!= len(self
.path
) - 3:
1659 die("Can handle only single ... wildcard, at end: %s" %
1661 self
.ends_triple_dot
= True
1663 def ensure_compatible(self
, other_path
):
1664 """Make sure the wildcards agree."""
1665 if self
.ends_triple_dot
!= other_path
.ends_triple_dot
:
1666 die("Both paths must end with ... if either does;\n" +
1667 "paths: %s %s" % (self
.path
, other_path
.path
))
1669 def match_wildcards(self
, test_path
):
1670 """See if this test_path matches us, and fill in the value
1671 of the wildcards if so. Returns a tuple of
1672 (True|False, wildcards[]). For now, only the ... at end
1673 is supported, so at most one wildcard."""
1674 if self
.ends_triple_dot
:
1675 dotless
= self
.path
[:-3]
1676 if test_path
.startswith(dotless
):
1677 wildcard
= test_path
[len(dotless
):]
1678 return (True, [ wildcard
])
1680 if test_path
== self
.path
:
1684 def match(self
, test_path
):
1685 """Just return if it matches; don't bother with the wildcards."""
1686 b
, _
= self
.match_wildcards(test_path
)
1689 def fill_in_wildcards(self
, wildcards
):
1690 """Return the relative path, with the wildcards filled in
1691 if there are any."""
1692 if self
.ends_triple_dot
:
1693 return self
.path
[:-3] + wildcards
[0]
1697 class Mapping(object):
1698 def __init__(self
, depot_side
, client_side
, overlay
, exclude
):
1699 # depot_side is without the trailing /... if it had one
1700 self
.depot_side
= View
.Path(depot_side
, is_depot
=True)
1701 self
.client_side
= View
.Path(client_side
, is_depot
=False)
1702 self
.overlay
= overlay
# started with "+"
1703 self
.exclude
= exclude
# started with "-"
1704 assert not (self
.overlay
and self
.exclude
)
1705 self
.depot_side
.ensure_compatible(self
.client_side
)
1713 return "View.Mapping: %s%s -> %s" % \
1714 (c
, self
.depot_side
.path
, self
.client_side
.path
)
1716 def map_depot_to_client(self
, depot_path
):
1717 """Calculate the client path if using this mapping on the
1718 given depot path; does not consider the effect of other
1719 mappings in a view. Even excluded mappings are returned."""
1720 matches
, wildcards
= self
.depot_side
.match_wildcards(depot_path
)
1723 client_path
= self
.client_side
.fill_in_wildcards(wildcards
)
1732 def append(self
, view_line
):
1733 """Parse a view line, splitting it into depot and client
1734 sides. Append to self.mappings, preserving order."""
1736 # Split the view line into exactly two words. P4 enforces
1737 # structure on these lines that simplifies this quite a bit.
1739 # Either or both words may be double-quoted.
1740 # Single quotes do not matter.
1741 # Double-quote marks cannot occur inside the words.
1742 # A + or - prefix is also inside the quotes.
1743 # There are no quotes unless they contain a space.
1744 # The line is already white-space stripped.
1745 # The two words are separated by a single space.
1747 if view_line
[0] == '"':
1748 # First word is double quoted. Find its end.
1749 close_quote_index
= view_line
.find('"', 1)
1750 if close_quote_index
<= 0:
1751 die("No first-word closing quote found: %s" % view_line
)
1752 depot_side
= view_line
[1:close_quote_index
]
1753 # skip closing quote and space
1754 rhs_index
= close_quote_index
+ 1 + 1
1756 space_index
= view_line
.find(" ")
1757 if space_index
<= 0:
1758 die("No word-splitting space found: %s" % view_line
)
1759 depot_side
= view_line
[0:space_index
]
1760 rhs_index
= space_index
+ 1
1762 if view_line
[rhs_index
] == '"':
1763 # Second word is double quoted. Make sure there is a
1764 # double quote at the end too.
1765 if not view_line
.endswith('"'):
1766 die("View line with rhs quote should end with one: %s" %
1769 client_side
= view_line
[rhs_index
+1:-1]
1771 client_side
= view_line
[rhs_index
:]
1773 # prefix + means overlay on previous mapping
1775 if depot_side
.startswith("+"):
1777 depot_side
= depot_side
[1:]
1779 # prefix - means exclude this path
1781 if depot_side
.startswith("-"):
1783 depot_side
= depot_side
[1:]
1785 m
= View
.Mapping(depot_side
, client_side
, overlay
, exclude
)
1786 self
.mappings
.append(m
)
1788 def map_in_client(self
, depot_path
):
1789 """Return the relative location in the client where this
1790 depot file should live. Returns "" if the file should
1791 not be mapped in the client."""
1796 # look at later entries first
1797 for m
in self
.mappings
[::-1]:
1799 # see where will this path end up in the client
1800 p
= m
.map_depot_to_client(depot_path
)
1803 # Depot path does not belong in client. Must remember
1804 # this, as previous items should not cause files to
1805 # exist in this path either. Remember that the list is
1806 # being walked from the end, which has higher precedence.
1807 # Overlap mappings do not exclude previous mappings.
1809 paths_filled
.append(m
.client_side
)
1812 # This mapping matched; no need to search any further.
1813 # But, the mapping could be rejected if the client path
1814 # has already been claimed by an earlier mapping (i.e.
1815 # one later in the list, which we are walking backwards).
1816 already_mapped_in_client
= False
1817 for f
in paths_filled
:
1818 # this is View.Path.match
1820 already_mapped_in_client
= True
1822 if not already_mapped_in_client
:
1823 # Include this file, unless it is from a line that
1824 # explicitly said to exclude it.
1828 # a match, even if rejected, always stops the search
1833 class P4Sync(Command
, P4UserMap
):
1834 delete_actions
= ( "delete", "move/delete", "purge" )
1837 Command
.__init
__(self
)
1838 P4UserMap
.__init
__(self
)
1840 optparse
.make_option("--branch", dest
="branch"),
1841 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
1842 optparse
.make_option("--changesfile", dest
="changesFile"),
1843 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
1844 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
1845 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
1846 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
1847 help="Import into refs/heads/ , not refs/remotes"),
1848 optparse
.make_option("--max-changes", dest
="maxChanges"),
1849 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
1850 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1851 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
1852 help="Only sync files that are included in the Perforce Client Spec")
1854 self
.description
= """Imports from Perforce into a git repository.\n
1856 //depot/my/project/ -- to import the current head
1857 //depot/my/project/@all -- to import everything
1858 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1860 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1862 self
.usage
+= " //depot/path[@revRange]"
1864 self
.createdBranches
= set()
1865 self
.committedChanges
= set()
1867 self
.detectBranches
= False
1868 self
.detectLabels
= False
1869 self
.importLabels
= False
1870 self
.changesFile
= ""
1871 self
.syncWithOrigin
= True
1872 self
.importIntoRemotes
= True
1873 self
.maxChanges
= ""
1874 self
.isWindows
= (platform
.system() == "Windows")
1875 self
.keepRepoPath
= False
1876 self
.depotPaths
= None
1877 self
.p4BranchesInGit
= []
1878 self
.cloneExclude
= []
1879 self
.useClientSpec
= False
1880 self
.useClientSpec_from_options
= False
1881 self
.clientSpecDirs
= None
1882 self
.tempBranches
= []
1883 self
.tempBranchLocation
= "git-p4-tmp"
1885 if gitConfig("git-p4.syncFromOrigin") == "false":
1886 self
.syncWithOrigin
= False
1888 # Force a checkpoint in fast-import and wait for it to finish
1889 def checkpoint(self
):
1890 self
.gitStream
.write("checkpoint\n\n")
1891 self
.gitStream
.write("progress checkpoint\n\n")
1892 out
= self
.gitOutput
.readline()
1894 print "checkpoint finished: " + out
1896 def extractFilesFromCommit(self
, commit
):
1897 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
1898 for path
in self
.cloneExclude
]
1901 while commit
.has_key("depotFile%s" % fnum
):
1902 path
= commit
["depotFile%s" % fnum
]
1904 if [p
for p
in self
.cloneExclude
1905 if p4PathStartsWith(path
, p
)]:
1908 found
= [p
for p
in self
.depotPaths
1909 if p4PathStartsWith(path
, p
)]
1916 file["rev"] = commit
["rev%s" % fnum
]
1917 file["action"] = commit
["action%s" % fnum
]
1918 file["type"] = commit
["type%s" % fnum
]
1923 def stripRepoPath(self
, path
, prefixes
):
1924 if self
.useClientSpec
:
1925 return self
.clientSpecDirs
.map_in_client(path
)
1927 if self
.keepRepoPath
:
1928 prefixes
= [re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])]
1931 if p4PathStartsWith(path
, p
):
1932 path
= path
[len(p
):]
1936 def splitFilesIntoBranches(self
, commit
):
1939 while commit
.has_key("depotFile%s" % fnum
):
1940 path
= commit
["depotFile%s" % fnum
]
1941 found
= [p
for p
in self
.depotPaths
1942 if p4PathStartsWith(path
, p
)]
1949 file["rev"] = commit
["rev%s" % fnum
]
1950 file["action"] = commit
["action%s" % fnum
]
1951 file["type"] = commit
["type%s" % fnum
]
1954 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
1955 relPath
= wildcard_decode(relPath
)
1957 for branch
in self
.knownBranches
.keys():
1959 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1960 if relPath
.startswith(branch
+ "/"):
1961 if branch
not in branches
:
1962 branches
[branch
] = []
1963 branches
[branch
].append(file)
1968 # output one file from the P4 stream
1969 # - helper for streamP4Files
1971 def streamOneP4File(self
, file, contents
):
1972 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
1973 relPath
= wildcard_decode(relPath
)
1975 sys
.stderr
.write("%s\n" % relPath
)
1977 (type_base
, type_mods
) = split_p4_type(file["type"])
1980 if "x" in type_mods
:
1982 if type_base
== "symlink":
1984 # p4 print on a symlink contains "target\n"; remove the newline
1985 data
= ''.join(contents
)
1986 contents
= [data
[:-1]]
1988 if type_base
== "utf16":
1989 # p4 delivers different text in the python output to -G
1990 # than it does when using "print -o", or normal p4 client
1991 # operations. utf16 is converted to ascii or utf8, perhaps.
1992 # But ascii text saved as -t utf16 is completely mangled.
1993 # Invoke print -o to get the real contents.
1994 text
= p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1997 if type_base
== "apple":
1998 # Apple filetype files will be streamed as a concatenation of
1999 # its appledouble header and the contents. This is useless
2000 # on both macs and non-macs. If using "print -q -o xx", it
2001 # will create "xx" with the data, and "%xx" with the header.
2002 # This is also not very useful.
2004 # Ideally, someday, this script can learn how to generate
2005 # appledouble files directly and import those to git, but
2006 # non-mac machines can never find a use for apple filetype.
2007 print "\nIgnoring apple filetype file %s" % file['depotFile']
2010 # Perhaps windows wants unicode, utf16 newlines translated too;
2011 # but this is not doing it.
2012 if self
.isWindows
and type_base
== "text":
2014 for data
in contents
:
2015 data
= data
.replace("\r\n", "\n")
2016 mangled
.append(data
)
2019 # Note that we do not try to de-mangle keywords on utf16 files,
2020 # even though in theory somebody may want that.
2021 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
2023 regexp
= re
.compile(pattern
, re
.VERBOSE
)
2024 text
= ''.join(contents
)
2025 text
= regexp
.sub(r
'$\1$', text
)
2028 self
.gitStream
.write("M %s inline %s\n" % (git_mode
, relPath
))
2033 length
= length
+ len(d
)
2035 self
.gitStream
.write("data %d\n" % length
)
2037 self
.gitStream
.write(d
)
2038 self
.gitStream
.write("\n")
2040 def streamOneP4Deletion(self
, file):
2041 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
2042 relPath
= wildcard_decode(relPath
)
2044 sys
.stderr
.write("delete %s\n" % relPath
)
2045 self
.gitStream
.write("D %s\n" % relPath
)
2047 # handle another chunk of streaming data
2048 def streamP4FilesCb(self
, marshalled
):
2050 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
2051 # start of a new file - output the old one first
2052 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2053 self
.stream_file
= {}
2054 self
.stream_contents
= []
2055 self
.stream_have_file_info
= False
2057 # pick up the new file information... for the
2058 # 'data' field we need to append to our array
2059 for k
in marshalled
.keys():
2061 self
.stream_contents
.append(marshalled
['data'])
2063 self
.stream_file
[k
] = marshalled
[k
]
2065 self
.stream_have_file_info
= True
2067 # Stream directly from "p4 files" into "git fast-import"
2068 def streamP4Files(self
, files
):
2074 # if using a client spec, only add the files that have
2075 # a path in the client
2076 if self
.clientSpecDirs
:
2077 if self
.clientSpecDirs
.map_in_client(f
['path']) == "":
2080 filesForCommit
.append(f
)
2081 if f
['action'] in self
.delete_actions
:
2082 filesToDelete
.append(f
)
2084 filesToRead
.append(f
)
2087 for f
in filesToDelete
:
2088 self
.streamOneP4Deletion(f
)
2090 if len(filesToRead
) > 0:
2091 self
.stream_file
= {}
2092 self
.stream_contents
= []
2093 self
.stream_have_file_info
= False
2095 # curry self argument
2096 def streamP4FilesCbSelf(entry
):
2097 self
.streamP4FilesCb(entry
)
2099 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
2101 p4CmdList(["-x", "-", "print"],
2103 cb
=streamP4FilesCbSelf
)
2106 if self
.stream_file
.has_key('depotFile'):
2107 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2109 def make_email(self
, userid
):
2110 if userid
in self
.users
:
2111 return self
.users
[userid
]
2113 return "%s <a@b>" % userid
2116 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
2118 print "writing tag %s for commit %s" % (labelName
, commit
)
2119 gitStream
.write("tag %s\n" % labelName
)
2120 gitStream
.write("from %s\n" % commit
)
2122 if labelDetails
.has_key('Owner'):
2123 owner
= labelDetails
["Owner"]
2127 # Try to use the owner of the p4 label, or failing that,
2128 # the current p4 user id.
2130 email
= self
.make_email(owner
)
2132 email
= self
.make_email(self
.p4UserId())
2133 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2135 gitStream
.write("tagger %s\n" % tagger
)
2137 print "labelDetails=",labelDetails
2138 if labelDetails
.has_key('Description'):
2139 description
= labelDetails
['Description']
2141 description
= 'Label from git p4'
2143 gitStream
.write("data %d\n" % len(description
))
2144 gitStream
.write(description
)
2145 gitStream
.write("\n")
2147 def commit(self
, details
, files
, branch
, branchPrefixes
, parent
= ""):
2148 epoch
= details
["time"]
2149 author
= details
["user"]
2150 self
.branchPrefixes
= branchPrefixes
2153 print "commit into %s" % branch
2155 # start with reading files; if that fails, we should not
2159 if [p
for p
in branchPrefixes
if p4PathStartsWith(f
['path'], p
)]:
2160 new_files
.append (f
)
2162 sys
.stderr
.write("Ignoring file outside of prefix: %s\n" % f
['path'])
2164 self
.gitStream
.write("commit %s\n" % branch
)
2165 # gitStream.write("mark :%s\n" % details["change"])
2166 self
.committedChanges
.add(int(details
["change"]))
2168 if author
not in self
.users
:
2169 self
.getUserMapFromPerforceServer()
2170 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2172 self
.gitStream
.write("committer %s\n" % committer
)
2174 self
.gitStream
.write("data <<EOT\n")
2175 self
.gitStream
.write(details
["desc"])
2176 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s"
2177 % (','.join (branchPrefixes
), details
["change"]))
2178 if len(details
['options']) > 0:
2179 self
.gitStream
.write(": options = %s" % details
['options'])
2180 self
.gitStream
.write("]\nEOT\n\n")
2184 print "parent %s" % parent
2185 self
.gitStream
.write("from %s\n" % parent
)
2187 self
.streamP4Files(new_files
)
2188 self
.gitStream
.write("\n")
2190 change
= int(details
["change"])
2192 if self
.labels
.has_key(change
):
2193 label
= self
.labels
[change
]
2194 labelDetails
= label
[0]
2195 labelRevisions
= label
[1]
2197 print "Change %s is labelled %s" % (change
, labelDetails
)
2199 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2200 for p
in branchPrefixes
])
2202 if len(files
) == len(labelRevisions
):
2206 if info
["action"] in self
.delete_actions
:
2208 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2210 if cleanedFiles
== labelRevisions
:
2211 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2215 print ("Tag %s does not match with change %s: files do not match."
2216 % (labelDetails
["label"], change
))
2220 print ("Tag %s does not match with change %s: file count is different."
2221 % (labelDetails
["label"], change
))
2223 # Build a dictionary of changelists and labels, for "detect-labels" option.
2224 def getLabels(self
):
2227 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2228 if len(l
) > 0 and not self
.silent
:
2229 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2232 label
= output
["label"]
2236 print "Querying files for label %s" % label
2237 for file in p4CmdList(["files"] +
2238 ["%s...@%s" % (p
, label
)
2239 for p
in self
.depotPaths
]):
2240 revisions
[file["depotFile"]] = file["rev"]
2241 change
= int(file["change"])
2242 if change
> newestChange
:
2243 newestChange
= change
2245 self
.labels
[newestChange
] = [output
, revisions
]
2248 print "Label changes: %s" % self
.labels
.keys()
2250 # Import p4 labels as git tags. A direct mapping does not
2251 # exist, so assume that if all the files are at the same revision
2252 # then we can use that, or it's something more complicated we should
2254 def importP4Labels(self
, stream
, p4Labels
):
2256 print "import p4 labels: " + ' '.join(p4Labels
)
2258 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2259 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2260 if len(validLabelRegexp
) == 0:
2261 validLabelRegexp
= defaultLabelRegexp
2262 m
= re
.compile(validLabelRegexp
)
2264 for name
in p4Labels
:
2267 if not m
.match(name
):
2269 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2272 if name
in ignoredP4Labels
:
2275 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2277 # get the most recent changelist for each file in this label
2278 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2279 for p
in self
.depotPaths
])
2281 if change
.has_key('change'):
2282 # find the corresponding git commit; take the oldest commit
2283 changelist
= int(change
['change'])
2284 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2285 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
])
2286 if len(gitCommit
) == 0:
2287 print "could not find git commit for changelist %d" % changelist
2289 gitCommit
= gitCommit
.strip()
2291 # Convert from p4 time format
2293 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2295 print "Could not convert label time %s" % labelDetail
['Update']
2298 when
= int(time
.mktime(tmwhen
))
2299 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2301 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2304 print "Label %s has no changelists - possibly deleted?" % name
2307 # We can't import this label; don't try again as it will get very
2308 # expensive repeatedly fetching all the files for labels that will
2309 # never be imported. If the label is moved in the future, the
2310 # ignore will need to be removed manually.
2311 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2313 def guessProjectName(self
):
2314 for p
in self
.depotPaths
:
2317 p
= p
[p
.strip().rfind("/") + 1:]
2318 if not p
.endswith("/"):
2322 def getBranchMapping(self
):
2323 lostAndFoundBranches
= set()
2325 user
= gitConfig("git-p4.branchUser")
2327 command
= "branches -u %s" % user
2329 command
= "branches"
2331 for info
in p4CmdList(command
):
2332 details
= p4Cmd(["branch", "-o", info
["branch"]])
2334 while details
.has_key("View%s" % viewIdx
):
2335 paths
= details
["View%s" % viewIdx
].split(" ")
2336 viewIdx
= viewIdx
+ 1
2337 # require standard //depot/foo/... //depot/bar/... mapping
2338 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2341 destination
= paths
[1]
2343 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2344 source
= source
[len(self
.depotPaths
[0]):-4]
2345 destination
= destination
[len(self
.depotPaths
[0]):-4]
2347 if destination
in self
.knownBranches
:
2349 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2350 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2353 self
.knownBranches
[destination
] = source
2355 lostAndFoundBranches
.discard(destination
)
2357 if source
not in self
.knownBranches
:
2358 lostAndFoundBranches
.add(source
)
2360 # Perforce does not strictly require branches to be defined, so we also
2361 # check git config for a branch list.
2363 # Example of branch definition in git config file:
2365 # branchList=main:branchA
2366 # branchList=main:branchB
2367 # branchList=branchA:branchC
2368 configBranches
= gitConfigList("git-p4.branchList")
2369 for branch
in configBranches
:
2371 (source
, destination
) = branch
.split(":")
2372 self
.knownBranches
[destination
] = source
2374 lostAndFoundBranches
.discard(destination
)
2376 if source
not in self
.knownBranches
:
2377 lostAndFoundBranches
.add(source
)
2380 for branch
in lostAndFoundBranches
:
2381 self
.knownBranches
[branch
] = branch
2383 def getBranchMappingFromGitBranches(self
):
2384 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2385 for branch
in branches
.keys():
2386 if branch
== "master":
2389 branch
= branch
[len(self
.projectName
):]
2390 self
.knownBranches
[branch
] = branch
2392 def listExistingP4GitBranches(self
):
2393 # branches holds mapping from name to commit
2394 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2395 self
.p4BranchesInGit
= branches
.keys()
2396 for branch
in branches
.keys():
2397 self
.initialParents
[self
.refPrefix
+ branch
] = branches
[branch
]
2399 def updateOptionDict(self
, d
):
2401 if self
.keepRepoPath
:
2402 option_keys
['keepRepoPath'] = 1
2404 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2406 def readOptions(self
, d
):
2407 self
.keepRepoPath
= (d
.has_key('options')
2408 and ('keepRepoPath' in d
['options']))
2410 def gitRefForBranch(self
, branch
):
2411 if branch
== "main":
2412 return self
.refPrefix
+ "master"
2414 if len(branch
) <= 0:
2417 return self
.refPrefix
+ self
.projectName
+ branch
2419 def gitCommitByP4Change(self
, ref
, change
):
2421 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2424 latestCommit
= parseRevision(ref
)
2428 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
2429 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
2434 log
= extractLogMessageFromGitCommit(next
)
2435 settings
= extractSettingsGitLog(log
)
2436 currentChange
= int(settings
['change'])
2438 print "current change %s" % currentChange
2440 if currentChange
== change
:
2442 print "found %s" % next
2445 if currentChange
< change
:
2446 earliestCommit
= "^%s" % next
2448 latestCommit
= "%s" % next
2452 def importNewBranch(self
, branch
, maxChange
):
2453 # make fast-import flush all changes to disk and update the refs using the checkpoint
2454 # command so that we can try to find the branch parent in the git history
2455 self
.gitStream
.write("checkpoint\n\n");
2456 self
.gitStream
.flush();
2457 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2458 range = "@1,%s" % maxChange
2459 #print "prefix" + branchPrefix
2460 changes
= p4ChangesForPaths([branchPrefix
], range)
2461 if len(changes
) <= 0:
2463 firstChange
= changes
[0]
2464 #print "first change in branch: %s" % firstChange
2465 sourceBranch
= self
.knownBranches
[branch
]
2466 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
2467 sourceRef
= self
.gitRefForBranch(sourceBranch
)
2468 #print "source " + sourceBranch
2470 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
2471 #print "branch parent: %s" % branchParentChange
2472 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
2473 if len(gitParent
) > 0:
2474 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
2475 #print "parent git commit: %s" % gitParent
2477 self
.importChanges(changes
)
2480 def searchParent(self
, parent
, branch
, target
):
2482 for blob
in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent
]):
2484 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
2487 print "Found parent of %s in commit %s" % (branch
, blob
)
2494 def importChanges(self
, changes
):
2496 for change
in changes
:
2497 description
= p4Cmd(["describe", str(change
)])
2498 self
.updateOptionDict(description
)
2501 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
2506 if self
.detectBranches
:
2507 branches
= self
.splitFilesIntoBranches(description
)
2508 for branch
in branches
.keys():
2510 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2514 filesForCommit
= branches
[branch
]
2517 print "branch is %s" % branch
2519 self
.updatedBranches
.add(branch
)
2521 if branch
not in self
.createdBranches
:
2522 self
.createdBranches
.add(branch
)
2523 parent
= self
.knownBranches
[branch
]
2524 if parent
== branch
:
2527 fullBranch
= self
.projectName
+ branch
2528 if fullBranch
not in self
.p4BranchesInGit
:
2530 print("\n Importing new branch %s" % fullBranch
);
2531 if self
.importNewBranch(branch
, change
- 1):
2533 self
.p4BranchesInGit
.append(fullBranch
)
2535 print("\n Resuming with change %s" % change
);
2538 print "parent determined through known branches: %s" % parent
2540 branch
= self
.gitRefForBranch(branch
)
2541 parent
= self
.gitRefForBranch(parent
)
2544 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
2546 if len(parent
) == 0 and branch
in self
.initialParents
:
2547 parent
= self
.initialParents
[branch
]
2548 del self
.initialParents
[branch
]
2552 tempBranch
= os
.path
.join(self
.tempBranchLocation
, "%d" % (change
))
2554 print "Creating temporary branch: " + tempBranch
2555 self
.commit(description
, filesForCommit
, tempBranch
, [branchPrefix
])
2556 self
.tempBranches
.append(tempBranch
)
2558 blob
= self
.searchParent(parent
, branch
, tempBranch
)
2560 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], blob
)
2563 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
2564 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], parent
)
2566 files
= self
.extractFilesFromCommit(description
)
2567 self
.commit(description
, files
, self
.branch
, self
.depotPaths
,
2569 self
.initialParent
= ""
2571 print self
.gitError
.read()
2574 def importHeadRevision(self
, revision
):
2575 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
2578 details
["user"] = "git perforce import user"
2579 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
2580 % (' '.join(self
.depotPaths
), revision
))
2581 details
["change"] = revision
2585 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
2587 for info
in p4CmdList(["files"] + fileArgs
):
2589 if 'code' in info
and info
['code'] == 'error':
2590 sys
.stderr
.write("p4 returned an error: %s\n"
2592 if info
['data'].find("must refer to client") >= 0:
2593 sys
.stderr
.write("This particular p4 error is misleading.\n")
2594 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
2595 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
2597 if 'p4ExitCode' in info
:
2598 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
2602 change
= int(info
["change"])
2603 if change
> newestRevision
:
2604 newestRevision
= change
2606 if info
["action"] in self
.delete_actions
:
2607 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2608 #fileCnt = fileCnt + 1
2611 for prop
in ["depotFile", "rev", "action", "type" ]:
2612 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
2614 fileCnt
= fileCnt
+ 1
2616 details
["change"] = newestRevision
2618 # Use time from top-most change so that all git p4 clones of
2619 # the same p4 repo have the same commit SHA1s.
2620 res
= p4CmdList("describe -s %d" % newestRevision
)
2623 if r
.has_key('time'):
2624 newestTime
= int(r
['time'])
2625 if newestTime
is None:
2626 die("\"describe -s\" on newest change %d did not give a time")
2627 details
["time"] = newestTime
2629 self
.updateOptionDict(details
)
2631 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
, self
.depotPaths
)
2633 print "IO error with git fast-import. Is your git version recent enough?"
2634 print self
.gitError
.read()
2637 def run(self
, args
):
2638 self
.depotPaths
= []
2639 self
.changeRange
= ""
2640 self
.initialParent
= ""
2641 self
.previousDepotPaths
= []
2643 # map from branch depot path to parent branch
2644 self
.knownBranches
= {}
2645 self
.initialParents
= {}
2646 self
.hasOrigin
= originP4BranchesExist()
2647 if not self
.syncWithOrigin
:
2648 self
.hasOrigin
= False
2650 if self
.importIntoRemotes
:
2651 self
.refPrefix
= "refs/remotes/p4/"
2653 self
.refPrefix
= "refs/heads/p4/"
2655 if self
.syncWithOrigin
and self
.hasOrigin
:
2657 print "Syncing with origin first by calling git fetch origin"
2658 system("git fetch origin")
2660 if len(self
.branch
) == 0:
2661 self
.branch
= self
.refPrefix
+ "master"
2662 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
2663 system("git update-ref %s refs/heads/p4" % self
.branch
)
2664 system("git branch -D p4");
2665 # create it /after/ importing, when master exists
2666 if not gitBranchExists(self
.refPrefix
+ "HEAD") and self
.importIntoRemotes
and gitBranchExists(self
.branch
):
2667 system("git symbolic-ref %sHEAD %s" % (self
.refPrefix
, self
.branch
))
2669 # accept either the command-line option, or the configuration variable
2670 if self
.useClientSpec
:
2671 # will use this after clone to set the variable
2672 self
.useClientSpec_from_options
= True
2674 if gitConfig("git-p4.useclientspec", "--bool") == "true":
2675 self
.useClientSpec
= True
2676 if self
.useClientSpec
:
2677 self
.clientSpecDirs
= getClientSpec()
2679 # TODO: should always look at previous commits,
2680 # merge with previous imports, if possible.
2683 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
2684 self
.listExistingP4GitBranches()
2686 if len(self
.p4BranchesInGit
) > 1:
2688 print "Importing from/into multiple branches"
2689 self
.detectBranches
= True
2692 print "branches: %s" % self
.p4BranchesInGit
2695 for branch
in self
.p4BranchesInGit
:
2696 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
2698 settings
= extractSettingsGitLog(logMsg
)
2700 self
.readOptions(settings
)
2701 if (settings
.has_key('depot-paths')
2702 and settings
.has_key ('change')):
2703 change
= int(settings
['change']) + 1
2704 p4Change
= max(p4Change
, change
)
2706 depotPaths
= sorted(settings
['depot-paths'])
2707 if self
.previousDepotPaths
== []:
2708 self
.previousDepotPaths
= depotPaths
2711 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
2712 prev_list
= prev
.split("/")
2713 cur_list
= cur
.split("/")
2714 for i
in range(0, min(len(cur_list
), len(prev_list
))):
2715 if cur_list
[i
] <> prev_list
[i
]:
2719 paths
.append ("/".join(cur_list
[:i
+ 1]))
2721 self
.previousDepotPaths
= paths
2724 self
.depotPaths
= sorted(self
.previousDepotPaths
)
2725 self
.changeRange
= "@%s,#head" % p4Change
2726 if not self
.detectBranches
:
2727 self
.initialParent
= parseRevision(self
.branch
)
2728 if not self
.silent
and not self
.detectBranches
:
2729 print "Performing incremental import into %s git branch" % self
.branch
2731 if not self
.branch
.startswith("refs/"):
2732 self
.branch
= "refs/heads/" + self
.branch
2734 if len(args
) == 0 and self
.depotPaths
:
2736 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
2738 if self
.depotPaths
and self
.depotPaths
!= args
:
2739 print ("previous import used depot path %s and now %s was specified. "
2740 "This doesn't work!" % (' '.join (self
.depotPaths
),
2744 self
.depotPaths
= sorted(args
)
2749 # Make sure no revision specifiers are used when --changesfile
2751 bad_changesfile
= False
2752 if len(self
.changesFile
) > 0:
2753 for p
in self
.depotPaths
:
2754 if p
.find("@") >= 0 or p
.find("#") >= 0:
2755 bad_changesfile
= True
2758 die("Option --changesfile is incompatible with revision specifiers")
2761 for p
in self
.depotPaths
:
2762 if p
.find("@") != -1:
2763 atIdx
= p
.index("@")
2764 self
.changeRange
= p
[atIdx
:]
2765 if self
.changeRange
== "@all":
2766 self
.changeRange
= ""
2767 elif ',' not in self
.changeRange
:
2768 revision
= self
.changeRange
2769 self
.changeRange
= ""
2771 elif p
.find("#") != -1:
2772 hashIdx
= p
.index("#")
2773 revision
= p
[hashIdx
:]
2775 elif self
.previousDepotPaths
== []:
2776 # pay attention to changesfile, if given, else import
2777 # the entire p4 tree at the head revision
2778 if len(self
.changesFile
) == 0:
2781 p
= re
.sub ("\.\.\.$", "", p
)
2782 if not p
.endswith("/"):
2787 self
.depotPaths
= newPaths
2789 self
.loadUserMapFromCache()
2791 if self
.detectLabels
:
2794 if self
.detectBranches
:
2795 ## FIXME - what's a P4 projectName ?
2796 self
.projectName
= self
.guessProjectName()
2799 self
.getBranchMappingFromGitBranches()
2801 self
.getBranchMapping()
2803 print "p4-git branches: %s" % self
.p4BranchesInGit
2804 print "initial parents: %s" % self
.initialParents
2805 for b
in self
.p4BranchesInGit
:
2809 b
= b
[len(self
.projectName
):]
2810 self
.createdBranches
.add(b
)
2812 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
2814 importProcess
= subprocess
.Popen(["git", "fast-import"],
2815 stdin
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
,
2816 stderr
=subprocess
.PIPE
);
2817 self
.gitOutput
= importProcess
.stdout
2818 self
.gitStream
= importProcess
.stdin
2819 self
.gitError
= importProcess
.stderr
2822 self
.importHeadRevision(revision
)
2826 if len(self
.changesFile
) > 0:
2827 output
= open(self
.changesFile
).readlines()
2830 changeSet
.add(int(line
))
2832 for change
in changeSet
:
2833 changes
.append(change
)
2837 # catch "git p4 sync" with no new branches, in a repo that
2838 # does not have any existing p4 branches
2839 if len(args
) == 0 and not self
.p4BranchesInGit
:
2840 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
2842 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
2844 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
)
2846 if len(self
.maxChanges
) > 0:
2847 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
2849 if len(changes
) == 0:
2851 print "No changes to import!"
2853 if not self
.silent
and not self
.detectBranches
:
2854 print "Import destination: %s" % self
.branch
2856 self
.updatedBranches
= set()
2858 self
.importChanges(changes
)
2862 if len(self
.updatedBranches
) > 0:
2863 sys
.stdout
.write("Updated branches: ")
2864 for b
in self
.updatedBranches
:
2865 sys
.stdout
.write("%s " % b
)
2866 sys
.stdout
.write("\n")
2868 if gitConfig("git-p4.importLabels", "--bool") == "true":
2869 self
.importLabels
= True
2871 if self
.importLabels
:
2872 p4Labels
= getP4Labels(self
.depotPaths
)
2873 gitTags
= getGitTags()
2875 missingP4Labels
= p4Labels
- gitTags
2876 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
2878 self
.gitStream
.close()
2879 if importProcess
.wait() != 0:
2880 die("fast-import failed: %s" % self
.gitError
.read())
2881 self
.gitOutput
.close()
2882 self
.gitError
.close()
2884 # Cleanup temporary branches created during import
2885 if self
.tempBranches
!= []:
2886 for branch
in self
.tempBranches
:
2887 read_pipe("git update-ref -d %s" % branch
)
2888 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
2892 class P4Rebase(Command
):
2894 Command
.__init
__(self
)
2896 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2898 self
.importLabels
= False
2899 self
.description
= ("Fetches the latest revision from perforce and "
2900 + "rebases the current work (branch) against it")
2902 def run(self
, args
):
2904 sync
.importLabels
= self
.importLabels
2907 return self
.rebase()
2910 if os
.system("git update-index --refresh") != 0:
2911 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2912 if len(read_pipe("git diff-index HEAD --")) > 0:
2913 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2915 [upstream
, settings
] = findUpstreamBranchPoint()
2916 if len(upstream
) == 0:
2917 die("Cannot find upstream branchpoint for rebase")
2919 # the branchpoint may be p4/foo~3, so strip off the parent
2920 upstream
= re
.sub("~[0-9]+$", "", upstream
)
2922 print "Rebasing the current branch onto %s" % upstream
2923 oldHead
= read_pipe("git rev-parse HEAD").strip()
2924 system("git rebase %s" % upstream
)
2925 system("git diff-tree --stat --summary -M %s HEAD" % oldHead
)
2928 class P4Clone(P4Sync
):
2930 P4Sync
.__init
__(self
)
2931 self
.description
= "Creates a new git repository and imports from Perforce into it"
2932 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
2934 optparse
.make_option("--destination", dest
="cloneDestination",
2935 action
='store', default
=None,
2936 help="where to leave result of the clone"),
2937 optparse
.make_option("-/", dest
="cloneExclude",
2938 action
="append", type="string",
2939 help="exclude depot path"),
2940 optparse
.make_option("--bare", dest
="cloneBare",
2941 action
="store_true", default
=False),
2943 self
.cloneDestination
= None
2944 self
.needsGit
= False
2945 self
.cloneBare
= False
2947 # This is required for the "append" cloneExclude action
2948 def ensure_value(self
, attr
, value
):
2949 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2950 setattr(self
, attr
, value
)
2951 return getattr(self
, attr
)
2953 def defaultDestination(self
, args
):
2954 ## TODO: use common prefix of args?
2956 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
2957 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
2958 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
2959 depotDir
= re
.sub(r
"/$", "", depotDir
)
2960 return os
.path
.split(depotDir
)[1]
2962 def run(self
, args
):
2966 if self
.keepRepoPath
and not self
.cloneDestination
:
2967 sys
.stderr
.write("Must specify destination for --keep-path\n")
2972 if not self
.cloneDestination
and len(depotPaths
) > 1:
2973 self
.cloneDestination
= depotPaths
[-1]
2974 depotPaths
= depotPaths
[:-1]
2976 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
2977 for p
in depotPaths
:
2978 if not p
.startswith("//"):
2981 if not self
.cloneDestination
:
2982 self
.cloneDestination
= self
.defaultDestination(args
)
2984 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
2986 if not os
.path
.exists(self
.cloneDestination
):
2987 os
.makedirs(self
.cloneDestination
)
2988 chdir(self
.cloneDestination
)
2990 init_cmd
= [ "git", "init" ]
2992 init_cmd
.append("--bare")
2993 subprocess
.check_call(init_cmd
)
2995 if not P4Sync
.run(self
, depotPaths
):
2997 if self
.branch
!= "master":
2998 if self
.importIntoRemotes
:
2999 masterbranch
= "refs/remotes/p4/master"
3001 masterbranch
= "refs/heads/p4/master"
3002 if gitBranchExists(masterbranch
):
3003 system("git branch master %s" % masterbranch
)
3004 if not self
.cloneBare
:
3005 system("git checkout -f")
3007 print "Could not detect main branch. No checkout/master branch created."
3009 # auto-set this variable if invoked with --use-client-spec
3010 if self
.useClientSpec_from_options
:
3011 system("git config --bool git-p4.useclientspec true")
3015 class P4Branches(Command
):
3017 Command
.__init
__(self
)
3019 self
.description
= ("Shows the git branches that hold imports and their "
3020 + "corresponding perforce depot paths")
3021 self
.verbose
= False
3023 def run(self
, args
):
3024 if originP4BranchesExist():
3025 createOrUpdateBranchesFromOrigin()
3027 cmdline
= "git rev-parse --symbolic "
3028 cmdline
+= " --remotes"
3030 for line
in read_pipe_lines(cmdline
):
3033 if not line
.startswith('p4/') or line
== "p4/HEAD":
3037 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
3038 settings
= extractSettingsGitLog(log
)
3040 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
3043 class HelpFormatter(optparse
.IndentedHelpFormatter
):
3045 optparse
.IndentedHelpFormatter
.__init
__(self
)
3047 def format_description(self
, description
):
3049 return description
+ "\n"
3053 def printUsage(commands
):
3054 print "usage: %s <command> [options]" % sys
.argv
[0]
3056 print "valid commands: %s" % ", ".join(commands
)
3058 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
3063 "submit" : P4Submit
,
3064 "commit" : P4Submit
,
3066 "rebase" : P4Rebase
,
3068 "rollback" : P4RollBack
,
3069 "branches" : P4Branches
3074 if len(sys
.argv
[1:]) == 0:
3075 printUsage(commands
.keys())
3079 cmdName
= sys
.argv
[1]
3081 klass
= commands
[cmdName
]
3084 print "unknown command %s" % cmdName
3086 printUsage(commands
.keys())
3089 options
= cmd
.options
3090 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
3094 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
3096 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
3098 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
3100 description
= cmd
.description
,
3101 formatter
= HelpFormatter())
3103 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
3105 verbose
= cmd
.verbose
3107 if cmd
.gitdir
== None:
3108 cmd
.gitdir
= os
.path
.abspath(".git")
3109 if not isValidGitDir(cmd
.gitdir
):
3110 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
3111 if os
.path
.exists(cmd
.gitdir
):
3112 cdup
= read_pipe("git rev-parse --show-cdup").strip()
3116 if not isValidGitDir(cmd
.gitdir
):
3117 if isValidGitDir(cmd
.gitdir
+ "/.git"):
3118 cmd
.gitdir
+= "/.git"
3120 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
3122 os
.environ
["GIT_DIR"] = cmd
.gitdir
3124 if not cmd
.run(args
):
3129 if __name__
== '__main__':