3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys
.hexversion
< 0x02040000:
12 # The limiter is the subprocess module
13 sys
.stderr
.write("git-p4: requires Python 2.4 or later.\n")
30 from subprocess
import CalledProcessError
32 # from python2.7:subprocess.py
33 # Exception classes used by this module.
34 class CalledProcessError(Exception):
35 """This exception is raised when a process run by check_call() returns
36 a non-zero exit status. The exit status will be stored in the
37 returncode attribute."""
38 def __init__(self
, returncode
, cmd
):
39 self
.returncode
= returncode
42 return "Command '%s' returned non-zero exit status %d" % (self
.cmd
, self
.returncode
)
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize
= 512
52 def p4_build_cmd(cmd
):
53 """Build a suitable p4 command line.
55 This consolidates building and returning a p4 command line into one
56 location. It means that hooking into the environment, or other configuration
57 can be done more easily.
61 user
= gitConfig("git-p4.user")
63 real_cmd
+= ["-u",user
]
65 password
= gitConfig("git-p4.password")
67 real_cmd
+= ["-P", password
]
69 port
= gitConfig("git-p4.port")
71 real_cmd
+= ["-p", port
]
73 host
= gitConfig("git-p4.host")
75 real_cmd
+= ["-H", host
]
77 client
= gitConfig("git-p4.client")
79 real_cmd
+= ["-c", client
]
82 if isinstance(cmd
,basestring
):
83 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
88 def chdir(path
, is_client_path
=False):
89 """Do chdir to the given path, and set the PWD environment
90 variable for use by P4. It does not look at getcwd() output.
91 Since we're not using the shell, it is necessary to set the
92 PWD environment variable explicitly.
94 Normally, expand the path to force it to be absolute. This
95 addresses the use of relative path names inside P4 settings,
96 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
97 as given; it looks for .p4config using PWD.
99 If is_client_path, the path was handed to us directly by p4,
100 and may be a symbolic link. Do not call os.getcwd() in this
101 case, because it will cause p4 to think that PWD is not inside
106 if not is_client_path
:
108 os
.environ
['PWD'] = path
111 """Return free space in bytes on the disk of the given dirname."""
112 if platform
.system() == 'Windows':
113 free_bytes
= ctypes
.c_ulonglong(0)
114 ctypes
.windll
.kernel32
.GetDiskFreeSpaceExW(ctypes
.c_wchar_p(os
.getcwd()), None, None, ctypes
.pointer(free_bytes
))
115 return free_bytes
.value
117 st
= os
.statvfs(os
.getcwd())
118 return st
.f_bavail
* st
.f_frsize
124 sys
.stderr
.write(msg
+ "\n")
127 def write_pipe(c
, stdin
):
129 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
131 expand
= isinstance(c
,basestring
)
132 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
134 val
= pipe
.write(stdin
)
137 die('Command failed: %s' % str(c
))
141 def p4_write_pipe(c
, stdin
):
142 real_cmd
= p4_build_cmd(c
)
143 return write_pipe(real_cmd
, stdin
)
145 def read_pipe(c
, ignore_error
=False):
147 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
149 expand
= isinstance(c
,basestring
)
150 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, shell
=expand
)
151 (out
, err
) = p
.communicate()
152 if p
.returncode
!= 0 and not ignore_error
:
153 die('Command failed: %s\nError: %s' % (str(c
), err
))
156 def p4_read_pipe(c
, ignore_error
=False):
157 real_cmd
= p4_build_cmd(c
)
158 return read_pipe(real_cmd
, ignore_error
)
160 def read_pipe_lines(c
):
162 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
164 expand
= isinstance(c
, basestring
)
165 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
167 val
= pipe
.readlines()
168 if pipe
.close() or p
.wait():
169 die('Command failed: %s' % str(c
))
173 def p4_read_pipe_lines(c
):
174 """Specifically invoke p4 on the command supplied. """
175 real_cmd
= p4_build_cmd(c
)
176 return read_pipe_lines(real_cmd
)
178 def p4_has_command(cmd
):
179 """Ask p4 for help on this command. If it returns an error, the
180 command does not exist in this version of p4."""
181 real_cmd
= p4_build_cmd(["help", cmd
])
182 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
183 stderr
=subprocess
.PIPE
)
185 return p
.returncode
== 0
187 def p4_has_move_command():
188 """See if the move command exists, that it supports -k, and that
189 it has not been administratively disabled. The arguments
190 must be correct, but the filenames do not have to exist. Use
191 ones with wildcards so even if they exist, it will fail."""
193 if not p4_has_command("move"):
195 cmd
= p4_build_cmd(["move", "-k", "@from", "@to"])
196 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
197 (out
, err
) = p
.communicate()
198 # return code will be 1 in either case
199 if err
.find("Invalid option") >= 0:
201 if err
.find("disabled") >= 0:
203 # assume it failed because @... was invalid changelist
206 def system(cmd
, ignore_error
=False):
207 expand
= isinstance(cmd
,basestring
)
209 sys
.stderr
.write("executing %s\n" % str(cmd
))
210 retcode
= subprocess
.call(cmd
, shell
=expand
)
211 if retcode
and not ignore_error
:
212 raise CalledProcessError(retcode
, cmd
)
217 """Specifically invoke p4 as the system command. """
218 real_cmd
= p4_build_cmd(cmd
)
219 expand
= isinstance(real_cmd
, basestring
)
220 retcode
= subprocess
.call(real_cmd
, shell
=expand
)
222 raise CalledProcessError(retcode
, real_cmd
)
224 _p4_version_string
= None
225 def p4_version_string():
226 """Read the version string, showing just the last line, which
227 hopefully is the interesting version bit.
230 Perforce - The Fast Software Configuration Management System.
231 Copyright 1995-2011 Perforce Software. All rights reserved.
232 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
234 global _p4_version_string
235 if not _p4_version_string
:
236 a
= p4_read_pipe_lines(["-V"])
237 _p4_version_string
= a
[-1].rstrip()
238 return _p4_version_string
240 def p4_integrate(src
, dest
):
241 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
243 def p4_sync(f
, *options
):
244 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
247 # forcibly add file names with wildcards
248 if wildcard_present(f
):
249 p4_system(["add", "-f", f
])
251 p4_system(["add", f
])
254 p4_system(["delete", wildcard_encode(f
)])
257 p4_system(["edit", wildcard_encode(f
)])
260 p4_system(["revert", wildcard_encode(f
)])
262 def p4_reopen(type, f
):
263 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
265 def p4_move(src
, dest
):
266 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
268 def p4_last_change():
269 results
= p4CmdList(["changes", "-m", "1"])
270 return int(results
[0]['change'])
272 def p4_describe(change
):
273 """Make sure it returns a valid result by checking for
274 the presence of field "time". Return a dict of the
277 ds
= p4CmdList(["describe", "-s", str(change
)])
279 die("p4 describe -s %d did not return 1 result: %s" % (change
, str(ds
)))
283 if "p4ExitCode" in d
:
284 die("p4 describe -s %d exited with %d: %s" % (change
, d
["p4ExitCode"],
287 if d
["code"] == "error":
288 die("p4 describe -s %d returned error code: %s" % (change
, str(d
)))
291 die("p4 describe -s %d returned no \"time\": %s" % (change
, str(d
)))
296 # Canonicalize the p4 type and return a tuple of the
297 # base type, plus any modifiers. See "p4 help filetypes"
298 # for a list and explanation.
300 def split_p4_type(p4type
):
302 p4_filetypes_historical
= {
303 "ctempobj": "binary+Sw",
309 "tempobj": "binary+FSw",
310 "ubinary": "binary+F",
311 "uresource": "resource+F",
312 "uxbinary": "binary+Fx",
313 "xbinary": "binary+x",
315 "xtempobj": "binary+Swx",
317 "xunicode": "unicode+x",
320 if p4type
in p4_filetypes_historical
:
321 p4type
= p4_filetypes_historical
[p4type
]
323 s
= p4type
.split("+")
331 # return the raw p4 type of a file (text, text+ko, etc)
334 results
= p4CmdList(["fstat", "-T", "headType", wildcard_encode(f
)])
335 return results
[0]['headType']
338 # Given a type base and modifier, return a regexp matching
339 # the keywords that can be expanded in the file
341 def p4_keywords_regexp_for_type(base
, type_mods
):
342 if base
in ("text", "unicode", "binary"):
344 if "ko" in type_mods
:
346 elif "k" in type_mods
:
347 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
351 \$ # Starts with a dollar, followed by...
352 (%s) # one of the keywords, followed by...
353 (:[^$\n]+)? # possibly an old expansion, followed by...
361 # Given a file, return a regexp matching the possible
362 # RCS keywords that will be expanded, or None for files
363 # with kw expansion turned off.
365 def p4_keywords_regexp_for_file(file):
366 if not os
.path
.exists(file):
369 (type_base
, type_mods
) = split_p4_type(p4_type(file))
370 return p4_keywords_regexp_for_type(type_base
, type_mods
)
372 def setP4ExecBit(file, mode
):
373 # Reopens an already open file and changes the execute bit to match
374 # the execute bit setting in the passed in mode.
378 if not isModeExec(mode
):
379 p4Type
= getP4OpenedType(file)
380 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
381 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
382 if p4Type
[-1] == "+":
383 p4Type
= p4Type
[0:-1]
385 p4_reopen(p4Type
, file)
387 def getP4OpenedType(file):
388 # Returns the perforce file type for the given file.
390 result
= p4_read_pipe(["opened", wildcard_encode(file)])
391 match
= re
.match(".*\((.+)\)( \*exclusive\*)?\r?$", result
)
393 return match
.group(1)
395 die("Could not determine file type for %s (result: '%s')" % (file, result
))
397 # Return the set of all p4 labels
398 def getP4Labels(depotPaths
):
400 if isinstance(depotPaths
,basestring
):
401 depotPaths
= [depotPaths
]
403 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
409 # Return the set of all git tags
412 for line
in read_pipe_lines(["git", "tag"]):
417 def diffTreePattern():
418 # This is a simple generator for the diff tree regex pattern. This could be
419 # a class variable if this and parseDiffTreeEntry were a part of a class.
420 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
424 def parseDiffTreeEntry(entry
):
425 """Parses a single diff tree entry into its component elements.
427 See git-diff-tree(1) manpage for details about the format of the diff
428 output. This method returns a dictionary with the following elements:
430 src_mode - The mode of the source file
431 dst_mode - The mode of the destination file
432 src_sha1 - The sha1 for the source file
433 dst_sha1 - The sha1 fr the destination file
434 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
435 status_score - The score for the status (applicable for 'C' and 'R'
436 statuses). This is None if there is no score.
437 src - The path for the source file.
438 dst - The path for the destination file. This is only present for
439 copy or renames. If it is not present, this is None.
441 If the pattern is not matched, None is returned."""
443 match
= diffTreePattern().next().match(entry
)
446 'src_mode': match
.group(1),
447 'dst_mode': match
.group(2),
448 'src_sha1': match
.group(3),
449 'dst_sha1': match
.group(4),
450 'status': match
.group(5),
451 'status_score': match
.group(6),
452 'src': match
.group(7),
453 'dst': match
.group(10)
457 def isModeExec(mode
):
458 # Returns True if the given git mode represents an executable file,
460 return mode
[-3:] == "755"
462 def isModeExecChanged(src_mode
, dst_mode
):
463 return isModeExec(src_mode
) != isModeExec(dst_mode
)
465 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
467 if isinstance(cmd
,basestring
):
474 cmd
= p4_build_cmd(cmd
)
476 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
478 # Use a temporary file to avoid deadlocks without
479 # subprocess.communicate(), which would put another copy
480 # of stdout into memory.
482 if stdin
is not None:
483 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
484 if isinstance(stdin
,basestring
):
485 stdin_file
.write(stdin
)
488 stdin_file
.write(i
+ '\n')
492 p4
= subprocess
.Popen(cmd
,
495 stdout
=subprocess
.PIPE
)
500 entry
= marshal
.load(p4
.stdout
)
510 entry
["p4ExitCode"] = exitCode
516 list = p4CmdList(cmd
)
522 def p4Where(depotPath
):
523 if not depotPath
.endswith("/"):
525 depotPathLong
= depotPath
+ "..."
526 outputList
= p4CmdList(["where", depotPathLong
])
528 for entry
in outputList
:
529 if "depotFile" in entry
:
530 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
531 # The base path always ends with "/...".
532 if entry
["depotFile"].find(depotPath
) == 0 and entry
["depotFile"][-4:] == "/...":
535 elif "data" in entry
:
536 data
= entry
.get("data")
537 space
= data
.find(" ")
538 if data
[:space
] == depotPath
:
543 if output
["code"] == "error":
547 clientPath
= output
.get("path")
548 elif "data" in output
:
549 data
= output
.get("data")
550 lastSpace
= data
.rfind(" ")
551 clientPath
= data
[lastSpace
+ 1:]
553 if clientPath
.endswith("..."):
554 clientPath
= clientPath
[:-3]
557 def currentGitBranch():
558 retcode
= system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error
=True)
563 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
565 def isValidGitDir(path
):
566 if (os
.path
.exists(path
+ "/HEAD")
567 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
571 def parseRevision(ref
):
572 return read_pipe("git rev-parse %s" % ref
).strip()
574 def branchExists(ref
):
575 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
579 def extractLogMessageFromGitCommit(commit
):
582 ## fixme: title is first line of commit, not 1st paragraph.
584 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
593 def extractSettingsGitLog(log
):
595 for line
in log
.split("\n"):
597 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
601 assignments
= m
.group(1).split (':')
602 for a
in assignments
:
604 key
= vals
[0].strip()
605 val
= ('='.join (vals
[1:])).strip()
606 if val
.endswith ('\"') and val
.startswith('"'):
611 paths
= values
.get("depot-paths")
613 paths
= values
.get("depot-path")
615 values
['depot-paths'] = paths
.split(',')
618 def gitBranchExists(branch
):
619 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
620 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
621 return proc
.wait() == 0;
625 def gitConfig(key
, typeSpecifier
=None):
626 if not _gitConfig
.has_key(key
):
627 cmd
= [ "git", "config" ]
629 cmd
+= [ typeSpecifier
]
631 s
= read_pipe(cmd
, ignore_error
=True)
632 _gitConfig
[key
] = s
.strip()
633 return _gitConfig
[key
]
635 def gitConfigBool(key
):
636 """Return a bool, using git config --bool. It is True only if the
637 variable is set to true, and False if set to false or not present
640 if not _gitConfig
.has_key(key
):
641 _gitConfig
[key
] = gitConfig(key
, '--bool') == "true"
642 return _gitConfig
[key
]
644 def gitConfigInt(key
):
645 if not _gitConfig
.has_key(key
):
646 cmd
= [ "git", "config", "--int", key
]
647 s
= read_pipe(cmd
, ignore_error
=True)
650 _gitConfig
[key
] = int(gitConfig(key
, '--int'))
652 _gitConfig
[key
] = None
653 return _gitConfig
[key
]
655 def gitConfigList(key
):
656 if not _gitConfig
.has_key(key
):
657 s
= read_pipe(["git", "config", "--get-all", key
], ignore_error
=True)
658 _gitConfig
[key
] = s
.strip().split(os
.linesep
)
659 if _gitConfig
[key
] == ['']:
661 return _gitConfig
[key
]
663 def p4BranchesInGit(branchesAreInRemotes
=True):
664 """Find all the branches whose names start with "p4/", looking
665 in remotes or heads as specified by the argument. Return
666 a dictionary of { branch: revision } for each one found.
667 The branch names are the short names, without any
672 cmdline
= "git rev-parse --symbolic "
673 if branchesAreInRemotes
:
674 cmdline
+= "--remotes"
676 cmdline
+= "--branches"
678 for line
in read_pipe_lines(cmdline
):
682 if not line
.startswith('p4/'):
684 # special symbolic ref to p4/master
685 if line
== "p4/HEAD":
688 # strip off p4/ prefix
689 branch
= line
[len("p4/"):]
691 branches
[branch
] = parseRevision(line
)
695 def branch_exists(branch
):
696 """Make sure that the given ref name really exists."""
698 cmd
= [ "git", "rev-parse", "--symbolic", "--verify", branch
]
699 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
700 out
, _
= p
.communicate()
703 # expect exactly one line of output: the branch name
704 return out
.rstrip() == branch
706 def findUpstreamBranchPoint(head
= "HEAD"):
707 branches
= p4BranchesInGit()
708 # map from depot-path to branch name
709 branchByDepotPath
= {}
710 for branch
in branches
.keys():
711 tip
= branches
[branch
]
712 log
= extractLogMessageFromGitCommit(tip
)
713 settings
= extractSettingsGitLog(log
)
714 if settings
.has_key("depot-paths"):
715 paths
= ",".join(settings
["depot-paths"])
716 branchByDepotPath
[paths
] = "remotes/p4/" + branch
720 while parent
< 65535:
721 commit
= head
+ "~%s" % parent
722 log
= extractLogMessageFromGitCommit(commit
)
723 settings
= extractSettingsGitLog(log
)
724 if settings
.has_key("depot-paths"):
725 paths
= ",".join(settings
["depot-paths"])
726 if branchByDepotPath
.has_key(paths
):
727 return [branchByDepotPath
[paths
], settings
]
731 return ["", settings
]
733 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
735 print ("Creating/updating branch(es) in %s based on origin branch(es)"
738 originPrefix
= "origin/p4/"
740 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
742 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
745 headName
= line
[len(originPrefix
):]
746 remoteHead
= localRefPrefix
+ headName
749 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
750 if (not original
.has_key('depot-paths')
751 or not original
.has_key('change')):
755 if not gitBranchExists(remoteHead
):
757 print "creating %s" % remoteHead
760 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
761 if settings
.has_key('change') > 0:
762 if settings
['depot-paths'] == original
['depot-paths']:
763 originP4Change
= int(original
['change'])
764 p4Change
= int(settings
['change'])
765 if originP4Change
> p4Change
:
766 print ("%s (%s) is newer than %s (%s). "
767 "Updating p4 branch from origin."
768 % (originHead
, originP4Change
,
769 remoteHead
, p4Change
))
772 print ("Ignoring: %s was imported from %s while "
773 "%s was imported from %s"
774 % (originHead
, ','.join(original
['depot-paths']),
775 remoteHead
, ','.join(settings
['depot-paths'])))
778 system("git update-ref %s %s" % (remoteHead
, originHead
))
780 def originP4BranchesExist():
781 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
784 def p4ParseNumericChangeRange(parts
):
785 changeStart
= int(parts
[0][1:])
786 if parts
[1] == '#head':
787 changeEnd
= p4_last_change()
789 changeEnd
= int(parts
[1])
791 return (changeStart
, changeEnd
)
793 def chooseBlockSize(blockSize
):
797 return defaultBlockSize
799 def p4ChangesForPaths(depotPaths
, changeRange
, requestedBlockSize
):
802 # Parse the change range into start and end. Try to find integer
803 # revision ranges as these can be broken up into blocks to avoid
804 # hitting server-side limits (maxrows, maxscanresults). But if
805 # that doesn't work, fall back to using the raw revision specifier
806 # strings, without using block mode.
808 if changeRange
is None or changeRange
== '':
810 changeEnd
= p4_last_change()
811 block_size
= chooseBlockSize(requestedBlockSize
)
813 parts
= changeRange
.split(',')
814 assert len(parts
) == 2
816 (changeStart
, changeEnd
) = p4ParseNumericChangeRange(parts
)
817 block_size
= chooseBlockSize(requestedBlockSize
)
819 changeStart
= parts
[0][1:]
821 if requestedBlockSize
:
822 die("cannot use --changes-block-size with non-numeric revisions")
827 # Retrieve changes a block at a time, to prevent running
828 # into a MaxResults/MaxScanRows error from the server.
834 end
= min(changeEnd
, changeStart
+ block_size
)
835 revisionRange
= "%d,%d" % (changeStart
, end
)
837 revisionRange
= "%s,%s" % (changeStart
, changeEnd
)
840 cmd
+= ["%s...@%s" % (p
, revisionRange
)]
842 # Insert changes in chronological order
843 for line
in reversed(p4_read_pipe_lines(cmd
)):
844 changes
.append(int(line
.split(" ")[1]))
852 changeStart
= end
+ 1
854 changes
= sorted(changes
)
857 def p4PathStartsWith(path
, prefix
):
858 # This method tries to remedy a potential mixed-case issue:
860 # If UserA adds //depot/DirA/file1
861 # and UserB adds //depot/dira/file2
863 # we may or may not have a problem. If you have core.ignorecase=true,
864 # we treat DirA and dira as the same directory
865 if gitConfigBool("core.ignorecase"):
866 return path
.lower().startswith(prefix
.lower())
867 return path
.startswith(prefix
)
870 """Look at the p4 client spec, create a View() object that contains
871 all the mappings, and return it."""
873 specList
= p4CmdList("client -o")
874 if len(specList
) != 1:
875 die('Output from "client -o" is %d lines, expecting 1' %
878 # dictionary of all client parameters
882 client_name
= entry
["Client"]
884 # just the keys that start with "View"
885 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
888 view
= View(client_name
)
890 # append the lines, in order, to the view
891 for view_num
in range(len(view_keys
)):
892 k
= "View%d" % view_num
893 if k
not in view_keys
:
894 die("Expected view key %s missing" % k
)
895 view
.append(entry
[k
])
900 """Grab the client directory."""
902 output
= p4CmdList("client -o")
904 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
907 if "Root" not in entry
:
908 die('Client has no "Root"')
913 # P4 wildcards are not allowed in filenames. P4 complains
914 # if you simply add them, but you can force it with "-f", in
915 # which case it translates them into %xx encoding internally.
917 def wildcard_decode(path
):
918 # Search for and fix just these four characters. Do % last so
919 # that fixing it does not inadvertently create new %-escapes.
920 # Cannot have * in a filename in windows; untested as to
921 # what p4 would do in such a case.
922 if not platform
.system() == "Windows":
923 path
= path
.replace("%2A", "*")
924 path
= path
.replace("%23", "#") \
925 .replace("%40", "@") \
929 def wildcard_encode(path
):
930 # do % first to avoid double-encoding the %s introduced here
931 path
= path
.replace("%", "%25") \
932 .replace("*", "%2A") \
933 .replace("#", "%23") \
937 def wildcard_present(path
):
938 m
= re
.search("[*#@%]", path
)
941 class LargeFileSystem(object):
942 """Base class for large file system support."""
944 def __init__(self
, writeToGitStream
):
945 self
.largeFiles
= set()
946 self
.writeToGitStream
= writeToGitStream
948 def generatePointer(self
, cloneDestination
, contentFile
):
949 """Return the content of a pointer file that is stored in Git instead of
950 the actual content."""
951 assert False, "Method 'generatePointer' required in " + self
.__class
__.__name
__
953 def pushFile(self
, localLargeFile
):
954 """Push the actual content which is not stored in the Git repository to
956 assert False, "Method 'pushFile' required in " + self
.__class
__.__name
__
958 def hasLargeFileExtension(self
, relPath
):
961 [relPath
.endswith('.' + e
) for e
in gitConfigList('git-p4.largeFileExtensions')],
965 def generateTempFile(self
, contents
):
966 contentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
970 return contentFile
.name
972 def exceedsLargeFileThreshold(self
, relPath
, contents
):
973 if gitConfigInt('git-p4.largeFileThreshold'):
974 contentsSize
= sum(len(d
) for d
in contents
)
975 if contentsSize
> gitConfigInt('git-p4.largeFileThreshold'):
977 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
978 contentsSize
= sum(len(d
) for d
in contents
)
979 if contentsSize
<= gitConfigInt('git-p4.largeFileCompressedThreshold'):
981 contentTempFile
= self
.generateTempFile(contents
)
982 compressedContentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
983 zf
= zipfile
.ZipFile(compressedContentFile
.name
, mode
='w')
984 zf
.write(contentTempFile
, compress_type
=zipfile
.ZIP_DEFLATED
)
986 compressedContentsSize
= zf
.infolist()[0].compress_size
987 os
.remove(contentTempFile
)
988 os
.remove(compressedContentFile
.name
)
989 if compressedContentsSize
> gitConfigInt('git-p4.largeFileCompressedThreshold'):
993 def addLargeFile(self
, relPath
):
994 self
.largeFiles
.add(relPath
)
996 def removeLargeFile(self
, relPath
):
997 self
.largeFiles
.remove(relPath
)
999 def isLargeFile(self
, relPath
):
1000 return relPath
in self
.largeFiles
1002 def processContent(self
, git_mode
, relPath
, contents
):
1003 """Processes the content of git fast import. This method decides if a
1004 file is stored in the large file system and handles all necessary
1006 if self
.exceedsLargeFileThreshold(relPath
, contents
) or self
.hasLargeFileExtension(relPath
):
1007 contentTempFile
= self
.generateTempFile(contents
)
1008 (git_mode
, contents
, localLargeFile
) = self
.generatePointer(contentTempFile
)
1010 # Move temp file to final location in large file system
1011 largeFileDir
= os
.path
.dirname(localLargeFile
)
1012 if not os
.path
.isdir(largeFileDir
):
1013 os
.makedirs(largeFileDir
)
1014 shutil
.move(contentTempFile
, localLargeFile
)
1015 self
.addLargeFile(relPath
)
1016 if gitConfigBool('git-p4.largeFilePush'):
1017 self
.pushFile(localLargeFile
)
1019 sys
.stderr
.write("%s moved to large file system (%s)\n" % (relPath
, localLargeFile
))
1020 return (git_mode
, contents
)
1022 class MockLFS(LargeFileSystem
):
1023 """Mock large file system for testing."""
1025 def generatePointer(self
, contentFile
):
1026 """The pointer content is the original content prefixed with "pointer-".
1027 The local filename of the large file storage is derived from the file content.
1029 with
open(contentFile
, 'r') as f
:
1032 pointerContents
= 'pointer-' + content
1033 localLargeFile
= os
.path
.join(os
.getcwd(), '.git', 'mock-storage', 'local', content
[:-1])
1034 return (gitMode
, pointerContents
, localLargeFile
)
1036 def pushFile(self
, localLargeFile
):
1037 """The remote filename of the large file storage is the same as the local
1038 one but in a different directory.
1040 remotePath
= os
.path
.join(os
.path
.dirname(localLargeFile
), '..', 'remote')
1041 if not os
.path
.exists(remotePath
):
1042 os
.makedirs(remotePath
)
1043 shutil
.copyfile(localLargeFile
, os
.path
.join(remotePath
, os
.path
.basename(localLargeFile
)))
1045 class GitLFS(LargeFileSystem
):
1046 """Git LFS as backend for the git-p4 large file system.
1047 See https://git-lfs.github.com/ for details."""
1049 def __init__(self
, *args
):
1050 LargeFileSystem
.__init
__(self
, *args
)
1051 self
.baseGitAttributes
= []
1053 def generatePointer(self
, contentFile
):
1054 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1055 mode and content which is stored in the Git repository instead of
1056 the actual content. Return also the new location of the actual
1059 pointerProcess
= subprocess
.Popen(
1060 ['git', 'lfs', 'pointer', '--file=' + contentFile
],
1061 stdout
=subprocess
.PIPE
1063 pointerFile
= pointerProcess
.stdout
.read()
1064 if pointerProcess
.wait():
1065 os
.remove(contentFile
)
1066 die('git-lfs pointer command failed. Did you install the extension?')
1067 pointerContents
= [i
+'\n' for i
in pointerFile
.split('\n')[2:][:-1]]
1068 oid
= pointerContents
[1].split(' ')[1].split(':')[1][:-1]
1069 localLargeFile
= os
.path
.join(
1071 '.git', 'lfs', 'objects', oid
[:2], oid
[2:4],
1074 # LFS Spec states that pointer files should not have the executable bit set.
1076 return (gitMode
, pointerContents
, localLargeFile
)
1078 def pushFile(self
, localLargeFile
):
1079 uploadProcess
= subprocess
.Popen(
1080 ['git', 'lfs', 'push', '--object-id', 'origin', os
.path
.basename(localLargeFile
)]
1082 if uploadProcess
.wait():
1083 die('git-lfs push command failed. Did you define a remote?')
1085 def generateGitAttributes(self
):
1087 self
.baseGitAttributes
+
1091 '# Git LFS (see https://git-lfs.github.com/)\n',
1094 ['*.' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1095 for f
in sorted(gitConfigList('git-p4.largeFileExtensions'))
1097 ['/' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1098 for f
in sorted(self
.largeFiles
) if not self
.hasLargeFileExtension(f
)
1102 def addLargeFile(self
, relPath
):
1103 LargeFileSystem
.addLargeFile(self
, relPath
)
1104 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1106 def removeLargeFile(self
, relPath
):
1107 LargeFileSystem
.removeLargeFile(self
, relPath
)
1108 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1110 def processContent(self
, git_mode
, relPath
, contents
):
1111 if relPath
== '.gitattributes':
1112 self
.baseGitAttributes
= contents
1113 return (git_mode
, self
.generateGitAttributes())
1115 return LargeFileSystem
.processContent(self
, git_mode
, relPath
, contents
)
1119 self
.usage
= "usage: %prog [options]"
1120 self
.needsGit
= True
1121 self
.verbose
= False
1125 self
.userMapFromPerforceServer
= False
1126 self
.myP4UserId
= None
1130 return self
.myP4UserId
1132 results
= p4CmdList("user -o")
1134 if r
.has_key('User'):
1135 self
.myP4UserId
= r
['User']
1137 die("Could not find your p4 user id")
1139 def p4UserIsMe(self
, p4User
):
1140 # return True if the given p4 user is actually me
1141 me
= self
.p4UserId()
1142 if not p4User
or p4User
!= me
:
1147 def getUserCacheFilename(self
):
1148 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
1149 return home
+ "/.gitp4-usercache.txt"
1151 def getUserMapFromPerforceServer(self
):
1152 if self
.userMapFromPerforceServer
:
1157 for output
in p4CmdList("users"):
1158 if not output
.has_key("User"):
1160 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
1161 self
.emails
[output
["Email"]] = output
["User"]
1165 for (key
, val
) in self
.users
.items():
1166 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
1168 open(self
.getUserCacheFilename(), "wb").write(s
)
1169 self
.userMapFromPerforceServer
= True
1171 def loadUserMapFromCache(self
):
1173 self
.userMapFromPerforceServer
= False
1175 cache
= open(self
.getUserCacheFilename(), "rb")
1176 lines
= cache
.readlines()
1179 entry
= line
.strip().split("\t")
1180 self
.users
[entry
[0]] = entry
[1]
1182 self
.getUserMapFromPerforceServer()
1184 class P4Debug(Command
):
1186 Command
.__init
__(self
)
1188 self
.description
= "A tool to debug the output of p4 -G."
1189 self
.needsGit
= False
1191 def run(self
, args
):
1193 for output
in p4CmdList(args
):
1194 print 'Element: %d' % j
1199 class P4RollBack(Command
):
1201 Command
.__init
__(self
)
1203 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
1205 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
1206 self
.rollbackLocalBranches
= False
1208 def run(self
, args
):
1211 maxChange
= int(args
[0])
1213 if "p4ExitCode" in p4Cmd("changes -m 1"):
1214 die("Problems executing p4");
1216 if self
.rollbackLocalBranches
:
1217 refPrefix
= "refs/heads/"
1218 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
1220 refPrefix
= "refs/remotes/"
1221 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
1224 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
1226 ref
= refPrefix
+ line
1227 log
= extractLogMessageFromGitCommit(ref
)
1228 settings
= extractSettingsGitLog(log
)
1230 depotPaths
= settings
['depot-paths']
1231 change
= settings
['change']
1235 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
1236 for p
in depotPaths
]))) == 0:
1237 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
1238 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
1241 while change
and int(change
) > maxChange
:
1244 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
1245 system("git update-ref %s \"%s^\"" % (ref
, ref
))
1246 log
= extractLogMessageFromGitCommit(ref
)
1247 settings
= extractSettingsGitLog(log
)
1250 depotPaths
= settings
['depot-paths']
1251 change
= settings
['change']
1254 print "%s rewound to %s" % (ref
, change
)
1258 class P4Submit(Command
, P4UserMap
):
1260 conflict_behavior_choices
= ("ask", "skip", "quit")
1263 Command
.__init
__(self
)
1264 P4UserMap
.__init
__(self
)
1266 optparse
.make_option("--origin", dest
="origin"),
1267 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
1268 # preserve the user, requires relevant p4 permissions
1269 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
1270 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
1271 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
1272 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
1273 optparse
.make_option("--conflict", dest
="conflict_behavior",
1274 choices
=self
.conflict_behavior_choices
),
1275 optparse
.make_option("--branch", dest
="branch"),
1277 self
.description
= "Submit changes from git to the perforce depot."
1278 self
.usage
+= " [name of git branch to submit into perforce depot]"
1280 self
.detectRenames
= False
1281 self
.preserveUser
= gitConfigBool("git-p4.preserveUser")
1282 self
.dry_run
= False
1283 self
.prepare_p4_only
= False
1284 self
.conflict_behavior
= None
1285 self
.isWindows
= (platform
.system() == "Windows")
1286 self
.exportLabels
= False
1287 self
.p4HasMoveCommand
= p4_has_move_command()
1290 if gitConfig('git-p4.largeFileSystem'):
1291 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1294 if len(p4CmdList("opened ...")) > 0:
1295 die("You have files opened with perforce! Close them before starting the sync.")
1297 def separate_jobs_from_description(self
, message
):
1298 """Extract and return a possible Jobs field in the commit
1299 message. It goes into a separate section in the p4 change
1302 A jobs line starts with "Jobs:" and looks like a new field
1303 in a form. Values are white-space separated on the same
1304 line or on following lines that start with a tab.
1306 This does not parse and extract the full git commit message
1307 like a p4 form. It just sees the Jobs: line as a marker
1308 to pass everything from then on directly into the p4 form,
1309 but outside the description section.
1311 Return a tuple (stripped log message, jobs string)."""
1313 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
1315 return (message
, None)
1317 jobtext
= message
[m
.start():]
1318 stripped_message
= message
[:m
.start()].rstrip()
1319 return (stripped_message
, jobtext
)
1321 def prepareLogMessage(self
, template
, message
, jobs
):
1322 """Edits the template returned from "p4 change -o" to insert
1323 the message in the Description field, and the jobs text in
1327 inDescriptionSection
= False
1329 for line
in template
.split("\n"):
1330 if line
.startswith("#"):
1331 result
+= line
+ "\n"
1334 if inDescriptionSection
:
1335 if line
.startswith("Files:") or line
.startswith("Jobs:"):
1336 inDescriptionSection
= False
1337 # insert Jobs section
1339 result
+= jobs
+ "\n"
1343 if line
.startswith("Description:"):
1344 inDescriptionSection
= True
1346 for messageLine
in message
.split("\n"):
1347 line
+= "\t" + messageLine
+ "\n"
1349 result
+= line
+ "\n"
1353 def patchRCSKeywords(self
, file, pattern
):
1354 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1355 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
1357 outFile
= os
.fdopen(handle
, "w+")
1358 inFile
= open(file, "r")
1359 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1360 for line
in inFile
.readlines():
1361 line
= regexp
.sub(r
'$\1$', line
)
1365 # Forcibly overwrite the original file
1367 shutil
.move(outFileName
, file)
1369 # cleanup our temporary file
1370 os
.unlink(outFileName
)
1371 print "Failed to strip RCS keywords in %s" % file
1374 print "Patched up RCS keywords in %s" % file
1376 def p4UserForCommit(self
,id):
1377 # Return the tuple (perforce user,git email) for a given git commit id
1378 self
.getUserMapFromPerforceServer()
1379 gitEmail
= read_pipe(["git", "log", "--max-count=1",
1380 "--format=%ae", id])
1381 gitEmail
= gitEmail
.strip()
1382 if not self
.emails
.has_key(gitEmail
):
1383 return (None,gitEmail
)
1385 return (self
.emails
[gitEmail
],gitEmail
)
1387 def checkValidP4Users(self
,commits
):
1388 # check if any git authors cannot be mapped to p4 users
1390 (user
,email
) = self
.p4UserForCommit(id)
1392 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
1393 if gitConfigBool("git-p4.allowMissingP4Users"):
1396 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
1398 def lastP4Changelist(self
):
1399 # Get back the last changelist number submitted in this client spec. This
1400 # then gets used to patch up the username in the change. If the same
1401 # client spec is being used by multiple processes then this might go
1403 results
= p4CmdList("client -o") # find the current client
1406 if r
.has_key('Client'):
1407 client
= r
['Client']
1410 die("could not get client spec")
1411 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
1413 if r
.has_key('change'):
1415 die("Could not get changelist number for last submit - cannot patch up user details")
1417 def modifyChangelistUser(self
, changelist
, newUser
):
1418 # fixup the user field of a changelist after it has been submitted.
1419 changes
= p4CmdList("change -o %s" % changelist
)
1420 if len(changes
) != 1:
1421 die("Bad output from p4 change modifying %s to user %s" %
1422 (changelist
, newUser
))
1425 if c
['User'] == newUser
: return # nothing to do
1427 input = marshal
.dumps(c
)
1429 result
= p4CmdList("change -f -i", stdin
=input)
1431 if r
.has_key('code'):
1432 if r
['code'] == 'error':
1433 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1434 if r
.has_key('data'):
1435 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1437 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1439 def canChangeChangelists(self
):
1440 # check to see if we have p4 admin or super-user permissions, either of
1441 # which are required to modify changelists.
1442 results
= p4CmdList(["protects", self
.depotPath
])
1444 if r
.has_key('perm'):
1445 if r
['perm'] == 'admin':
1447 if r
['perm'] == 'super':
1451 def prepareSubmitTemplate(self
):
1452 """Run "p4 change -o" to grab a change specification template.
1453 This does not use "p4 -G", as it is nice to keep the submission
1454 template in original order, since a human might edit it.
1456 Remove lines in the Files section that show changes to files
1457 outside the depot path we're committing into."""
1459 [upstream
, settings
] = findUpstreamBranchPoint()
1462 inFilesSection
= False
1463 for line
in p4_read_pipe_lines(['change', '-o']):
1464 if line
.endswith("\r\n"):
1465 line
= line
[:-2] + "\n"
1467 if line
.startswith("\t"):
1468 # path starts and ends with a tab
1470 lastTab
= path
.rfind("\t")
1472 path
= path
[:lastTab
]
1473 if settings
.has_key('depot-paths'):
1474 if not [p
for p
in settings
['depot-paths']
1475 if p4PathStartsWith(path
, p
)]:
1478 if not p4PathStartsWith(path
, self
.depotPath
):
1481 inFilesSection
= False
1483 if line
.startswith("Files:"):
1484 inFilesSection
= True
1490 def edit_template(self
, template_file
):
1491 """Invoke the editor to let the user change the submission
1492 message. Return true if okay to continue with the submit."""
1494 # if configured to skip the editing part, just submit
1495 if gitConfigBool("git-p4.skipSubmitEdit"):
1498 # look at the modification time, to check later if the user saved
1500 mtime
= os
.stat(template_file
).st_mtime
1503 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1504 editor
= os
.environ
.get("P4EDITOR")
1506 editor
= read_pipe("git var GIT_EDITOR").strip()
1507 system(["sh", "-c", ('%s "$@"' % editor
), editor
, template_file
])
1509 # If the file was not saved, prompt to see if this patch should
1510 # be skipped. But skip this verification step if configured so.
1511 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1514 # modification time updated means user saved the file
1515 if os
.stat(template_file
).st_mtime
> mtime
:
1519 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1525 def get_diff_description(self
, editedFiles
, filesToAdd
):
1527 if os
.environ
.has_key("P4DIFF"):
1528 del(os
.environ
["P4DIFF"])
1530 for editedFile
in editedFiles
:
1531 diff
+= p4_read_pipe(['diff', '-du',
1532 wildcard_encode(editedFile
)])
1536 for newFile
in filesToAdd
:
1537 newdiff
+= "==== new file ====\n"
1538 newdiff
+= "--- /dev/null\n"
1539 newdiff
+= "+++ %s\n" % newFile
1540 f
= open(newFile
, "r")
1541 for line
in f
.readlines():
1542 newdiff
+= "+" + line
1545 return (diff
+ newdiff
).replace('\r\n', '\n')
1547 def applyCommit(self
, id):
1548 """Apply one commit, return True if it succeeded."""
1550 print "Applying", read_pipe(["git", "show", "-s",
1551 "--format=format:%h %s", id])
1553 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1555 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1557 filesToDelete
= set()
1559 pureRenameCopy
= set()
1560 filesToChangeExecBit
= {}
1563 diff
= parseDiffTreeEntry(line
)
1564 modifier
= diff
['status']
1568 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1569 filesToChangeExecBit
[path
] = diff
['dst_mode']
1570 editedFiles
.add(path
)
1571 elif modifier
== "A":
1572 filesToAdd
.add(path
)
1573 filesToChangeExecBit
[path
] = diff
['dst_mode']
1574 if path
in filesToDelete
:
1575 filesToDelete
.remove(path
)
1576 elif modifier
== "D":
1577 filesToDelete
.add(path
)
1578 if path
in filesToAdd
:
1579 filesToAdd
.remove(path
)
1580 elif modifier
== "C":
1581 src
, dest
= diff
['src'], diff
['dst']
1582 p4_integrate(src
, dest
)
1583 pureRenameCopy
.add(dest
)
1584 if diff
['src_sha1'] != diff
['dst_sha1']:
1586 pureRenameCopy
.discard(dest
)
1587 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1589 pureRenameCopy
.discard(dest
)
1590 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1592 # turn off read-only attribute
1593 os
.chmod(dest
, stat
.S_IWRITE
)
1595 editedFiles
.add(dest
)
1596 elif modifier
== "R":
1597 src
, dest
= diff
['src'], diff
['dst']
1598 if self
.p4HasMoveCommand
:
1599 p4_edit(src
) # src must be open before move
1600 p4_move(src
, dest
) # opens for (move/delete, move/add)
1602 p4_integrate(src
, dest
)
1603 if diff
['src_sha1'] != diff
['dst_sha1']:
1606 pureRenameCopy
.add(dest
)
1607 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1608 if not self
.p4HasMoveCommand
:
1609 p4_edit(dest
) # with move: already open, writable
1610 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1611 if not self
.p4HasMoveCommand
:
1613 os
.chmod(dest
, stat
.S_IWRITE
)
1615 filesToDelete
.add(src
)
1616 editedFiles
.add(dest
)
1618 die("unknown modifier %s for %s" % (modifier
, path
))
1620 diffcmd
= "git diff-tree --full-index -p \"%s\"" % (id)
1621 patchcmd
= diffcmd
+ " | git apply "
1622 tryPatchCmd
= patchcmd
+ "--check -"
1623 applyPatchCmd
= patchcmd
+ "--check --apply -"
1624 patch_succeeded
= True
1626 if os
.system(tryPatchCmd
) != 0:
1627 fixed_rcs_keywords
= False
1628 patch_succeeded
= False
1629 print "Unfortunately applying the change failed!"
1631 # Patch failed, maybe it's just RCS keyword woes. Look through
1632 # the patch to see if that's possible.
1633 if gitConfigBool("git-p4.attemptRCSCleanup"):
1637 for file in editedFiles | filesToDelete
:
1638 # did this file's delta contain RCS keywords?
1639 pattern
= p4_keywords_regexp_for_file(file)
1642 # this file is a possibility...look for RCS keywords.
1643 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1644 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1645 if regexp
.search(line
):
1647 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1648 kwfiles
[file] = pattern
1651 for file in kwfiles
:
1653 print "zapping %s with %s" % (line
,pattern
)
1654 # File is being deleted, so not open in p4. Must
1655 # disable the read-only bit on windows.
1656 if self
.isWindows
and file not in editedFiles
:
1657 os
.chmod(file, stat
.S_IWRITE
)
1658 self
.patchRCSKeywords(file, kwfiles
[file])
1659 fixed_rcs_keywords
= True
1661 if fixed_rcs_keywords
:
1662 print "Retrying the patch with RCS keywords cleaned up"
1663 if os
.system(tryPatchCmd
) == 0:
1664 patch_succeeded
= True
1666 if not patch_succeeded
:
1667 for f
in editedFiles
:
1672 # Apply the patch for real, and do add/delete/+x handling.
1674 system(applyPatchCmd
)
1676 for f
in filesToAdd
:
1678 for f
in filesToDelete
:
1682 # Set/clear executable bits
1683 for f
in filesToChangeExecBit
.keys():
1684 mode
= filesToChangeExecBit
[f
]
1685 setP4ExecBit(f
, mode
)
1688 # Build p4 change description, starting with the contents
1689 # of the git commit message.
1691 logMessage
= extractLogMessageFromGitCommit(id)
1692 logMessage
= logMessage
.strip()
1693 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1695 template
= self
.prepareSubmitTemplate()
1696 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1698 if self
.preserveUser
:
1699 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
1701 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1702 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1703 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1704 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1706 separatorLine
= "######## everything below this line is just the diff #######\n"
1707 if not self
.prepare_p4_only
:
1708 submitTemplate
+= separatorLine
1709 submitTemplate
+= self
.get_diff_description(editedFiles
, filesToAdd
)
1711 (handle
, fileName
) = tempfile
.mkstemp()
1712 tmpFile
= os
.fdopen(handle
, "w+b")
1714 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1715 tmpFile
.write(submitTemplate
)
1718 if self
.prepare_p4_only
:
1720 # Leave the p4 tree prepared, and the submit template around
1721 # and let the user decide what to do next
1724 print "P4 workspace prepared for submission."
1725 print "To submit or revert, go to client workspace"
1726 print " " + self
.clientPath
1728 print "To submit, use \"p4 submit\" to write a new description,"
1729 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1730 " \"git p4\"." % fileName
1731 print "You can delete the file \"%s\" when finished." % fileName
1733 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
1734 print "To preserve change ownership by user %s, you must\n" \
1735 "do \"p4 change -f <change>\" after submitting and\n" \
1736 "edit the User field."
1738 print "After submitting, renamed files must be re-synced."
1739 print "Invoke \"p4 sync -f\" on each of these files:"
1740 for f
in pureRenameCopy
:
1744 print "To revert the changes, use \"p4 revert ...\", and delete"
1745 print "the submit template file \"%s\"" % fileName
1747 print "Since the commit adds new files, they must be deleted:"
1748 for f
in filesToAdd
:
1754 # Let the user edit the change description, then submit it.
1759 if self
.edit_template(fileName
):
1760 # read the edited message and submit
1761 tmpFile
= open(fileName
, "rb")
1762 message
= tmpFile
.read()
1765 message
= message
.replace("\r\n", "\n")
1766 submitTemplate
= message
[:message
.index(separatorLine
)]
1767 p4_write_pipe(['submit', '-i'], submitTemplate
)
1769 if self
.preserveUser
:
1771 # Get last changelist number. Cannot easily get it from
1772 # the submit command output as the output is
1774 changelist
= self
.lastP4Changelist()
1775 self
.modifyChangelistUser(changelist
, p4User
)
1777 # The rename/copy happened by applying a patch that created a
1778 # new file. This leaves it writable, which confuses p4.
1779 for f
in pureRenameCopy
:
1786 print "Submission cancelled, undoing p4 changes."
1787 for f
in editedFiles
:
1789 for f
in filesToAdd
:
1792 for f
in filesToDelete
:
1798 # Export git tags as p4 labels. Create a p4 label and then tag
1800 def exportGitTags(self
, gitTags
):
1801 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1802 if len(validLabelRegexp
) == 0:
1803 validLabelRegexp
= defaultLabelRegexp
1804 m
= re
.compile(validLabelRegexp
)
1806 for name
in gitTags
:
1808 if not m
.match(name
):
1810 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1813 # Get the p4 commit this corresponds to
1814 logMessage
= extractLogMessageFromGitCommit(name
)
1815 values
= extractSettingsGitLog(logMessage
)
1817 if not values
.has_key('change'):
1818 # a tag pointing to something not sent to p4; ignore
1820 print "git tag %s does not give a p4 commit" % name
1823 changelist
= values
['change']
1825 # Get the tag details.
1829 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1832 if re
.match(r
'tag\s+', l
):
1834 elif re
.match(r
'\s*$', l
):
1841 body
= ["lightweight tag imported by git p4\n"]
1843 # Create the label - use the same view as the client spec we are using
1844 clientSpec
= getClientSpec()
1846 labelTemplate
= "Label: %s\n" % name
1847 labelTemplate
+= "Description:\n"
1849 labelTemplate
+= "\t" + b
+ "\n"
1850 labelTemplate
+= "View:\n"
1851 for depot_side
in clientSpec
.mappings
:
1852 labelTemplate
+= "\t%s\n" % depot_side
1855 print "Would create p4 label %s for tag" % name
1856 elif self
.prepare_p4_only
:
1857 print "Not creating p4 label %s for tag due to option" \
1858 " --prepare-p4-only" % name
1860 p4_write_pipe(["label", "-i"], labelTemplate
)
1863 p4_system(["tag", "-l", name
] +
1864 ["%s@%s" % (depot_side
, changelist
) for depot_side
in clientSpec
.mappings
])
1867 print "created p4 label for tag %s" % name
1869 def run(self
, args
):
1871 self
.master
= currentGitBranch()
1872 elif len(args
) == 1:
1873 self
.master
= args
[0]
1874 if not branchExists(self
.master
):
1875 die("Branch %s does not exist" % self
.master
)
1880 allowSubmit
= gitConfig("git-p4.allowSubmit")
1881 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1882 die("%s is not in git-p4.allowSubmit" % self
.master
)
1884 [upstream
, settings
] = findUpstreamBranchPoint()
1885 self
.depotPath
= settings
['depot-paths'][0]
1886 if len(self
.origin
) == 0:
1887 self
.origin
= upstream
1889 if self
.preserveUser
:
1890 if not self
.canChangeChangelists():
1891 die("Cannot preserve user names without p4 super-user or admin permissions")
1893 # if not set from the command line, try the config file
1894 if self
.conflict_behavior
is None:
1895 val
= gitConfig("git-p4.conflict")
1897 if val
not in self
.conflict_behavior_choices
:
1898 die("Invalid value '%s' for config git-p4.conflict" % val
)
1901 self
.conflict_behavior
= val
1904 print "Origin branch is " + self
.origin
1906 if len(self
.depotPath
) == 0:
1907 print "Internal error: cannot locate perforce depot path from existing branches"
1910 self
.useClientSpec
= False
1911 if gitConfigBool("git-p4.useclientspec"):
1912 self
.useClientSpec
= True
1913 if self
.useClientSpec
:
1914 self
.clientSpecDirs
= getClientSpec()
1916 # Check for the existance of P4 branches
1917 branchesDetected
= (len(p4BranchesInGit().keys()) > 1)
1919 if self
.useClientSpec
and not branchesDetected
:
1920 # all files are relative to the client spec
1921 self
.clientPath
= getClientRoot()
1923 self
.clientPath
= p4Where(self
.depotPath
)
1925 if self
.clientPath
== "":
1926 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1928 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1929 self
.oldWorkingDirectory
= os
.getcwd()
1931 # ensure the clientPath exists
1932 new_client_dir
= False
1933 if not os
.path
.exists(self
.clientPath
):
1934 new_client_dir
= True
1935 os
.makedirs(self
.clientPath
)
1937 chdir(self
.clientPath
, is_client_path
=True)
1939 print "Would synchronize p4 checkout in %s" % self
.clientPath
1941 print "Synchronizing p4 checkout..."
1943 # old one was destroyed, and maybe nobody told p4
1944 p4_sync("...", "-f")
1951 commitish
= self
.master
1955 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self
.origin
, commitish
)]):
1956 commits
.append(line
.strip())
1959 if self
.preserveUser
or gitConfigBool("git-p4.skipUserNameCheck"):
1960 self
.checkAuthorship
= False
1962 self
.checkAuthorship
= True
1964 if self
.preserveUser
:
1965 self
.checkValidP4Users(commits
)
1968 # Build up a set of options to be passed to diff when
1969 # submitting each commit to p4.
1971 if self
.detectRenames
:
1972 # command-line -M arg
1973 self
.diffOpts
= "-M"
1975 # If not explicitly set check the config variable
1976 detectRenames
= gitConfig("git-p4.detectRenames")
1978 if detectRenames
.lower() == "false" or detectRenames
== "":
1980 elif detectRenames
.lower() == "true":
1981 self
.diffOpts
= "-M"
1983 self
.diffOpts
= "-M%s" % detectRenames
1985 # no command-line arg for -C or --find-copies-harder, just
1987 detectCopies
= gitConfig("git-p4.detectCopies")
1988 if detectCopies
.lower() == "false" or detectCopies
== "":
1990 elif detectCopies
.lower() == "true":
1991 self
.diffOpts
+= " -C"
1993 self
.diffOpts
+= " -C%s" % detectCopies
1995 if gitConfigBool("git-p4.detectCopiesHarder"):
1996 self
.diffOpts
+= " --find-copies-harder"
1999 # Apply the commits, one at a time. On failure, ask if should
2000 # continue to try the rest of the patches, or quit.
2005 last
= len(commits
) - 1
2006 for i
, commit
in enumerate(commits
):
2008 print " ", read_pipe(["git", "show", "-s",
2009 "--format=format:%h %s", commit
])
2012 ok
= self
.applyCommit(commit
)
2014 applied
.append(commit
)
2016 if self
.prepare_p4_only
and i
< last
:
2017 print "Processing only the first commit due to option" \
2018 " --prepare-p4-only"
2023 # prompt for what to do, or use the option/variable
2024 if self
.conflict_behavior
== "ask":
2025 print "What do you want to do?"
2026 response
= raw_input("[s]kip this commit but apply"
2027 " the rest, or [q]uit? ")
2030 elif self
.conflict_behavior
== "skip":
2032 elif self
.conflict_behavior
== "quit":
2035 die("Unknown conflict_behavior '%s'" %
2036 self
.conflict_behavior
)
2038 if response
[0] == "s":
2039 print "Skipping this commit, but applying the rest"
2041 if response
[0] == "q":
2048 chdir(self
.oldWorkingDirectory
)
2052 elif self
.prepare_p4_only
:
2054 elif len(commits
) == len(applied
):
2055 print "All commits applied!"
2059 sync
.branch
= self
.branch
2066 if len(applied
) == 0:
2067 print "No commits applied."
2069 print "Applied only the commits marked with '*':"
2075 print star
, read_pipe(["git", "show", "-s",
2076 "--format=format:%h %s", c
])
2077 print "You will have to do 'git p4 sync' and rebase."
2079 if gitConfigBool("git-p4.exportLabels"):
2080 self
.exportLabels
= True
2082 if self
.exportLabels
:
2083 p4Labels
= getP4Labels(self
.depotPath
)
2084 gitTags
= getGitTags()
2086 missingGitTags
= gitTags
- p4Labels
2087 self
.exportGitTags(missingGitTags
)
2089 # exit with error unless everything applied perfectly
2090 if len(commits
) != len(applied
):
2096 """Represent a p4 view ("p4 help views"), and map files in a
2097 repo according to the view."""
2099 def __init__(self
, client_name
):
2101 self
.client_prefix
= "//%s/" % client_name
2102 # cache results of "p4 where" to lookup client file locations
2103 self
.client_spec_path_cache
= {}
2105 def append(self
, view_line
):
2106 """Parse a view line, splitting it into depot and client
2107 sides. Append to self.mappings, preserving order. This
2108 is only needed for tag creation."""
2110 # Split the view line into exactly two words. P4 enforces
2111 # structure on these lines that simplifies this quite a bit.
2113 # Either or both words may be double-quoted.
2114 # Single quotes do not matter.
2115 # Double-quote marks cannot occur inside the words.
2116 # A + or - prefix is also inside the quotes.
2117 # There are no quotes unless they contain a space.
2118 # The line is already white-space stripped.
2119 # The two words are separated by a single space.
2121 if view_line
[0] == '"':
2122 # First word is double quoted. Find its end.
2123 close_quote_index
= view_line
.find('"', 1)
2124 if close_quote_index
<= 0:
2125 die("No first-word closing quote found: %s" % view_line
)
2126 depot_side
= view_line
[1:close_quote_index
]
2127 # skip closing quote and space
2128 rhs_index
= close_quote_index
+ 1 + 1
2130 space_index
= view_line
.find(" ")
2131 if space_index
<= 0:
2132 die("No word-splitting space found: %s" % view_line
)
2133 depot_side
= view_line
[0:space_index
]
2134 rhs_index
= space_index
+ 1
2136 # prefix + means overlay on previous mapping
2137 if depot_side
.startswith("+"):
2138 depot_side
= depot_side
[1:]
2140 # prefix - means exclude this path, leave out of mappings
2142 if depot_side
.startswith("-"):
2144 depot_side
= depot_side
[1:]
2147 self
.mappings
.append(depot_side
)
2149 def convert_client_path(self
, clientFile
):
2150 # chop off //client/ part to make it relative
2151 if not clientFile
.startswith(self
.client_prefix
):
2152 die("No prefix '%s' on clientFile '%s'" %
2153 (self
.client_prefix
, clientFile
))
2154 return clientFile
[len(self
.client_prefix
):]
2156 def update_client_spec_path_cache(self
, files
):
2157 """ Caching file paths by "p4 where" batch query """
2159 # List depot file paths exclude that already cached
2160 fileArgs
= [f
['path'] for f
in files
if f
['path'] not in self
.client_spec_path_cache
]
2162 if len(fileArgs
) == 0:
2163 return # All files in cache
2165 where_result
= p4CmdList(["-x", "-", "where"], stdin
=fileArgs
)
2166 for res
in where_result
:
2167 if "code" in res
and res
["code"] == "error":
2168 # assume error is "... file(s) not in client view"
2170 if "clientFile" not in res
:
2171 die("No clientFile in 'p4 where' output")
2173 # it will list all of them, but only one not unmap-ped
2175 if gitConfigBool("core.ignorecase"):
2176 res
['depotFile'] = res
['depotFile'].lower()
2177 self
.client_spec_path_cache
[res
['depotFile']] = self
.convert_client_path(res
["clientFile"])
2179 # not found files or unmap files set to ""
2180 for depotFile
in fileArgs
:
2181 if gitConfigBool("core.ignorecase"):
2182 depotFile
= depotFile
.lower()
2183 if depotFile
not in self
.client_spec_path_cache
:
2184 self
.client_spec_path_cache
[depotFile
] = ""
2186 def map_in_client(self
, depot_path
):
2187 """Return the relative location in the client where this
2188 depot file should live. Returns "" if the file should
2189 not be mapped in the client."""
2191 if gitConfigBool("core.ignorecase"):
2192 depot_path
= depot_path
.lower()
2194 if depot_path
in self
.client_spec_path_cache
:
2195 return self
.client_spec_path_cache
[depot_path
]
2197 die( "Error: %s is not found in client spec path" % depot_path
)
2200 class P4Sync(Command
, P4UserMap
):
2201 delete_actions
= ( "delete", "move/delete", "purge" )
2204 Command
.__init
__(self
)
2205 P4UserMap
.__init
__(self
)
2207 optparse
.make_option("--branch", dest
="branch"),
2208 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
2209 optparse
.make_option("--changesfile", dest
="changesFile"),
2210 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
2211 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
2212 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2213 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
2214 help="Import into refs/heads/ , not refs/remotes"),
2215 optparse
.make_option("--max-changes", dest
="maxChanges",
2216 help="Maximum number of changes to import"),
2217 optparse
.make_option("--changes-block-size", dest
="changes_block_size", type="int",
2218 help="Internal block size to use when iteratively calling p4 changes"),
2219 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
2220 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2221 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
2222 help="Only sync files that are included in the Perforce Client Spec"),
2223 optparse
.make_option("-/", dest
="cloneExclude",
2224 action
="append", type="string",
2225 help="exclude depot path"),
2227 self
.description
= """Imports from Perforce into a git repository.\n
2229 //depot/my/project/ -- to import the current head
2230 //depot/my/project/@all -- to import everything
2231 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2233 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2235 self
.usage
+= " //depot/path[@revRange]"
2237 self
.createdBranches
= set()
2238 self
.committedChanges
= set()
2240 self
.detectBranches
= False
2241 self
.detectLabels
= False
2242 self
.importLabels
= False
2243 self
.changesFile
= ""
2244 self
.syncWithOrigin
= True
2245 self
.importIntoRemotes
= True
2246 self
.maxChanges
= ""
2247 self
.changes_block_size
= None
2248 self
.keepRepoPath
= False
2249 self
.depotPaths
= None
2250 self
.p4BranchesInGit
= []
2251 self
.cloneExclude
= []
2252 self
.useClientSpec
= False
2253 self
.useClientSpec_from_options
= False
2254 self
.clientSpecDirs
= None
2255 self
.tempBranches
= []
2256 self
.tempBranchLocation
= "git-p4-tmp"
2257 self
.largeFileSystem
= None
2259 if gitConfig('git-p4.largeFileSystem'):
2260 largeFileSystemConstructor
= globals()[gitConfig('git-p4.largeFileSystem')]
2261 self
.largeFileSystem
= largeFileSystemConstructor(
2262 lambda git_mode
, relPath
, contents
: self
.writeToGitStream(git_mode
, relPath
, contents
)
2265 if gitConfig("git-p4.syncFromOrigin") == "false":
2266 self
.syncWithOrigin
= False
2268 # This is required for the "append" cloneExclude action
2269 def ensure_value(self
, attr
, value
):
2270 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2271 setattr(self
, attr
, value
)
2272 return getattr(self
, attr
)
2274 # Force a checkpoint in fast-import and wait for it to finish
2275 def checkpoint(self
):
2276 self
.gitStream
.write("checkpoint\n\n")
2277 self
.gitStream
.write("progress checkpoint\n\n")
2278 out
= self
.gitOutput
.readline()
2280 print "checkpoint finished: " + out
2282 def extractFilesFromCommit(self
, commit
):
2283 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
2284 for path
in self
.cloneExclude
]
2287 while commit
.has_key("depotFile%s" % fnum
):
2288 path
= commit
["depotFile%s" % fnum
]
2290 if [p
for p
in self
.cloneExclude
2291 if p4PathStartsWith(path
, p
)]:
2294 found
= [p
for p
in self
.depotPaths
2295 if p4PathStartsWith(path
, p
)]
2302 file["rev"] = commit
["rev%s" % fnum
]
2303 file["action"] = commit
["action%s" % fnum
]
2304 file["type"] = commit
["type%s" % fnum
]
2309 def stripRepoPath(self
, path
, prefixes
):
2310 """When streaming files, this is called to map a p4 depot path
2311 to where it should go in git. The prefixes are either
2312 self.depotPaths, or self.branchPrefixes in the case of
2313 branch detection."""
2315 if self
.useClientSpec
:
2316 # branch detection moves files up a level (the branch name)
2317 # from what client spec interpretation gives
2318 path
= self
.clientSpecDirs
.map_in_client(path
)
2319 if self
.detectBranches
:
2320 for b
in self
.knownBranches
:
2321 if path
.startswith(b
+ "/"):
2322 path
= path
[len(b
)+1:]
2324 elif self
.keepRepoPath
:
2325 # Preserve everything in relative path name except leading
2326 # //depot/; just look at first prefix as they all should
2327 # be in the same depot.
2328 depot
= re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])
2329 if p4PathStartsWith(path
, depot
):
2330 path
= path
[len(depot
):]
2334 if p4PathStartsWith(path
, p
):
2335 path
= path
[len(p
):]
2338 path
= wildcard_decode(path
)
2341 def splitFilesIntoBranches(self
, commit
):
2342 """Look at each depotFile in the commit to figure out to what
2343 branch it belongs."""
2345 if self
.clientSpecDirs
:
2346 files
= self
.extractFilesFromCommit(commit
)
2347 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2351 while commit
.has_key("depotFile%s" % fnum
):
2352 path
= commit
["depotFile%s" % fnum
]
2353 found
= [p
for p
in self
.depotPaths
2354 if p4PathStartsWith(path
, p
)]
2361 file["rev"] = commit
["rev%s" % fnum
]
2362 file["action"] = commit
["action%s" % fnum
]
2363 file["type"] = commit
["type%s" % fnum
]
2366 # start with the full relative path where this file would
2368 if self
.useClientSpec
:
2369 relPath
= self
.clientSpecDirs
.map_in_client(path
)
2371 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
2373 for branch
in self
.knownBranches
.keys():
2374 # add a trailing slash so that a commit into qt/4.2foo
2375 # doesn't end up in qt/4.2, e.g.
2376 if relPath
.startswith(branch
+ "/"):
2377 if branch
not in branches
:
2378 branches
[branch
] = []
2379 branches
[branch
].append(file)
2384 def writeToGitStream(self
, gitMode
, relPath
, contents
):
2385 self
.gitStream
.write('M %s inline %s\n' % (gitMode
, relPath
))
2386 self
.gitStream
.write('data %d\n' % sum(len(d
) for d
in contents
))
2388 self
.gitStream
.write(d
)
2389 self
.gitStream
.write('\n')
2391 # output one file from the P4 stream
2392 # - helper for streamP4Files
2394 def streamOneP4File(self
, file, contents
):
2395 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
2397 size
= int(self
.stream_file
['fileSize'])
2398 sys
.stdout
.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath
, size
/1024/1024))
2401 (type_base
, type_mods
) = split_p4_type(file["type"])
2404 if "x" in type_mods
:
2406 if type_base
== "symlink":
2408 # p4 print on a symlink sometimes contains "target\n";
2409 # if it does, remove the newline
2410 data
= ''.join(contents
)
2412 # Some version of p4 allowed creating a symlink that pointed
2413 # to nothing. This causes p4 errors when checking out such
2414 # a change, and errors here too. Work around it by ignoring
2415 # the bad symlink; hopefully a future change fixes it.
2416 print "\nIgnoring empty symlink in %s" % file['depotFile']
2418 elif data
[-1] == '\n':
2419 contents
= [data
[:-1]]
2423 if type_base
== "utf16":
2424 # p4 delivers different text in the python output to -G
2425 # than it does when using "print -o", or normal p4 client
2426 # operations. utf16 is converted to ascii or utf8, perhaps.
2427 # But ascii text saved as -t utf16 is completely mangled.
2428 # Invoke print -o to get the real contents.
2430 # On windows, the newlines will always be mangled by print, so put
2431 # them back too. This is not needed to the cygwin windows version,
2432 # just the native "NT" type.
2435 text
= p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2436 except Exception as e
:
2437 if 'Translation of file content failed' in str(e
):
2438 type_base
= 'binary'
2442 if p4_version_string().find('/NT') >= 0:
2443 text
= text
.replace('\r\n', '\n')
2446 if type_base
== "apple":
2447 # Apple filetype files will be streamed as a concatenation of
2448 # its appledouble header and the contents. This is useless
2449 # on both macs and non-macs. If using "print -q -o xx", it
2450 # will create "xx" with the data, and "%xx" with the header.
2451 # This is also not very useful.
2453 # Ideally, someday, this script can learn how to generate
2454 # appledouble files directly and import those to git, but
2455 # non-mac machines can never find a use for apple filetype.
2456 print "\nIgnoring apple filetype file %s" % file['depotFile']
2459 # Note that we do not try to de-mangle keywords on utf16 files,
2460 # even though in theory somebody may want that.
2461 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
2463 regexp
= re
.compile(pattern
, re
.VERBOSE
)
2464 text
= ''.join(contents
)
2465 text
= regexp
.sub(r
'$\1$', text
)
2469 relPath
.decode('ascii')
2472 if gitConfig('git-p4.pathEncoding'):
2473 encoding
= gitConfig('git-p4.pathEncoding')
2474 relPath
= relPath
.decode(encoding
, 'replace').encode('utf8', 'replace')
2476 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding
, relPath
)
2478 if self
.largeFileSystem
:
2479 (git_mode
, contents
) = self
.largeFileSystem
.processContent(git_mode
, relPath
, contents
)
2481 self
.writeToGitStream(git_mode
, relPath
, contents
)
2483 def streamOneP4Deletion(self
, file):
2484 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
2486 sys
.stdout
.write("delete %s\n" % relPath
)
2488 self
.gitStream
.write("D %s\n" % relPath
)
2490 if self
.largeFileSystem
and self
.largeFileSystem
.isLargeFile(relPath
):
2491 self
.largeFileSystem
.removeLargeFile(relPath
)
2493 # handle another chunk of streaming data
2494 def streamP4FilesCb(self
, marshalled
):
2496 # catch p4 errors and complain
2498 if "code" in marshalled
:
2499 if marshalled
["code"] == "error":
2500 if "data" in marshalled
:
2501 err
= marshalled
["data"].rstrip()
2503 if not err
and 'fileSize' in self
.stream_file
:
2504 required_bytes
= int((4 * int(self
.stream_file
["fileSize"])) - calcDiskFree())
2505 if required_bytes
> 0:
2506 err
= 'Not enough space left on %s! Free at least %i MB.' % (
2507 os
.getcwd(), required_bytes
/1024/1024
2512 if self
.stream_have_file_info
:
2513 if "depotFile" in self
.stream_file
:
2514 f
= self
.stream_file
["depotFile"]
2515 # force a failure in fast-import, else an empty
2516 # commit will be made
2517 self
.gitStream
.write("\n")
2518 self
.gitStream
.write("die-now\n")
2519 self
.gitStream
.close()
2520 # ignore errors, but make sure it exits first
2521 self
.importProcess
.wait()
2523 die("Error from p4 print for %s: %s" % (f
, err
))
2525 die("Error from p4 print: %s" % err
)
2527 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
2528 # start of a new file - output the old one first
2529 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2530 self
.stream_file
= {}
2531 self
.stream_contents
= []
2532 self
.stream_have_file_info
= False
2534 # pick up the new file information... for the
2535 # 'data' field we need to append to our array
2536 for k
in marshalled
.keys():
2538 if 'streamContentSize' not in self
.stream_file
:
2539 self
.stream_file
['streamContentSize'] = 0
2540 self
.stream_file
['streamContentSize'] += len(marshalled
['data'])
2541 self
.stream_contents
.append(marshalled
['data'])
2543 self
.stream_file
[k
] = marshalled
[k
]
2546 'streamContentSize' in self
.stream_file
and
2547 'fileSize' in self
.stream_file
and
2548 'depotFile' in self
.stream_file
):
2549 size
= int(self
.stream_file
["fileSize"])
2551 progress
= 100*self
.stream_file
['streamContentSize']/size
2552 sys
.stdout
.write('\r%s %d%% (%i MB)' % (self
.stream_file
['depotFile'], progress
, int(size
/1024/1024)))
2555 self
.stream_have_file_info
= True
2557 # Stream directly from "p4 files" into "git fast-import"
2558 def streamP4Files(self
, files
):
2564 # if using a client spec, only add the files that have
2565 # a path in the client
2566 if self
.clientSpecDirs
:
2567 if self
.clientSpecDirs
.map_in_client(f
['path']) == "":
2570 filesForCommit
.append(f
)
2571 if f
['action'] in self
.delete_actions
:
2572 filesToDelete
.append(f
)
2574 filesToRead
.append(f
)
2577 for f
in filesToDelete
:
2578 self
.streamOneP4Deletion(f
)
2580 if len(filesToRead
) > 0:
2581 self
.stream_file
= {}
2582 self
.stream_contents
= []
2583 self
.stream_have_file_info
= False
2585 # curry self argument
2586 def streamP4FilesCbSelf(entry
):
2587 self
.streamP4FilesCb(entry
)
2589 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
2591 p4CmdList(["-x", "-", "print"],
2593 cb
=streamP4FilesCbSelf
)
2596 if self
.stream_file
.has_key('depotFile'):
2597 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2599 def make_email(self
, userid
):
2600 if userid
in self
.users
:
2601 return self
.users
[userid
]
2603 return "%s <a@b>" % userid
2605 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
2606 """ Stream a p4 tag.
2607 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2611 print "writing tag %s for commit %s" % (labelName
, commit
)
2612 gitStream
.write("tag %s\n" % labelName
)
2613 gitStream
.write("from %s\n" % commit
)
2615 if labelDetails
.has_key('Owner'):
2616 owner
= labelDetails
["Owner"]
2620 # Try to use the owner of the p4 label, or failing that,
2621 # the current p4 user id.
2623 email
= self
.make_email(owner
)
2625 email
= self
.make_email(self
.p4UserId())
2626 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2628 gitStream
.write("tagger %s\n" % tagger
)
2630 print "labelDetails=",labelDetails
2631 if labelDetails
.has_key('Description'):
2632 description
= labelDetails
['Description']
2634 description
= 'Label from git p4'
2636 gitStream
.write("data %d\n" % len(description
))
2637 gitStream
.write(description
)
2638 gitStream
.write("\n")
2640 def commit(self
, details
, files
, branch
, parent
= ""):
2641 epoch
= details
["time"]
2642 author
= details
["user"]
2645 print "commit into %s" % branch
2647 # start with reading files; if that fails, we should not
2651 if [p
for p
in self
.branchPrefixes
if p4PathStartsWith(f
['path'], p
)]:
2652 new_files
.append (f
)
2654 sys
.stderr
.write("Ignoring file outside of prefix: %s\n" % f
['path'])
2656 if self
.clientSpecDirs
:
2657 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2659 self
.gitStream
.write("commit %s\n" % branch
)
2660 self
.gitStream
.write("mark :%s\n" % details
["change"])
2661 self
.committedChanges
.add(int(details
["change"]))
2663 if author
not in self
.users
:
2664 self
.getUserMapFromPerforceServer()
2665 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2667 self
.gitStream
.write("committer %s\n" % committer
)
2669 self
.gitStream
.write("data <<EOT\n")
2670 self
.gitStream
.write(details
["desc"])
2671 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2672 (','.join(self
.branchPrefixes
), details
["change"]))
2673 if len(details
['options']) > 0:
2674 self
.gitStream
.write(": options = %s" % details
['options'])
2675 self
.gitStream
.write("]\nEOT\n\n")
2679 print "parent %s" % parent
2680 self
.gitStream
.write("from %s\n" % parent
)
2682 self
.streamP4Files(new_files
)
2683 self
.gitStream
.write("\n")
2685 change
= int(details
["change"])
2687 if self
.labels
.has_key(change
):
2688 label
= self
.labels
[change
]
2689 labelDetails
= label
[0]
2690 labelRevisions
= label
[1]
2692 print "Change %s is labelled %s" % (change
, labelDetails
)
2694 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2695 for p
in self
.branchPrefixes
])
2697 if len(files
) == len(labelRevisions
):
2701 if info
["action"] in self
.delete_actions
:
2703 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2705 if cleanedFiles
== labelRevisions
:
2706 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2710 print ("Tag %s does not match with change %s: files do not match."
2711 % (labelDetails
["label"], change
))
2715 print ("Tag %s does not match with change %s: file count is different."
2716 % (labelDetails
["label"], change
))
2718 # Build a dictionary of changelists and labels, for "detect-labels" option.
2719 def getLabels(self
):
2722 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2723 if len(l
) > 0 and not self
.silent
:
2724 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2727 label
= output
["label"]
2731 print "Querying files for label %s" % label
2732 for file in p4CmdList(["files"] +
2733 ["%s...@%s" % (p
, label
)
2734 for p
in self
.depotPaths
]):
2735 revisions
[file["depotFile"]] = file["rev"]
2736 change
= int(file["change"])
2737 if change
> newestChange
:
2738 newestChange
= change
2740 self
.labels
[newestChange
] = [output
, revisions
]
2743 print "Label changes: %s" % self
.labels
.keys()
2745 # Import p4 labels as git tags. A direct mapping does not
2746 # exist, so assume that if all the files are at the same revision
2747 # then we can use that, or it's something more complicated we should
2749 def importP4Labels(self
, stream
, p4Labels
):
2751 print "import p4 labels: " + ' '.join(p4Labels
)
2753 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2754 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2755 if len(validLabelRegexp
) == 0:
2756 validLabelRegexp
= defaultLabelRegexp
2757 m
= re
.compile(validLabelRegexp
)
2759 for name
in p4Labels
:
2762 if not m
.match(name
):
2764 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2767 if name
in ignoredP4Labels
:
2770 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2772 # get the most recent changelist for each file in this label
2773 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2774 for p
in self
.depotPaths
])
2776 if change
.has_key('change'):
2777 # find the corresponding git commit; take the oldest commit
2778 changelist
= int(change
['change'])
2779 if changelist
in self
.committedChanges
:
2780 gitCommit
= ":%d" % changelist
# use a fast-import mark
2783 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2784 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
], ignore_error
=True)
2785 if len(gitCommit
) == 0:
2786 print "importing label %s: could not find git commit for changelist %d" % (name
, changelist
)
2789 gitCommit
= gitCommit
.strip()
2792 # Convert from p4 time format
2794 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2796 print "Could not convert label time %s" % labelDetails
['Update']
2799 when
= int(time
.mktime(tmwhen
))
2800 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2802 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2805 print "Label %s has no changelists - possibly deleted?" % name
2808 # We can't import this label; don't try again as it will get very
2809 # expensive repeatedly fetching all the files for labels that will
2810 # never be imported. If the label is moved in the future, the
2811 # ignore will need to be removed manually.
2812 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2814 def guessProjectName(self
):
2815 for p
in self
.depotPaths
:
2818 p
= p
[p
.strip().rfind("/") + 1:]
2819 if not p
.endswith("/"):
2823 def getBranchMapping(self
):
2824 lostAndFoundBranches
= set()
2826 user
= gitConfig("git-p4.branchUser")
2828 command
= "branches -u %s" % user
2830 command
= "branches"
2832 for info
in p4CmdList(command
):
2833 details
= p4Cmd(["branch", "-o", info
["branch"]])
2835 while details
.has_key("View%s" % viewIdx
):
2836 paths
= details
["View%s" % viewIdx
].split(" ")
2837 viewIdx
= viewIdx
+ 1
2838 # require standard //depot/foo/... //depot/bar/... mapping
2839 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2842 destination
= paths
[1]
2844 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2845 source
= source
[len(self
.depotPaths
[0]):-4]
2846 destination
= destination
[len(self
.depotPaths
[0]):-4]
2848 if destination
in self
.knownBranches
:
2850 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2851 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2854 self
.knownBranches
[destination
] = source
2856 lostAndFoundBranches
.discard(destination
)
2858 if source
not in self
.knownBranches
:
2859 lostAndFoundBranches
.add(source
)
2861 # Perforce does not strictly require branches to be defined, so we also
2862 # check git config for a branch list.
2864 # Example of branch definition in git config file:
2866 # branchList=main:branchA
2867 # branchList=main:branchB
2868 # branchList=branchA:branchC
2869 configBranches
= gitConfigList("git-p4.branchList")
2870 for branch
in configBranches
:
2872 (source
, destination
) = branch
.split(":")
2873 self
.knownBranches
[destination
] = source
2875 lostAndFoundBranches
.discard(destination
)
2877 if source
not in self
.knownBranches
:
2878 lostAndFoundBranches
.add(source
)
2881 for branch
in lostAndFoundBranches
:
2882 self
.knownBranches
[branch
] = branch
2884 def getBranchMappingFromGitBranches(self
):
2885 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2886 for branch
in branches
.keys():
2887 if branch
== "master":
2890 branch
= branch
[len(self
.projectName
):]
2891 self
.knownBranches
[branch
] = branch
2893 def updateOptionDict(self
, d
):
2895 if self
.keepRepoPath
:
2896 option_keys
['keepRepoPath'] = 1
2898 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2900 def readOptions(self
, d
):
2901 self
.keepRepoPath
= (d
.has_key('options')
2902 and ('keepRepoPath' in d
['options']))
2904 def gitRefForBranch(self
, branch
):
2905 if branch
== "main":
2906 return self
.refPrefix
+ "master"
2908 if len(branch
) <= 0:
2911 return self
.refPrefix
+ self
.projectName
+ branch
2913 def gitCommitByP4Change(self
, ref
, change
):
2915 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2918 latestCommit
= parseRevision(ref
)
2922 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
2923 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
2928 log
= extractLogMessageFromGitCommit(next
)
2929 settings
= extractSettingsGitLog(log
)
2930 currentChange
= int(settings
['change'])
2932 print "current change %s" % currentChange
2934 if currentChange
== change
:
2936 print "found %s" % next
2939 if currentChange
< change
:
2940 earliestCommit
= "^%s" % next
2942 latestCommit
= "%s" % next
2946 def importNewBranch(self
, branch
, maxChange
):
2947 # make fast-import flush all changes to disk and update the refs using the checkpoint
2948 # command so that we can try to find the branch parent in the git history
2949 self
.gitStream
.write("checkpoint\n\n");
2950 self
.gitStream
.flush();
2951 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2952 range = "@1,%s" % maxChange
2953 #print "prefix" + branchPrefix
2954 changes
= p4ChangesForPaths([branchPrefix
], range, self
.changes_block_size
)
2955 if len(changes
) <= 0:
2957 firstChange
= changes
[0]
2958 #print "first change in branch: %s" % firstChange
2959 sourceBranch
= self
.knownBranches
[branch
]
2960 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
2961 sourceRef
= self
.gitRefForBranch(sourceBranch
)
2962 #print "source " + sourceBranch
2964 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
2965 #print "branch parent: %s" % branchParentChange
2966 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
2967 if len(gitParent
) > 0:
2968 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
2969 #print "parent git commit: %s" % gitParent
2971 self
.importChanges(changes
)
2974 def searchParent(self
, parent
, branch
, target
):
2976 for blob
in read_pipe_lines(["git", "rev-list", "--reverse",
2977 "--no-merges", parent
]):
2979 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
2982 print "Found parent of %s in commit %s" % (branch
, blob
)
2989 def importChanges(self
, changes
):
2991 for change
in changes
:
2992 description
= p4_describe(change
)
2993 self
.updateOptionDict(description
)
2996 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
3001 if self
.detectBranches
:
3002 branches
= self
.splitFilesIntoBranches(description
)
3003 for branch
in branches
.keys():
3005 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3006 self
.branchPrefixes
= [ branchPrefix
]
3010 filesForCommit
= branches
[branch
]
3013 print "branch is %s" % branch
3015 self
.updatedBranches
.add(branch
)
3017 if branch
not in self
.createdBranches
:
3018 self
.createdBranches
.add(branch
)
3019 parent
= self
.knownBranches
[branch
]
3020 if parent
== branch
:
3023 fullBranch
= self
.projectName
+ branch
3024 if fullBranch
not in self
.p4BranchesInGit
:
3026 print("\n Importing new branch %s" % fullBranch
);
3027 if self
.importNewBranch(branch
, change
- 1):
3029 self
.p4BranchesInGit
.append(fullBranch
)
3031 print("\n Resuming with change %s" % change
);
3034 print "parent determined through known branches: %s" % parent
3036 branch
= self
.gitRefForBranch(branch
)
3037 parent
= self
.gitRefForBranch(parent
)
3040 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
3042 if len(parent
) == 0 and branch
in self
.initialParents
:
3043 parent
= self
.initialParents
[branch
]
3044 del self
.initialParents
[branch
]
3048 tempBranch
= "%s/%d" % (self
.tempBranchLocation
, change
)
3050 print "Creating temporary branch: " + tempBranch
3051 self
.commit(description
, filesForCommit
, tempBranch
)
3052 self
.tempBranches
.append(tempBranch
)
3054 blob
= self
.searchParent(parent
, branch
, tempBranch
)
3056 self
.commit(description
, filesForCommit
, branch
, blob
)
3059 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
3060 self
.commit(description
, filesForCommit
, branch
, parent
)
3062 files
= self
.extractFilesFromCommit(description
)
3063 self
.commit(description
, files
, self
.branch
,
3065 # only needed once, to connect to the previous commit
3066 self
.initialParent
= ""
3068 print self
.gitError
.read()
3071 def importHeadRevision(self
, revision
):
3072 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
3075 details
["user"] = "git perforce import user"
3076 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
3077 % (' '.join(self
.depotPaths
), revision
))
3078 details
["change"] = revision
3082 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
3084 for info
in p4CmdList(["files"] + fileArgs
):
3086 if 'code' in info
and info
['code'] == 'error':
3087 sys
.stderr
.write("p4 returned an error: %s\n"
3089 if info
['data'].find("must refer to client") >= 0:
3090 sys
.stderr
.write("This particular p4 error is misleading.\n")
3091 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
3092 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
3094 if 'p4ExitCode' in info
:
3095 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
3099 change
= int(info
["change"])
3100 if change
> newestRevision
:
3101 newestRevision
= change
3103 if info
["action"] in self
.delete_actions
:
3104 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3105 #fileCnt = fileCnt + 1
3108 for prop
in ["depotFile", "rev", "action", "type" ]:
3109 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
3111 fileCnt
= fileCnt
+ 1
3113 details
["change"] = newestRevision
3115 # Use time from top-most change so that all git p4 clones of
3116 # the same p4 repo have the same commit SHA1s.
3117 res
= p4_describe(newestRevision
)
3118 details
["time"] = res
["time"]
3120 self
.updateOptionDict(details
)
3122 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
3124 print "IO error with git fast-import. Is your git version recent enough?"
3125 print self
.gitError
.read()
3128 def run(self
, args
):
3129 self
.depotPaths
= []
3130 self
.changeRange
= ""
3131 self
.previousDepotPaths
= []
3132 self
.hasOrigin
= False
3134 # map from branch depot path to parent branch
3135 self
.knownBranches
= {}
3136 self
.initialParents
= {}
3138 if self
.importIntoRemotes
:
3139 self
.refPrefix
= "refs/remotes/p4/"
3141 self
.refPrefix
= "refs/heads/p4/"
3143 if self
.syncWithOrigin
:
3144 self
.hasOrigin
= originP4BranchesExist()
3147 print 'Syncing with origin first, using "git fetch origin"'
3148 system("git fetch origin")
3150 branch_arg_given
= bool(self
.branch
)
3151 if len(self
.branch
) == 0:
3152 self
.branch
= self
.refPrefix
+ "master"
3153 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
3154 system("git update-ref %s refs/heads/p4" % self
.branch
)
3155 system("git branch -D p4")
3157 # accept either the command-line option, or the configuration variable
3158 if self
.useClientSpec
:
3159 # will use this after clone to set the variable
3160 self
.useClientSpec_from_options
= True
3162 if gitConfigBool("git-p4.useclientspec"):
3163 self
.useClientSpec
= True
3164 if self
.useClientSpec
:
3165 self
.clientSpecDirs
= getClientSpec()
3167 # TODO: should always look at previous commits,
3168 # merge with previous imports, if possible.
3171 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
3173 # branches holds mapping from branch name to sha1
3174 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3176 # restrict to just this one, disabling detect-branches
3177 if branch_arg_given
:
3178 short
= self
.branch
.split("/")[-1]
3179 if short
in branches
:
3180 self
.p4BranchesInGit
= [ short
]
3182 self
.p4BranchesInGit
= branches
.keys()
3184 if len(self
.p4BranchesInGit
) > 1:
3186 print "Importing from/into multiple branches"
3187 self
.detectBranches
= True
3188 for branch
in branches
.keys():
3189 self
.initialParents
[self
.refPrefix
+ branch
] = \
3193 print "branches: %s" % self
.p4BranchesInGit
3196 for branch
in self
.p4BranchesInGit
:
3197 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
3199 settings
= extractSettingsGitLog(logMsg
)
3201 self
.readOptions(settings
)
3202 if (settings
.has_key('depot-paths')
3203 and settings
.has_key ('change')):
3204 change
= int(settings
['change']) + 1
3205 p4Change
= max(p4Change
, change
)
3207 depotPaths
= sorted(settings
['depot-paths'])
3208 if self
.previousDepotPaths
== []:
3209 self
.previousDepotPaths
= depotPaths
3212 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
3213 prev_list
= prev
.split("/")
3214 cur_list
= cur
.split("/")
3215 for i
in range(0, min(len(cur_list
), len(prev_list
))):
3216 if cur_list
[i
] <> prev_list
[i
]:
3220 paths
.append ("/".join(cur_list
[:i
+ 1]))
3222 self
.previousDepotPaths
= paths
3225 self
.depotPaths
= sorted(self
.previousDepotPaths
)
3226 self
.changeRange
= "@%s,#head" % p4Change
3227 if not self
.silent
and not self
.detectBranches
:
3228 print "Performing incremental import into %s git branch" % self
.branch
3230 # accept multiple ref name abbreviations:
3231 # refs/foo/bar/branch -> use it exactly
3232 # p4/branch -> prepend refs/remotes/ or refs/heads/
3233 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3234 if not self
.branch
.startswith("refs/"):
3235 if self
.importIntoRemotes
:
3236 prepend
= "refs/remotes/"
3238 prepend
= "refs/heads/"
3239 if not self
.branch
.startswith("p4/"):
3241 self
.branch
= prepend
+ self
.branch
3243 if len(args
) == 0 and self
.depotPaths
:
3245 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
3247 if self
.depotPaths
and self
.depotPaths
!= args
:
3248 print ("previous import used depot path %s and now %s was specified. "
3249 "This doesn't work!" % (' '.join (self
.depotPaths
),
3253 self
.depotPaths
= sorted(args
)
3258 # Make sure no revision specifiers are used when --changesfile
3260 bad_changesfile
= False
3261 if len(self
.changesFile
) > 0:
3262 for p
in self
.depotPaths
:
3263 if p
.find("@") >= 0 or p
.find("#") >= 0:
3264 bad_changesfile
= True
3267 die("Option --changesfile is incompatible with revision specifiers")
3270 for p
in self
.depotPaths
:
3271 if p
.find("@") != -1:
3272 atIdx
= p
.index("@")
3273 self
.changeRange
= p
[atIdx
:]
3274 if self
.changeRange
== "@all":
3275 self
.changeRange
= ""
3276 elif ',' not in self
.changeRange
:
3277 revision
= self
.changeRange
3278 self
.changeRange
= ""
3280 elif p
.find("#") != -1:
3281 hashIdx
= p
.index("#")
3282 revision
= p
[hashIdx
:]
3284 elif self
.previousDepotPaths
== []:
3285 # pay attention to changesfile, if given, else import
3286 # the entire p4 tree at the head revision
3287 if len(self
.changesFile
) == 0:
3290 p
= re
.sub ("\.\.\.$", "", p
)
3291 if not p
.endswith("/"):
3296 self
.depotPaths
= newPaths
3298 # --detect-branches may change this for each branch
3299 self
.branchPrefixes
= self
.depotPaths
3301 self
.loadUserMapFromCache()
3303 if self
.detectLabels
:
3306 if self
.detectBranches
:
3307 ## FIXME - what's a P4 projectName ?
3308 self
.projectName
= self
.guessProjectName()
3311 self
.getBranchMappingFromGitBranches()
3313 self
.getBranchMapping()
3315 print "p4-git branches: %s" % self
.p4BranchesInGit
3316 print "initial parents: %s" % self
.initialParents
3317 for b
in self
.p4BranchesInGit
:
3321 b
= b
[len(self
.projectName
):]
3322 self
.createdBranches
.add(b
)
3324 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
3326 self
.importProcess
= subprocess
.Popen(["git", "fast-import"],
3327 stdin
=subprocess
.PIPE
,
3328 stdout
=subprocess
.PIPE
,
3329 stderr
=subprocess
.PIPE
);
3330 self
.gitOutput
= self
.importProcess
.stdout
3331 self
.gitStream
= self
.importProcess
.stdin
3332 self
.gitError
= self
.importProcess
.stderr
3335 self
.importHeadRevision(revision
)
3339 if len(self
.changesFile
) > 0:
3340 output
= open(self
.changesFile
).readlines()
3343 changeSet
.add(int(line
))
3345 for change
in changeSet
:
3346 changes
.append(change
)
3350 # catch "git p4 sync" with no new branches, in a repo that
3351 # does not have any existing p4 branches
3353 if not self
.p4BranchesInGit
:
3354 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3356 # The default branch is master, unless --branch is used to
3357 # specify something else. Make sure it exists, or complain
3358 # nicely about how to use --branch.
3359 if not self
.detectBranches
:
3360 if not branch_exists(self
.branch
):
3361 if branch_arg_given
:
3362 die("Error: branch %s does not exist." % self
.branch
)
3364 die("Error: no branch %s; perhaps specify one with --branch." %
3368 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
3370 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
, self
.changes_block_size
)
3372 if len(self
.maxChanges
) > 0:
3373 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
3375 if len(changes
) == 0:
3377 print "No changes to import!"
3379 if not self
.silent
and not self
.detectBranches
:
3380 print "Import destination: %s" % self
.branch
3382 self
.updatedBranches
= set()
3384 if not self
.detectBranches
:
3386 # start a new branch
3387 self
.initialParent
= ""
3389 # build on a previous revision
3390 self
.initialParent
= parseRevision(self
.branch
)
3392 self
.importChanges(changes
)
3396 if len(self
.updatedBranches
) > 0:
3397 sys
.stdout
.write("Updated branches: ")
3398 for b
in self
.updatedBranches
:
3399 sys
.stdout
.write("%s " % b
)
3400 sys
.stdout
.write("\n")
3402 if gitConfigBool("git-p4.importLabels"):
3403 self
.importLabels
= True
3405 if self
.importLabels
:
3406 p4Labels
= getP4Labels(self
.depotPaths
)
3407 gitTags
= getGitTags()
3409 missingP4Labels
= p4Labels
- gitTags
3410 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
3412 self
.gitStream
.close()
3413 if self
.importProcess
.wait() != 0:
3414 die("fast-import failed: %s" % self
.gitError
.read())
3415 self
.gitOutput
.close()
3416 self
.gitError
.close()
3418 # Cleanup temporary branches created during import
3419 if self
.tempBranches
!= []:
3420 for branch
in self
.tempBranches
:
3421 read_pipe("git update-ref -d %s" % branch
)
3422 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
3424 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3425 # a convenient shortcut refname "p4".
3426 if self
.importIntoRemotes
:
3427 head_ref
= self
.refPrefix
+ "HEAD"
3428 if not gitBranchExists(head_ref
) and gitBranchExists(self
.branch
):
3429 system(["git", "symbolic-ref", head_ref
, self
.branch
])
3433 class P4Rebase(Command
):
3435 Command
.__init
__(self
)
3437 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
3439 self
.importLabels
= False
3440 self
.description
= ("Fetches the latest revision from perforce and "
3441 + "rebases the current work (branch) against it")
3443 def run(self
, args
):
3445 sync
.importLabels
= self
.importLabels
3448 return self
.rebase()
3451 if os
.system("git update-index --refresh") != 0:
3452 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3453 if len(read_pipe("git diff-index HEAD --")) > 0:
3454 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3456 [upstream
, settings
] = findUpstreamBranchPoint()
3457 if len(upstream
) == 0:
3458 die("Cannot find upstream branchpoint for rebase")
3460 # the branchpoint may be p4/foo~3, so strip off the parent
3461 upstream
= re
.sub("~[0-9]+$", "", upstream
)
3463 print "Rebasing the current branch onto %s" % upstream
3464 oldHead
= read_pipe("git rev-parse HEAD").strip()
3465 system("git rebase %s" % upstream
)
3466 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead
)
3469 class P4Clone(P4Sync
):
3471 P4Sync
.__init
__(self
)
3472 self
.description
= "Creates a new git repository and imports from Perforce into it"
3473 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
3475 optparse
.make_option("--destination", dest
="cloneDestination",
3476 action
='store', default
=None,
3477 help="where to leave result of the clone"),
3478 optparse
.make_option("--bare", dest
="cloneBare",
3479 action
="store_true", default
=False),
3481 self
.cloneDestination
= None
3482 self
.needsGit
= False
3483 self
.cloneBare
= False
3485 def defaultDestination(self
, args
):
3486 ## TODO: use common prefix of args?
3488 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
3489 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
3490 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
3491 depotDir
= re
.sub(r
"/$", "", depotDir
)
3492 return os
.path
.split(depotDir
)[1]
3494 def run(self
, args
):
3498 if self
.keepRepoPath
and not self
.cloneDestination
:
3499 sys
.stderr
.write("Must specify destination for --keep-path\n")
3504 if not self
.cloneDestination
and len(depotPaths
) > 1:
3505 self
.cloneDestination
= depotPaths
[-1]
3506 depotPaths
= depotPaths
[:-1]
3508 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
3509 for p
in depotPaths
:
3510 if not p
.startswith("//"):
3511 sys
.stderr
.write('Depot paths must start with "//": %s\n' % p
)
3514 if not self
.cloneDestination
:
3515 self
.cloneDestination
= self
.defaultDestination(args
)
3517 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
3519 if not os
.path
.exists(self
.cloneDestination
):
3520 os
.makedirs(self
.cloneDestination
)
3521 chdir(self
.cloneDestination
)
3523 init_cmd
= [ "git", "init" ]
3525 init_cmd
.append("--bare")
3526 retcode
= subprocess
.call(init_cmd
)
3528 raise CalledProcessError(retcode
, init_cmd
)
3530 if not P4Sync
.run(self
, depotPaths
):
3533 # create a master branch and check out a work tree
3534 if gitBranchExists(self
.branch
):
3535 system([ "git", "branch", "master", self
.branch
])
3536 if not self
.cloneBare
:
3537 system([ "git", "checkout", "-f" ])
3539 print 'Not checking out any branch, use ' \
3540 '"git checkout -q -b master <branch>"'
3542 # auto-set this variable if invoked with --use-client-spec
3543 if self
.useClientSpec_from_options
:
3544 system("git config --bool git-p4.useclientspec true")
3548 class P4Branches(Command
):
3550 Command
.__init
__(self
)
3552 self
.description
= ("Shows the git branches that hold imports and their "
3553 + "corresponding perforce depot paths")
3554 self
.verbose
= False
3556 def run(self
, args
):
3557 if originP4BranchesExist():
3558 createOrUpdateBranchesFromOrigin()
3560 cmdline
= "git rev-parse --symbolic "
3561 cmdline
+= " --remotes"
3563 for line
in read_pipe_lines(cmdline
):
3566 if not line
.startswith('p4/') or line
== "p4/HEAD":
3570 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
3571 settings
= extractSettingsGitLog(log
)
3573 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
3576 class HelpFormatter(optparse
.IndentedHelpFormatter
):
3578 optparse
.IndentedHelpFormatter
.__init
__(self
)
3580 def format_description(self
, description
):
3582 return description
+ "\n"
3586 def printUsage(commands
):
3587 print "usage: %s <command> [options]" % sys
.argv
[0]
3589 print "valid commands: %s" % ", ".join(commands
)
3591 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
3596 "submit" : P4Submit
,
3597 "commit" : P4Submit
,
3599 "rebase" : P4Rebase
,
3601 "rollback" : P4RollBack
,
3602 "branches" : P4Branches
3607 if len(sys
.argv
[1:]) == 0:
3608 printUsage(commands
.keys())
3611 cmdName
= sys
.argv
[1]
3613 klass
= commands
[cmdName
]
3616 print "unknown command %s" % cmdName
3618 printUsage(commands
.keys())
3621 options
= cmd
.options
3622 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
3626 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
3628 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
3630 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
3632 description
= cmd
.description
,
3633 formatter
= HelpFormatter())
3635 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
3637 verbose
= cmd
.verbose
3639 if cmd
.gitdir
== None:
3640 cmd
.gitdir
= os
.path
.abspath(".git")
3641 if not isValidGitDir(cmd
.gitdir
):
3642 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
3643 if os
.path
.exists(cmd
.gitdir
):
3644 cdup
= read_pipe("git rev-parse --show-cdup").strip()
3648 if not isValidGitDir(cmd
.gitdir
):
3649 if isValidGitDir(cmd
.gitdir
+ "/.git"):
3650 cmd
.gitdir
+= "/.git"
3652 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
3654 os
.environ
["GIT_DIR"] = cmd
.gitdir
3656 if not cmd
.run(args
):
3661 if __name__
== '__main__':