3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys
.hexversion
< 0x02040000:
12 # The limiter is the subprocess module
13 sys
.stderr
.write("git-p4: requires Python 2.4 or later.\n")
30 # support basestring in python3
34 # 'unicode' is undefined, must be Python 3
38 basestring
= (str,bytes
)
40 # 'unicode' exists, must be Python 2
44 basestring
= basestring
47 from subprocess
import CalledProcessError
49 # from python2.7:subprocess.py
50 # Exception classes used by this module.
51 class CalledProcessError(Exception):
52 """This exception is raised when a process run by check_call() returns
53 a non-zero exit status. The exit status will be stored in the
54 returncode attribute."""
55 def __init__(self
, returncode
, cmd
):
56 self
.returncode
= returncode
59 return "Command '%s' returned non-zero exit status %d" % (self
.cmd
, self
.returncode
)
63 # Only labels/tags matching this will be imported/exported
64 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
66 # The block size is reduced automatically if required
67 defaultBlockSize
= 1<<20
69 p4_access_checked
= False
71 def p4_build_cmd(cmd
):
72 """Build a suitable p4 command line.
74 This consolidates building and returning a p4 command line into one
75 location. It means that hooking into the environment, or other configuration
76 can be done more easily.
80 user
= gitConfig("git-p4.user")
82 real_cmd
+= ["-u",user
]
84 password
= gitConfig("git-p4.password")
86 real_cmd
+= ["-P", password
]
88 port
= gitConfig("git-p4.port")
90 real_cmd
+= ["-p", port
]
92 host
= gitConfig("git-p4.host")
94 real_cmd
+= ["-H", host
]
96 client
= gitConfig("git-p4.client")
98 real_cmd
+= ["-c", client
]
100 retries
= gitConfigInt("git-p4.retries")
102 # Perform 3 retries by default
105 # Provide a way to not pass this option by setting git-p4.retries to 0
106 real_cmd
+= ["-r", str(retries
)]
108 if isinstance(cmd
,basestring
):
109 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
113 # now check that we can actually talk to the server
114 global p4_access_checked
115 if not p4_access_checked
:
116 p4_access_checked
= True # suppress access checks in p4_check_access itself
122 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
123 This won't automatically add ".git" to a directory.
125 d
= read_pipe(["git", "--git-dir", path
, "rev-parse", "--git-dir"], True).strip()
126 if not d
or len(d
) == 0:
131 def chdir(path
, is_client_path
=False):
132 """Do chdir to the given path, and set the PWD environment
133 variable for use by P4. It does not look at getcwd() output.
134 Since we're not using the shell, it is necessary to set the
135 PWD environment variable explicitly.
137 Normally, expand the path to force it to be absolute. This
138 addresses the use of relative path names inside P4 settings,
139 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
140 as given; it looks for .p4config using PWD.
142 If is_client_path, the path was handed to us directly by p4,
143 and may be a symbolic link. Do not call os.getcwd() in this
144 case, because it will cause p4 to think that PWD is not inside
149 if not is_client_path
:
151 os
.environ
['PWD'] = path
154 """Return free space in bytes on the disk of the given dirname."""
155 if platform
.system() == 'Windows':
156 free_bytes
= ctypes
.c_ulonglong(0)
157 ctypes
.windll
.kernel32
.GetDiskFreeSpaceExW(ctypes
.c_wchar_p(os
.getcwd()), None, None, ctypes
.pointer(free_bytes
))
158 return free_bytes
.value
160 st
= os
.statvfs(os
.getcwd())
161 return st
.f_bavail
* st
.f_frsize
167 sys
.stderr
.write(msg
+ "\n")
170 def write_pipe(c
, stdin
):
172 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
174 expand
= isinstance(c
,basestring
)
175 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
177 val
= pipe
.write(stdin
)
180 die('Command failed: %s' % str(c
))
184 def p4_write_pipe(c
, stdin
):
185 real_cmd
= p4_build_cmd(c
)
186 return write_pipe(real_cmd
, stdin
)
188 def read_pipe_full(c
):
189 """ Read output from command. Returns a tuple
190 of the return status, stdout text and stderr
194 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
196 expand
= isinstance(c
,basestring
)
197 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, shell
=expand
)
198 (out
, err
) = p
.communicate()
199 return (p
.returncode
, out
, err
)
201 def read_pipe(c
, ignore_error
=False):
202 """ Read output from command. Returns the output text on
203 success. On failure, terminates execution, unless
204 ignore_error is True, when it returns an empty string.
206 (retcode
, out
, err
) = read_pipe_full(c
)
211 die('Command failed: %s\nError: %s' % (str(c
), err
))
214 def read_pipe_text(c
):
215 """ Read output from a command with trailing whitespace stripped.
216 On error, returns None.
218 (retcode
, out
, err
) = read_pipe_full(c
)
224 def p4_read_pipe(c
, ignore_error
=False):
225 real_cmd
= p4_build_cmd(c
)
226 return read_pipe(real_cmd
, ignore_error
)
228 def read_pipe_lines(c
):
230 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
232 expand
= isinstance(c
, basestring
)
233 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
235 val
= pipe
.readlines()
236 if pipe
.close() or p
.wait():
237 die('Command failed: %s' % str(c
))
241 def p4_read_pipe_lines(c
):
242 """Specifically invoke p4 on the command supplied. """
243 real_cmd
= p4_build_cmd(c
)
244 return read_pipe_lines(real_cmd
)
246 def p4_has_command(cmd
):
247 """Ask p4 for help on this command. If it returns an error, the
248 command does not exist in this version of p4."""
249 real_cmd
= p4_build_cmd(["help", cmd
])
250 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
251 stderr
=subprocess
.PIPE
)
253 return p
.returncode
== 0
255 def p4_has_move_command():
256 """See if the move command exists, that it supports -k, and that
257 it has not been administratively disabled. The arguments
258 must be correct, but the filenames do not have to exist. Use
259 ones with wildcards so even if they exist, it will fail."""
261 if not p4_has_command("move"):
263 cmd
= p4_build_cmd(["move", "-k", "@from", "@to"])
264 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
265 (out
, err
) = p
.communicate()
266 # return code will be 1 in either case
267 if err
.find("Invalid option") >= 0:
269 if err
.find("disabled") >= 0:
271 # assume it failed because @... was invalid changelist
274 def system(cmd
, ignore_error
=False):
275 expand
= isinstance(cmd
,basestring
)
277 sys
.stderr
.write("executing %s\n" % str(cmd
))
278 retcode
= subprocess
.call(cmd
, shell
=expand
)
279 if retcode
and not ignore_error
:
280 raise CalledProcessError(retcode
, cmd
)
285 """Specifically invoke p4 as the system command. """
286 real_cmd
= p4_build_cmd(cmd
)
287 expand
= isinstance(real_cmd
, basestring
)
288 retcode
= subprocess
.call(real_cmd
, shell
=expand
)
290 raise CalledProcessError(retcode
, real_cmd
)
292 def die_bad_access(s
):
293 die("failure accessing depot: {0}".format(s
.rstrip()))
295 def p4_check_access(min_expiration
=1):
296 """ Check if we can access Perforce - account still logged in
298 results
= p4CmdList(["login", "-s"])
300 if len(results
) == 0:
301 # should never get here: always get either some results, or a p4ExitCode
302 assert("could not parse response from perforce")
306 if 'p4ExitCode' in result
:
307 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
308 die_bad_access("could not run p4")
310 code
= result
.get("code")
312 # we get here if we couldn't connect and there was nothing to unmarshal
313 die_bad_access("could not connect")
316 expiry
= result
.get("TicketExpiration")
319 if expiry
> min_expiration
:
323 die_bad_access("perforce ticket expires in {0} seconds".format(expiry
))
326 # account without a timeout - all ok
329 elif code
== "error":
330 data
= result
.get("data")
332 die_bad_access("p4 error: {0}".format(data
))
334 die_bad_access("unknown error")
338 die_bad_access("unknown error code {0}".format(code
))
340 _p4_version_string
= None
341 def p4_version_string():
342 """Read the version string, showing just the last line, which
343 hopefully is the interesting version bit.
346 Perforce - The Fast Software Configuration Management System.
347 Copyright 1995-2011 Perforce Software. All rights reserved.
348 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
350 global _p4_version_string
351 if not _p4_version_string
:
352 a
= p4_read_pipe_lines(["-V"])
353 _p4_version_string
= a
[-1].rstrip()
354 return _p4_version_string
356 def p4_integrate(src
, dest
):
357 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
359 def p4_sync(f
, *options
):
360 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
363 # forcibly add file names with wildcards
364 if wildcard_present(f
):
365 p4_system(["add", "-f", f
])
367 p4_system(["add", f
])
370 p4_system(["delete", wildcard_encode(f
)])
372 def p4_edit(f
, *options
):
373 p4_system(["edit"] + list(options
) + [wildcard_encode(f
)])
376 p4_system(["revert", wildcard_encode(f
)])
378 def p4_reopen(type, f
):
379 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
381 def p4_reopen_in_change(changelist
, files
):
382 cmd
= ["reopen", "-c", str(changelist
)] + files
385 def p4_move(src
, dest
):
386 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
388 def p4_last_change():
389 results
= p4CmdList(["changes", "-m", "1"], skip_info
=True)
390 return int(results
[0]['change'])
392 def p4_describe(change
, shelved
=False):
393 """Make sure it returns a valid result by checking for
394 the presence of field "time". Return a dict of the
397 cmd
= ["describe", "-s"]
402 ds
= p4CmdList(cmd
, skip_info
=True)
404 die("p4 describe -s %d did not return 1 result: %s" % (change
, str(ds
)))
408 if "p4ExitCode" in d
:
409 die("p4 describe -s %d exited with %d: %s" % (change
, d
["p4ExitCode"],
412 if d
["code"] == "error":
413 die("p4 describe -s %d returned error code: %s" % (change
, str(d
)))
416 die("p4 describe -s %d returned no \"time\": %s" % (change
, str(d
)))
421 # Canonicalize the p4 type and return a tuple of the
422 # base type, plus any modifiers. See "p4 help filetypes"
423 # for a list and explanation.
425 def split_p4_type(p4type
):
427 p4_filetypes_historical
= {
428 "ctempobj": "binary+Sw",
434 "tempobj": "binary+FSw",
435 "ubinary": "binary+F",
436 "uresource": "resource+F",
437 "uxbinary": "binary+Fx",
438 "xbinary": "binary+x",
440 "xtempobj": "binary+Swx",
442 "xunicode": "unicode+x",
445 if p4type
in p4_filetypes_historical
:
446 p4type
= p4_filetypes_historical
[p4type
]
448 s
= p4type
.split("+")
456 # return the raw p4 type of a file (text, text+ko, etc)
459 results
= p4CmdList(["fstat", "-T", "headType", wildcard_encode(f
)])
460 return results
[0]['headType']
463 # Given a type base and modifier, return a regexp matching
464 # the keywords that can be expanded in the file
466 def p4_keywords_regexp_for_type(base
, type_mods
):
467 if base
in ("text", "unicode", "binary"):
469 if "ko" in type_mods
:
471 elif "k" in type_mods
:
472 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
476 \$ # Starts with a dollar, followed by...
477 (%s) # one of the keywords, followed by...
478 (:[^$\n]+)? # possibly an old expansion, followed by...
486 # Given a file, return a regexp matching the possible
487 # RCS keywords that will be expanded, or None for files
488 # with kw expansion turned off.
490 def p4_keywords_regexp_for_file(file):
491 if not os
.path
.exists(file):
494 (type_base
, type_mods
) = split_p4_type(p4_type(file))
495 return p4_keywords_regexp_for_type(type_base
, type_mods
)
497 def setP4ExecBit(file, mode
):
498 # Reopens an already open file and changes the execute bit to match
499 # the execute bit setting in the passed in mode.
503 if not isModeExec(mode
):
504 p4Type
= getP4OpenedType(file)
505 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
506 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
507 if p4Type
[-1] == "+":
508 p4Type
= p4Type
[0:-1]
510 p4_reopen(p4Type
, file)
512 def getP4OpenedType(file):
513 # Returns the perforce file type for the given file.
515 result
= p4_read_pipe(["opened", wildcard_encode(file)])
516 match
= re
.match(".*\((.+)\)( \*exclusive\*)?\r?$", result
)
518 return match
.group(1)
520 die("Could not determine file type for %s (result: '%s')" % (file, result
))
522 # Return the set of all p4 labels
523 def getP4Labels(depotPaths
):
525 if isinstance(depotPaths
,basestring
):
526 depotPaths
= [depotPaths
]
528 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
534 # Return the set of all git tags
537 for line
in read_pipe_lines(["git", "tag"]):
542 def diffTreePattern():
543 # This is a simple generator for the diff tree regex pattern. This could be
544 # a class variable if this and parseDiffTreeEntry were a part of a class.
545 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
549 def parseDiffTreeEntry(entry
):
550 """Parses a single diff tree entry into its component elements.
552 See git-diff-tree(1) manpage for details about the format of the diff
553 output. This method returns a dictionary with the following elements:
555 src_mode - The mode of the source file
556 dst_mode - The mode of the destination file
557 src_sha1 - The sha1 for the source file
558 dst_sha1 - The sha1 fr the destination file
559 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
560 status_score - The score for the status (applicable for 'C' and 'R'
561 statuses). This is None if there is no score.
562 src - The path for the source file.
563 dst - The path for the destination file. This is only present for
564 copy or renames. If it is not present, this is None.
566 If the pattern is not matched, None is returned."""
568 match
= diffTreePattern().next().match(entry
)
571 'src_mode': match
.group(1),
572 'dst_mode': match
.group(2),
573 'src_sha1': match
.group(3),
574 'dst_sha1': match
.group(4),
575 'status': match
.group(5),
576 'status_score': match
.group(6),
577 'src': match
.group(7),
578 'dst': match
.group(10)
582 def isModeExec(mode
):
583 # Returns True if the given git mode represents an executable file,
585 return mode
[-3:] == "755"
587 class P4Exception(Exception):
588 """ Base class for exceptions from the p4 client """
589 def __init__(self
, exit_code
):
590 self
.p4ExitCode
= exit_code
592 class P4ServerException(P4Exception
):
593 """ Base class for exceptions where we get some kind of marshalled up result from the server """
594 def __init__(self
, exit_code
, p4_result
):
595 super(P4ServerException
, self
).__init
__(exit_code
)
596 self
.p4_result
= p4_result
597 self
.code
= p4_result
[0]['code']
598 self
.data
= p4_result
[0]['data']
600 class P4RequestSizeException(P4ServerException
):
601 """ One of the maxresults or maxscanrows errors """
602 def __init__(self
, exit_code
, p4_result
, limit
):
603 super(P4RequestSizeException
, self
).__init
__(exit_code
, p4_result
)
606 def isModeExecChanged(src_mode
, dst_mode
):
607 return isModeExec(src_mode
) != isModeExec(dst_mode
)
609 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None, skip_info
=False,
610 errors_as_exceptions
=False):
612 if isinstance(cmd
,basestring
):
619 cmd
= p4_build_cmd(cmd
)
621 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
623 # Use a temporary file to avoid deadlocks without
624 # subprocess.communicate(), which would put another copy
625 # of stdout into memory.
627 if stdin
is not None:
628 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
629 if isinstance(stdin
,basestring
):
630 stdin_file
.write(stdin
)
633 stdin_file
.write(i
+ '\n')
637 p4
= subprocess
.Popen(cmd
,
640 stdout
=subprocess
.PIPE
)
645 entry
= marshal
.load(p4
.stdout
)
647 if 'code' in entry
and entry
['code'] == 'info':
657 if errors_as_exceptions
:
659 data
= result
[0].get('data')
661 m
= re
.search('Too many rows scanned \(over (\d+)\)', data
)
663 m
= re
.search('Request too large \(over (\d+)\)', data
)
666 limit
= int(m
.group(1))
667 raise P4RequestSizeException(exitCode
, result
, limit
)
669 raise P4ServerException(exitCode
, result
)
671 raise P4Exception(exitCode
)
674 entry
["p4ExitCode"] = exitCode
680 list = p4CmdList(cmd
)
686 def p4Where(depotPath
):
687 if not depotPath
.endswith("/"):
689 depotPathLong
= depotPath
+ "..."
690 outputList
= p4CmdList(["where", depotPathLong
])
692 for entry
in outputList
:
693 if "depotFile" in entry
:
694 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
695 # The base path always ends with "/...".
696 if entry
["depotFile"].find(depotPath
) == 0 and entry
["depotFile"][-4:] == "/...":
699 elif "data" in entry
:
700 data
= entry
.get("data")
701 space
= data
.find(" ")
702 if data
[:space
] == depotPath
:
707 if output
["code"] == "error":
711 clientPath
= output
.get("path")
712 elif "data" in output
:
713 data
= output
.get("data")
714 lastSpace
= data
.rfind(" ")
715 clientPath
= data
[lastSpace
+ 1:]
717 if clientPath
.endswith("..."):
718 clientPath
= clientPath
[:-3]
721 def currentGitBranch():
722 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
724 def isValidGitDir(path
):
725 return git_dir(path
) != None
727 def parseRevision(ref
):
728 return read_pipe("git rev-parse %s" % ref
).strip()
730 def branchExists(ref
):
731 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
735 def extractLogMessageFromGitCommit(commit
):
738 ## fixme: title is first line of commit, not 1st paragraph.
740 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
749 def extractSettingsGitLog(log
):
751 for line
in log
.split("\n"):
753 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
757 assignments
= m
.group(1).split (':')
758 for a
in assignments
:
760 key
= vals
[0].strip()
761 val
= ('='.join (vals
[1:])).strip()
762 if val
.endswith ('\"') and val
.startswith('"'):
767 paths
= values
.get("depot-paths")
769 paths
= values
.get("depot-path")
771 values
['depot-paths'] = paths
.split(',')
774 def gitBranchExists(branch
):
775 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
776 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
777 return proc
.wait() == 0;
779 def gitUpdateRef(ref
, newvalue
):
780 subprocess
.check_call(["git", "update-ref", ref
, newvalue
])
782 def gitDeleteRef(ref
):
783 subprocess
.check_call(["git", "update-ref", "-d", ref
])
787 def gitConfig(key
, typeSpecifier
=None):
788 if key
not in _gitConfig
:
789 cmd
= [ "git", "config" ]
791 cmd
+= [ typeSpecifier
]
793 s
= read_pipe(cmd
, ignore_error
=True)
794 _gitConfig
[key
] = s
.strip()
795 return _gitConfig
[key
]
797 def gitConfigBool(key
):
798 """Return a bool, using git config --bool. It is True only if the
799 variable is set to true, and False if set to false or not present
802 if key
not in _gitConfig
:
803 _gitConfig
[key
] = gitConfig(key
, '--bool') == "true"
804 return _gitConfig
[key
]
806 def gitConfigInt(key
):
807 if key
not in _gitConfig
:
808 cmd
= [ "git", "config", "--int", key
]
809 s
= read_pipe(cmd
, ignore_error
=True)
812 _gitConfig
[key
] = int(gitConfig(key
, '--int'))
814 _gitConfig
[key
] = None
815 return _gitConfig
[key
]
817 def gitConfigList(key
):
818 if key
not in _gitConfig
:
819 s
= read_pipe(["git", "config", "--get-all", key
], ignore_error
=True)
820 _gitConfig
[key
] = s
.strip().splitlines()
821 if _gitConfig
[key
] == ['']:
823 return _gitConfig
[key
]
825 def p4BranchesInGit(branchesAreInRemotes
=True):
826 """Find all the branches whose names start with "p4/", looking
827 in remotes or heads as specified by the argument. Return
828 a dictionary of { branch: revision } for each one found.
829 The branch names are the short names, without any
834 cmdline
= "git rev-parse --symbolic "
835 if branchesAreInRemotes
:
836 cmdline
+= "--remotes"
838 cmdline
+= "--branches"
840 for line
in read_pipe_lines(cmdline
):
844 if not line
.startswith('p4/'):
846 # special symbolic ref to p4/master
847 if line
== "p4/HEAD":
850 # strip off p4/ prefix
851 branch
= line
[len("p4/"):]
853 branches
[branch
] = parseRevision(line
)
857 def branch_exists(branch
):
858 """Make sure that the given ref name really exists."""
860 cmd
= [ "git", "rev-parse", "--symbolic", "--verify", branch
]
861 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
862 out
, _
= p
.communicate()
865 # expect exactly one line of output: the branch name
866 return out
.rstrip() == branch
868 def findUpstreamBranchPoint(head
= "HEAD"):
869 branches
= p4BranchesInGit()
870 # map from depot-path to branch name
871 branchByDepotPath
= {}
872 for branch
in branches
.keys():
873 tip
= branches
[branch
]
874 log
= extractLogMessageFromGitCommit(tip
)
875 settings
= extractSettingsGitLog(log
)
876 if "depot-paths" in settings
:
877 paths
= ",".join(settings
["depot-paths"])
878 branchByDepotPath
[paths
] = "remotes/p4/" + branch
882 while parent
< 65535:
883 commit
= head
+ "~%s" % parent
884 log
= extractLogMessageFromGitCommit(commit
)
885 settings
= extractSettingsGitLog(log
)
886 if "depot-paths" in settings
:
887 paths
= ",".join(settings
["depot-paths"])
888 if paths
in branchByDepotPath
:
889 return [branchByDepotPath
[paths
], settings
]
893 return ["", settings
]
895 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
897 print("Creating/updating branch(es) in %s based on origin branch(es)"
900 originPrefix
= "origin/p4/"
902 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
904 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
907 headName
= line
[len(originPrefix
):]
908 remoteHead
= localRefPrefix
+ headName
911 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
912 if ('depot-paths' not in original
913 or 'change' not in original
):
917 if not gitBranchExists(remoteHead
):
919 print("creating %s" % remoteHead
)
922 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
923 if 'change' in settings
:
924 if settings
['depot-paths'] == original
['depot-paths']:
925 originP4Change
= int(original
['change'])
926 p4Change
= int(settings
['change'])
927 if originP4Change
> p4Change
:
928 print("%s (%s) is newer than %s (%s). "
929 "Updating p4 branch from origin."
930 % (originHead
, originP4Change
,
931 remoteHead
, p4Change
))
934 print("Ignoring: %s was imported from %s while "
935 "%s was imported from %s"
936 % (originHead
, ','.join(original
['depot-paths']),
937 remoteHead
, ','.join(settings
['depot-paths'])))
940 system("git update-ref %s %s" % (remoteHead
, originHead
))
942 def originP4BranchesExist():
943 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
946 def p4ParseNumericChangeRange(parts
):
947 changeStart
= int(parts
[0][1:])
948 if parts
[1] == '#head':
949 changeEnd
= p4_last_change()
951 changeEnd
= int(parts
[1])
953 return (changeStart
, changeEnd
)
955 def chooseBlockSize(blockSize
):
959 return defaultBlockSize
961 def p4ChangesForPaths(depotPaths
, changeRange
, requestedBlockSize
):
964 # Parse the change range into start and end. Try to find integer
965 # revision ranges as these can be broken up into blocks to avoid
966 # hitting server-side limits (maxrows, maxscanresults). But if
967 # that doesn't work, fall back to using the raw revision specifier
968 # strings, without using block mode.
970 if changeRange
is None or changeRange
== '':
972 changeEnd
= p4_last_change()
973 block_size
= chooseBlockSize(requestedBlockSize
)
975 parts
= changeRange
.split(',')
976 assert len(parts
) == 2
978 (changeStart
, changeEnd
) = p4ParseNumericChangeRange(parts
)
979 block_size
= chooseBlockSize(requestedBlockSize
)
981 changeStart
= parts
[0][1:]
983 if requestedBlockSize
:
984 die("cannot use --changes-block-size with non-numeric revisions")
989 # Retrieve changes a block at a time, to prevent running
990 # into a MaxResults/MaxScanRows error from the server. If
991 # we _do_ hit one of those errors, turn down the block size
997 end
= min(changeEnd
, changeStart
+ block_size
)
998 revisionRange
= "%d,%d" % (changeStart
, end
)
1000 revisionRange
= "%s,%s" % (changeStart
, changeEnd
)
1002 for p
in depotPaths
:
1003 cmd
+= ["%s...@%s" % (p
, revisionRange
)]
1007 result
= p4CmdList(cmd
, errors_as_exceptions
=True)
1008 except P4RequestSizeException
as e
:
1010 block_size
= e
.limit
1011 elif block_size
> e
.limit
:
1012 block_size
= e
.limit
1014 block_size
= max(2, block_size
// 2)
1016 if verbose
: print("block size error, retrying with block size {0}".format(block_size
))
1018 except P4Exception
as e
:
1019 die('Error retrieving changes description ({0})'.format(e
.p4ExitCode
))
1021 # Insert changes in chronological order
1022 for entry
in reversed(result
):
1023 if 'change' not in entry
:
1025 changes
.add(int(entry
['change']))
1030 if end
>= changeEnd
:
1033 changeStart
= end
+ 1
1035 changes
= sorted(changes
)
1038 def p4PathStartsWith(path
, prefix
):
1039 # This method tries to remedy a potential mixed-case issue:
1041 # If UserA adds //depot/DirA/file1
1042 # and UserB adds //depot/dira/file2
1044 # we may or may not have a problem. If you have core.ignorecase=true,
1045 # we treat DirA and dira as the same directory
1046 if gitConfigBool("core.ignorecase"):
1047 return path
.lower().startswith(prefix
.lower())
1048 return path
.startswith(prefix
)
1050 def getClientSpec():
1051 """Look at the p4 client spec, create a View() object that contains
1052 all the mappings, and return it."""
1054 specList
= p4CmdList("client -o")
1055 if len(specList
) != 1:
1056 die('Output from "client -o" is %d lines, expecting 1' %
1059 # dictionary of all client parameters
1062 # the //client/ name
1063 client_name
= entry
["Client"]
1065 # just the keys that start with "View"
1066 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
1068 # hold this new View
1069 view
= View(client_name
)
1071 # append the lines, in order, to the view
1072 for view_num
in range(len(view_keys
)):
1073 k
= "View%d" % view_num
1074 if k
not in view_keys
:
1075 die("Expected view key %s missing" % k
)
1076 view
.append(entry
[k
])
1080 def getClientRoot():
1081 """Grab the client directory."""
1083 output
= p4CmdList("client -o")
1084 if len(output
) != 1:
1085 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
1088 if "Root" not in entry
:
1089 die('Client has no "Root"')
1091 return entry
["Root"]
1094 # P4 wildcards are not allowed in filenames. P4 complains
1095 # if you simply add them, but you can force it with "-f", in
1096 # which case it translates them into %xx encoding internally.
1098 def wildcard_decode(path
):
1099 # Search for and fix just these four characters. Do % last so
1100 # that fixing it does not inadvertently create new %-escapes.
1101 # Cannot have * in a filename in windows; untested as to
1102 # what p4 would do in such a case.
1103 if not platform
.system() == "Windows":
1104 path
= path
.replace("%2A", "*")
1105 path
= path
.replace("%23", "#") \
1106 .replace("%40", "@") \
1107 .replace("%25", "%")
1110 def wildcard_encode(path
):
1111 # do % first to avoid double-encoding the %s introduced here
1112 path
= path
.replace("%", "%25") \
1113 .replace("*", "%2A") \
1114 .replace("#", "%23") \
1115 .replace("@", "%40")
1118 def wildcard_present(path
):
1119 m
= re
.search("[*#@%]", path
)
1120 return m
is not None
1122 class LargeFileSystem(object):
1123 """Base class for large file system support."""
1125 def __init__(self
, writeToGitStream
):
1126 self
.largeFiles
= set()
1127 self
.writeToGitStream
= writeToGitStream
1129 def generatePointer(self
, cloneDestination
, contentFile
):
1130 """Return the content of a pointer file that is stored in Git instead of
1131 the actual content."""
1132 assert False, "Method 'generatePointer' required in " + self
.__class
__.__name
__
1134 def pushFile(self
, localLargeFile
):
1135 """Push the actual content which is not stored in the Git repository to
1137 assert False, "Method 'pushFile' required in " + self
.__class
__.__name
__
1139 def hasLargeFileExtension(self
, relPath
):
1141 lambda a
, b
: a
or b
,
1142 [relPath
.endswith('.' + e
) for e
in gitConfigList('git-p4.largeFileExtensions')],
1146 def generateTempFile(self
, contents
):
1147 contentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
1149 contentFile
.write(d
)
1151 return contentFile
.name
1153 def exceedsLargeFileThreshold(self
, relPath
, contents
):
1154 if gitConfigInt('git-p4.largeFileThreshold'):
1155 contentsSize
= sum(len(d
) for d
in contents
)
1156 if contentsSize
> gitConfigInt('git-p4.largeFileThreshold'):
1158 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1159 contentsSize
= sum(len(d
) for d
in contents
)
1160 if contentsSize
<= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1162 contentTempFile
= self
.generateTempFile(contents
)
1163 compressedContentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
1164 zf
= zipfile
.ZipFile(compressedContentFile
.name
, mode
='w')
1165 zf
.write(contentTempFile
, compress_type
=zipfile
.ZIP_DEFLATED
)
1167 compressedContentsSize
= zf
.infolist()[0].compress_size
1168 os
.remove(contentTempFile
)
1169 os
.remove(compressedContentFile
.name
)
1170 if compressedContentsSize
> gitConfigInt('git-p4.largeFileCompressedThreshold'):
1174 def addLargeFile(self
, relPath
):
1175 self
.largeFiles
.add(relPath
)
1177 def removeLargeFile(self
, relPath
):
1178 self
.largeFiles
.remove(relPath
)
1180 def isLargeFile(self
, relPath
):
1181 return relPath
in self
.largeFiles
1183 def processContent(self
, git_mode
, relPath
, contents
):
1184 """Processes the content of git fast import. This method decides if a
1185 file is stored in the large file system and handles all necessary
1187 if self
.exceedsLargeFileThreshold(relPath
, contents
) or self
.hasLargeFileExtension(relPath
):
1188 contentTempFile
= self
.generateTempFile(contents
)
1189 (pointer_git_mode
, contents
, localLargeFile
) = self
.generatePointer(contentTempFile
)
1190 if pointer_git_mode
:
1191 git_mode
= pointer_git_mode
1193 # Move temp file to final location in large file system
1194 largeFileDir
= os
.path
.dirname(localLargeFile
)
1195 if not os
.path
.isdir(largeFileDir
):
1196 os
.makedirs(largeFileDir
)
1197 shutil
.move(contentTempFile
, localLargeFile
)
1198 self
.addLargeFile(relPath
)
1199 if gitConfigBool('git-p4.largeFilePush'):
1200 self
.pushFile(localLargeFile
)
1202 sys
.stderr
.write("%s moved to large file system (%s)\n" % (relPath
, localLargeFile
))
1203 return (git_mode
, contents
)
1205 class MockLFS(LargeFileSystem
):
1206 """Mock large file system for testing."""
1208 def generatePointer(self
, contentFile
):
1209 """The pointer content is the original content prefixed with "pointer-".
1210 The local filename of the large file storage is derived from the file content.
1212 with
open(contentFile
, 'r') as f
:
1215 pointerContents
= 'pointer-' + content
1216 localLargeFile
= os
.path
.join(os
.getcwd(), '.git', 'mock-storage', 'local', content
[:-1])
1217 return (gitMode
, pointerContents
, localLargeFile
)
1219 def pushFile(self
, localLargeFile
):
1220 """The remote filename of the large file storage is the same as the local
1221 one but in a different directory.
1223 remotePath
= os
.path
.join(os
.path
.dirname(localLargeFile
), '..', 'remote')
1224 if not os
.path
.exists(remotePath
):
1225 os
.makedirs(remotePath
)
1226 shutil
.copyfile(localLargeFile
, os
.path
.join(remotePath
, os
.path
.basename(localLargeFile
)))
1228 class GitLFS(LargeFileSystem
):
1229 """Git LFS as backend for the git-p4 large file system.
1230 See https://git-lfs.github.com/ for details."""
1232 def __init__(self
, *args
):
1233 LargeFileSystem
.__init
__(self
, *args
)
1234 self
.baseGitAttributes
= []
1236 def generatePointer(self
, contentFile
):
1237 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1238 mode and content which is stored in the Git repository instead of
1239 the actual content. Return also the new location of the actual
1242 if os
.path
.getsize(contentFile
) == 0:
1243 return (None, '', None)
1245 pointerProcess
= subprocess
.Popen(
1246 ['git', 'lfs', 'pointer', '--file=' + contentFile
],
1247 stdout
=subprocess
.PIPE
1249 pointerFile
= pointerProcess
.stdout
.read()
1250 if pointerProcess
.wait():
1251 os
.remove(contentFile
)
1252 die('git-lfs pointer command failed. Did you install the extension?')
1254 # Git LFS removed the preamble in the output of the 'pointer' command
1255 # starting from version 1.2.0. Check for the preamble here to support
1257 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1258 if pointerFile
.startswith('Git LFS pointer for'):
1259 pointerFile
= re
.sub(r
'Git LFS pointer for.*\n\n', '', pointerFile
)
1261 oid
= re
.search(r
'^oid \w+:(\w+)', pointerFile
, re
.MULTILINE
).group(1)
1262 localLargeFile
= os
.path
.join(
1264 '.git', 'lfs', 'objects', oid
[:2], oid
[2:4],
1267 # LFS Spec states that pointer files should not have the executable bit set.
1269 return (gitMode
, pointerFile
, localLargeFile
)
1271 def pushFile(self
, localLargeFile
):
1272 uploadProcess
= subprocess
.Popen(
1273 ['git', 'lfs', 'push', '--object-id', 'origin', os
.path
.basename(localLargeFile
)]
1275 if uploadProcess
.wait():
1276 die('git-lfs push command failed. Did you define a remote?')
1278 def generateGitAttributes(self
):
1280 self
.baseGitAttributes
+
1284 '# Git LFS (see https://git-lfs.github.com/)\n',
1287 ['*.' + f
.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1288 for f
in sorted(gitConfigList('git-p4.largeFileExtensions'))
1290 ['/' + f
.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1291 for f
in sorted(self
.largeFiles
) if not self
.hasLargeFileExtension(f
)
1295 def addLargeFile(self
, relPath
):
1296 LargeFileSystem
.addLargeFile(self
, relPath
)
1297 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1299 def removeLargeFile(self
, relPath
):
1300 LargeFileSystem
.removeLargeFile(self
, relPath
)
1301 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1303 def processContent(self
, git_mode
, relPath
, contents
):
1304 if relPath
== '.gitattributes':
1305 self
.baseGitAttributes
= contents
1306 return (git_mode
, self
.generateGitAttributes())
1308 return LargeFileSystem
.processContent(self
, git_mode
, relPath
, contents
)
1311 delete_actions
= ( "delete", "move/delete", "purge" )
1312 add_actions
= ( "add", "move/add" )
1315 self
.usage
= "usage: %prog [options]"
1316 self
.needsGit
= True
1317 self
.verbose
= False
1319 # This is required for the "append" cloneExclude action
1320 def ensure_value(self
, attr
, value
):
1321 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
1322 setattr(self
, attr
, value
)
1323 return getattr(self
, attr
)
1327 self
.userMapFromPerforceServer
= False
1328 self
.myP4UserId
= None
1332 return self
.myP4UserId
1334 results
= p4CmdList("user -o")
1337 self
.myP4UserId
= r
['User']
1339 die("Could not find your p4 user id")
1341 def p4UserIsMe(self
, p4User
):
1342 # return True if the given p4 user is actually me
1343 me
= self
.p4UserId()
1344 if not p4User
or p4User
!= me
:
1349 def getUserCacheFilename(self
):
1350 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
1351 return home
+ "/.gitp4-usercache.txt"
1353 def getUserMapFromPerforceServer(self
):
1354 if self
.userMapFromPerforceServer
:
1359 for output
in p4CmdList("users"):
1360 if "User" not in output
:
1362 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
1363 self
.emails
[output
["Email"]] = output
["User"]
1365 mapUserConfigRegex
= re
.compile(r
"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re
.VERBOSE
)
1366 for mapUserConfig
in gitConfigList("git-p4.mapUser"):
1367 mapUser
= mapUserConfigRegex
.findall(mapUserConfig
)
1368 if mapUser
and len(mapUser
[0]) == 3:
1369 user
= mapUser
[0][0]
1370 fullname
= mapUser
[0][1]
1371 email
= mapUser
[0][2]
1372 self
.users
[user
] = fullname
+ " <" + email
+ ">"
1373 self
.emails
[email
] = user
1376 for (key
, val
) in self
.users
.items():
1377 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
1379 open(self
.getUserCacheFilename(), "wb").write(s
)
1380 self
.userMapFromPerforceServer
= True
1382 def loadUserMapFromCache(self
):
1384 self
.userMapFromPerforceServer
= False
1386 cache
= open(self
.getUserCacheFilename(), "rb")
1387 lines
= cache
.readlines()
1390 entry
= line
.strip().split("\t")
1391 self
.users
[entry
[0]] = entry
[1]
1393 self
.getUserMapFromPerforceServer()
1395 class P4Debug(Command
):
1397 Command
.__init
__(self
)
1399 self
.description
= "A tool to debug the output of p4 -G."
1400 self
.needsGit
= False
1402 def run(self
, args
):
1404 for output
in p4CmdList(args
):
1405 print('Element: %d' % j
)
1410 class P4RollBack(Command
):
1412 Command
.__init
__(self
)
1414 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
1416 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
1417 self
.rollbackLocalBranches
= False
1419 def run(self
, args
):
1422 maxChange
= int(args
[0])
1424 if "p4ExitCode" in p4Cmd("changes -m 1"):
1425 die("Problems executing p4");
1427 if self
.rollbackLocalBranches
:
1428 refPrefix
= "refs/heads/"
1429 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
1431 refPrefix
= "refs/remotes/"
1432 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
1435 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
1437 ref
= refPrefix
+ line
1438 log
= extractLogMessageFromGitCommit(ref
)
1439 settings
= extractSettingsGitLog(log
)
1441 depotPaths
= settings
['depot-paths']
1442 change
= settings
['change']
1446 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
1447 for p
in depotPaths
]))) == 0:
1448 print("Branch %s did not exist at change %s, deleting." % (ref
, maxChange
))
1449 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
1452 while change
and int(change
) > maxChange
:
1455 print("%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
))
1456 system("git update-ref %s \"%s^\"" % (ref
, ref
))
1457 log
= extractLogMessageFromGitCommit(ref
)
1458 settings
= extractSettingsGitLog(log
)
1461 depotPaths
= settings
['depot-paths']
1462 change
= settings
['change']
1465 print("%s rewound to %s" % (ref
, change
))
1469 class P4Submit(Command
, P4UserMap
):
1471 conflict_behavior_choices
= ("ask", "skip", "quit")
1474 Command
.__init
__(self
)
1475 P4UserMap
.__init
__(self
)
1477 optparse
.make_option("--origin", dest
="origin"),
1478 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
1479 # preserve the user, requires relevant p4 permissions
1480 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
1481 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
1482 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
1483 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
1484 optparse
.make_option("--conflict", dest
="conflict_behavior",
1485 choices
=self
.conflict_behavior_choices
),
1486 optparse
.make_option("--branch", dest
="branch"),
1487 optparse
.make_option("--shelve", dest
="shelve", action
="store_true",
1488 help="Shelve instead of submit. Shelved files are reverted, "
1489 "restoring the workspace to the state before the shelve"),
1490 optparse
.make_option("--update-shelve", dest
="update_shelve", action
="append", type="int",
1491 metavar
="CHANGELIST",
1492 help="update an existing shelved changelist, implies --shelve, "
1493 "repeat in-order for multiple shelved changelists"),
1494 optparse
.make_option("--commit", dest
="commit", metavar
="COMMIT",
1495 help="submit only the specified commit(s), one commit or xxx..xxx"),
1496 optparse
.make_option("--disable-rebase", dest
="disable_rebase", action
="store_true",
1497 help="Disable rebase after submit is completed. Can be useful if you "
1498 "work from a local git branch that is not master"),
1499 optparse
.make_option("--disable-p4sync", dest
="disable_p4sync", action
="store_true",
1500 help="Skip Perforce sync of p4/master after submit or shelve"),
1502 self
.description
= """Submit changes from git to the perforce depot.\n
1503 The `p4-pre-submit` hook is executed if it exists and is executable.
1504 The hook takes no parameters and nothing from standard input. Exiting with
1505 non-zero status from this script prevents `git-p4 submit` from launching.
1507 One usage scenario is to run unit tests in the hook."""
1509 self
.usage
+= " [name of git branch to submit into perforce depot]"
1511 self
.detectRenames
= False
1512 self
.preserveUser
= gitConfigBool("git-p4.preserveUser")
1513 self
.dry_run
= False
1515 self
.update_shelve
= list()
1517 self
.disable_rebase
= gitConfigBool("git-p4.disableRebase")
1518 self
.disable_p4sync
= gitConfigBool("git-p4.disableP4Sync")
1519 self
.prepare_p4_only
= False
1520 self
.conflict_behavior
= None
1521 self
.isWindows
= (platform
.system() == "Windows")
1522 self
.exportLabels
= False
1523 self
.p4HasMoveCommand
= p4_has_move_command()
1526 if gitConfig('git-p4.largeFileSystem'):
1527 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1530 if len(p4CmdList("opened ...")) > 0:
1531 die("You have files opened with perforce! Close them before starting the sync.")
1533 def separate_jobs_from_description(self
, message
):
1534 """Extract and return a possible Jobs field in the commit
1535 message. It goes into a separate section in the p4 change
1538 A jobs line starts with "Jobs:" and looks like a new field
1539 in a form. Values are white-space separated on the same
1540 line or on following lines that start with a tab.
1542 This does not parse and extract the full git commit message
1543 like a p4 form. It just sees the Jobs: line as a marker
1544 to pass everything from then on directly into the p4 form,
1545 but outside the description section.
1547 Return a tuple (stripped log message, jobs string)."""
1549 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
1551 return (message
, None)
1553 jobtext
= message
[m
.start():]
1554 stripped_message
= message
[:m
.start()].rstrip()
1555 return (stripped_message
, jobtext
)
1557 def prepareLogMessage(self
, template
, message
, jobs
):
1558 """Edits the template returned from "p4 change -o" to insert
1559 the message in the Description field, and the jobs text in
1563 inDescriptionSection
= False
1565 for line
in template
.split("\n"):
1566 if line
.startswith("#"):
1567 result
+= line
+ "\n"
1570 if inDescriptionSection
:
1571 if line
.startswith("Files:") or line
.startswith("Jobs:"):
1572 inDescriptionSection
= False
1573 # insert Jobs section
1575 result
+= jobs
+ "\n"
1579 if line
.startswith("Description:"):
1580 inDescriptionSection
= True
1582 for messageLine
in message
.split("\n"):
1583 line
+= "\t" + messageLine
+ "\n"
1585 result
+= line
+ "\n"
1589 def patchRCSKeywords(self
, file, pattern
):
1590 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1591 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
1593 outFile
= os
.fdopen(handle
, "w+")
1594 inFile
= open(file, "r")
1595 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1596 for line
in inFile
.readlines():
1597 line
= regexp
.sub(r
'$\1$', line
)
1601 # Forcibly overwrite the original file
1603 shutil
.move(outFileName
, file)
1605 # cleanup our temporary file
1606 os
.unlink(outFileName
)
1607 print("Failed to strip RCS keywords in %s" % file)
1610 print("Patched up RCS keywords in %s" % file)
1612 def p4UserForCommit(self
,id):
1613 # Return the tuple (perforce user,git email) for a given git commit id
1614 self
.getUserMapFromPerforceServer()
1615 gitEmail
= read_pipe(["git", "log", "--max-count=1",
1616 "--format=%ae", id])
1617 gitEmail
= gitEmail
.strip()
1618 if gitEmail
not in self
.emails
:
1619 return (None,gitEmail
)
1621 return (self
.emails
[gitEmail
],gitEmail
)
1623 def checkValidP4Users(self
,commits
):
1624 # check if any git authors cannot be mapped to p4 users
1626 (user
,email
) = self
.p4UserForCommit(id)
1628 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
1629 if gitConfigBool("git-p4.allowMissingP4Users"):
1632 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
1634 def lastP4Changelist(self
):
1635 # Get back the last changelist number submitted in this client spec. This
1636 # then gets used to patch up the username in the change. If the same
1637 # client spec is being used by multiple processes then this might go
1639 results
= p4CmdList("client -o") # find the current client
1643 client
= r
['Client']
1646 die("could not get client spec")
1647 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
1651 die("Could not get changelist number for last submit - cannot patch up user details")
1653 def modifyChangelistUser(self
, changelist
, newUser
):
1654 # fixup the user field of a changelist after it has been submitted.
1655 changes
= p4CmdList("change -o %s" % changelist
)
1656 if len(changes
) != 1:
1657 die("Bad output from p4 change modifying %s to user %s" %
1658 (changelist
, newUser
))
1661 if c
['User'] == newUser
: return # nothing to do
1663 input = marshal
.dumps(c
)
1665 result
= p4CmdList("change -f -i", stdin
=input)
1668 if r
['code'] == 'error':
1669 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1671 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1673 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1675 def canChangeChangelists(self
):
1676 # check to see if we have p4 admin or super-user permissions, either of
1677 # which are required to modify changelists.
1678 results
= p4CmdList(["protects", self
.depotPath
])
1681 if r
['perm'] == 'admin':
1683 if r
['perm'] == 'super':
1687 def prepareSubmitTemplate(self
, changelist
=None):
1688 """Run "p4 change -o" to grab a change specification template.
1689 This does not use "p4 -G", as it is nice to keep the submission
1690 template in original order, since a human might edit it.
1692 Remove lines in the Files section that show changes to files
1693 outside the depot path we're committing into."""
1695 [upstream
, settings
] = findUpstreamBranchPoint()
1698 # A Perforce Change Specification.
1700 # Change: The change number. 'new' on a new changelist.
1701 # Date: The date this specification was last modified.
1702 # Client: The client on which the changelist was created. Read-only.
1703 # User: The user who created the changelist.
1704 # Status: Either 'pending' or 'submitted'. Read-only.
1705 # Type: Either 'public' or 'restricted'. Default is 'public'.
1706 # Description: Comments about the changelist. Required.
1707 # Jobs: What opened jobs are to be closed by this changelist.
1708 # You may delete jobs from this list. (New changelists only.)
1709 # Files: What opened files from the default changelist are to be added
1710 # to this changelist. You may delete files from this list.
1711 # (New changelists only.)
1714 inFilesSection
= False
1716 args
= ['change', '-o']
1718 args
.append(str(changelist
))
1719 for entry
in p4CmdList(args
):
1720 if 'code' not in entry
:
1722 if entry
['code'] == 'stat':
1723 change_entry
= entry
1725 if not change_entry
:
1726 die('Failed to decode output of p4 change -o')
1727 for key
, value
in change_entry
.iteritems():
1728 if key
.startswith('File'):
1729 if 'depot-paths' in settings
:
1730 if not [p
for p
in settings
['depot-paths']
1731 if p4PathStartsWith(value
, p
)]:
1734 if not p4PathStartsWith(value
, self
.depotPath
):
1736 files_list
.append(value
)
1738 # Output in the order expected by prepareLogMessage
1739 for key
in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1740 if key
not in change_entry
:
1743 template
+= key
+ ':'
1744 if key
== 'Description':
1746 for field_line
in change_entry
[key
].splitlines():
1747 template
+= '\t'+field_line
+'\n'
1748 if len(files_list
) > 0:
1750 template
+= 'Files:\n'
1751 for path
in files_list
:
1752 template
+= '\t'+path
+'\n'
1755 def edit_template(self
, template_file
):
1756 """Invoke the editor to let the user change the submission
1757 message. Return true if okay to continue with the submit."""
1759 # if configured to skip the editing part, just submit
1760 if gitConfigBool("git-p4.skipSubmitEdit"):
1763 # look at the modification time, to check later if the user saved
1765 mtime
= os
.stat(template_file
).st_mtime
1768 if "P4EDITOR" in os
.environ
and (os
.environ
.get("P4EDITOR") != ""):
1769 editor
= os
.environ
.get("P4EDITOR")
1771 editor
= read_pipe("git var GIT_EDITOR").strip()
1772 system(["sh", "-c", ('%s "$@"' % editor
), editor
, template_file
])
1774 # If the file was not saved, prompt to see if this patch should
1775 # be skipped. But skip this verification step if configured so.
1776 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1779 # modification time updated means user saved the file
1780 if os
.stat(template_file
).st_mtime
> mtime
:
1784 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1790 def get_diff_description(self
, editedFiles
, filesToAdd
, symlinks
):
1792 if "P4DIFF" in os
.environ
:
1793 del(os
.environ
["P4DIFF"])
1795 for editedFile
in editedFiles
:
1796 diff
+= p4_read_pipe(['diff', '-du',
1797 wildcard_encode(editedFile
)])
1801 for newFile
in filesToAdd
:
1802 newdiff
+= "==== new file ====\n"
1803 newdiff
+= "--- /dev/null\n"
1804 newdiff
+= "+++ %s\n" % newFile
1806 is_link
= os
.path
.islink(newFile
)
1807 expect_link
= newFile
in symlinks
1809 if is_link
and expect_link
:
1810 newdiff
+= "+%s\n" % os
.readlink(newFile
)
1812 f
= open(newFile
, "r")
1813 for line
in f
.readlines():
1814 newdiff
+= "+" + line
1817 return (diff
+ newdiff
).replace('\r\n', '\n')
1819 def applyCommit(self
, id):
1820 """Apply one commit, return True if it succeeded."""
1822 print("Applying", read_pipe(["git", "show", "-s",
1823 "--format=format:%h %s", id]))
1825 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1827 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1829 filesToChangeType
= set()
1830 filesToDelete
= set()
1832 pureRenameCopy
= set()
1834 filesToChangeExecBit
= {}
1838 diff
= parseDiffTreeEntry(line
)
1839 modifier
= diff
['status']
1841 all_files
.append(path
)
1845 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1846 filesToChangeExecBit
[path
] = diff
['dst_mode']
1847 editedFiles
.add(path
)
1848 elif modifier
== "A":
1849 filesToAdd
.add(path
)
1850 filesToChangeExecBit
[path
] = diff
['dst_mode']
1851 if path
in filesToDelete
:
1852 filesToDelete
.remove(path
)
1854 dst_mode
= int(diff
['dst_mode'], 8)
1855 if dst_mode
== 0o120000:
1858 elif modifier
== "D":
1859 filesToDelete
.add(path
)
1860 if path
in filesToAdd
:
1861 filesToAdd
.remove(path
)
1862 elif modifier
== "C":
1863 src
, dest
= diff
['src'], diff
['dst']
1864 all_files
.append(dest
)
1865 p4_integrate(src
, dest
)
1866 pureRenameCopy
.add(dest
)
1867 if diff
['src_sha1'] != diff
['dst_sha1']:
1869 pureRenameCopy
.discard(dest
)
1870 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1872 pureRenameCopy
.discard(dest
)
1873 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1875 # turn off read-only attribute
1876 os
.chmod(dest
, stat
.S_IWRITE
)
1878 editedFiles
.add(dest
)
1879 elif modifier
== "R":
1880 src
, dest
= diff
['src'], diff
['dst']
1881 all_files
.append(dest
)
1882 if self
.p4HasMoveCommand
:
1883 p4_edit(src
) # src must be open before move
1884 p4_move(src
, dest
) # opens for (move/delete, move/add)
1886 p4_integrate(src
, dest
)
1887 if diff
['src_sha1'] != diff
['dst_sha1']:
1890 pureRenameCopy
.add(dest
)
1891 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1892 if not self
.p4HasMoveCommand
:
1893 p4_edit(dest
) # with move: already open, writable
1894 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1895 if not self
.p4HasMoveCommand
:
1897 os
.chmod(dest
, stat
.S_IWRITE
)
1899 filesToDelete
.add(src
)
1900 editedFiles
.add(dest
)
1901 elif modifier
== "T":
1902 filesToChangeType
.add(path
)
1904 die("unknown modifier %s for %s" % (modifier
, path
))
1906 diffcmd
= "git diff-tree --full-index -p \"%s\"" % (id)
1907 patchcmd
= diffcmd
+ " | git apply "
1908 tryPatchCmd
= patchcmd
+ "--check -"
1909 applyPatchCmd
= patchcmd
+ "--check --apply -"
1910 patch_succeeded
= True
1912 if os
.system(tryPatchCmd
) != 0:
1913 fixed_rcs_keywords
= False
1914 patch_succeeded
= False
1915 print("Unfortunately applying the change failed!")
1917 # Patch failed, maybe it's just RCS keyword woes. Look through
1918 # the patch to see if that's possible.
1919 if gitConfigBool("git-p4.attemptRCSCleanup"):
1923 for file in editedFiles | filesToDelete
:
1924 # did this file's delta contain RCS keywords?
1925 pattern
= p4_keywords_regexp_for_file(file)
1928 # this file is a possibility...look for RCS keywords.
1929 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1930 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1931 if regexp
.search(line
):
1933 print("got keyword match on %s in %s in %s" % (pattern
, line
, file))
1934 kwfiles
[file] = pattern
1937 for file in kwfiles
:
1939 print("zapping %s with %s" % (line
,pattern
))
1940 # File is being deleted, so not open in p4. Must
1941 # disable the read-only bit on windows.
1942 if self
.isWindows
and file not in editedFiles
:
1943 os
.chmod(file, stat
.S_IWRITE
)
1944 self
.patchRCSKeywords(file, kwfiles
[file])
1945 fixed_rcs_keywords
= True
1947 if fixed_rcs_keywords
:
1948 print("Retrying the patch with RCS keywords cleaned up")
1949 if os
.system(tryPatchCmd
) == 0:
1950 patch_succeeded
= True
1952 if not patch_succeeded
:
1953 for f
in editedFiles
:
1958 # Apply the patch for real, and do add/delete/+x handling.
1960 system(applyPatchCmd
)
1962 for f
in filesToChangeType
:
1963 p4_edit(f
, "-t", "auto")
1964 for f
in filesToAdd
:
1966 for f
in filesToDelete
:
1970 # Set/clear executable bits
1971 for f
in filesToChangeExecBit
.keys():
1972 mode
= filesToChangeExecBit
[f
]
1973 setP4ExecBit(f
, mode
)
1976 if len(self
.update_shelve
) > 0:
1977 update_shelve
= self
.update_shelve
.pop(0)
1978 p4_reopen_in_change(update_shelve
, all_files
)
1981 # Build p4 change description, starting with the contents
1982 # of the git commit message.
1984 logMessage
= extractLogMessageFromGitCommit(id)
1985 logMessage
= logMessage
.strip()
1986 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1988 template
= self
.prepareSubmitTemplate(update_shelve
)
1989 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1991 if self
.preserveUser
:
1992 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
1994 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1995 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1996 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1997 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1999 separatorLine
= "######## everything below this line is just the diff #######\n"
2000 if not self
.prepare_p4_only
:
2001 submitTemplate
+= separatorLine
2002 submitTemplate
+= self
.get_diff_description(editedFiles
, filesToAdd
, symlinks
)
2004 (handle
, fileName
) = tempfile
.mkstemp()
2005 tmpFile
= os
.fdopen(handle
, "w+b")
2007 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
2008 tmpFile
.write(submitTemplate
)
2011 if self
.prepare_p4_only
:
2013 # Leave the p4 tree prepared, and the submit template around
2014 # and let the user decide what to do next
2017 print("P4 workspace prepared for submission.")
2018 print("To submit or revert, go to client workspace")
2019 print(" " + self
.clientPath
)
2021 print("To submit, use \"p4 submit\" to write a new description,")
2022 print("or \"p4 submit -i <%s\" to use the one prepared by" \
2023 " \"git p4\"." % fileName
)
2024 print("You can delete the file \"%s\" when finished." % fileName
)
2026 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
2027 print("To preserve change ownership by user %s, you must\n" \
2028 "do \"p4 change -f <change>\" after submitting and\n" \
2029 "edit the User field.")
2031 print("After submitting, renamed files must be re-synced.")
2032 print("Invoke \"p4 sync -f\" on each of these files:")
2033 for f
in pureRenameCopy
:
2037 print("To revert the changes, use \"p4 revert ...\", and delete")
2038 print("the submit template file \"%s\"" % fileName
)
2040 print("Since the commit adds new files, they must be deleted:")
2041 for f
in filesToAdd
:
2047 # Let the user edit the change description, then submit it.
2052 if self
.edit_template(fileName
):
2053 # read the edited message and submit
2054 tmpFile
= open(fileName
, "rb")
2055 message
= tmpFile
.read()
2058 message
= message
.replace("\r\n", "\n")
2059 submitTemplate
= message
[:message
.index(separatorLine
)]
2062 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate
)
2064 p4_write_pipe(['shelve', '-i'], submitTemplate
)
2066 p4_write_pipe(['submit', '-i'], submitTemplate
)
2067 # The rename/copy happened by applying a patch that created a
2068 # new file. This leaves it writable, which confuses p4.
2069 for f
in pureRenameCopy
:
2072 if self
.preserveUser
:
2074 # Get last changelist number. Cannot easily get it from
2075 # the submit command output as the output is
2077 changelist
= self
.lastP4Changelist()
2078 self
.modifyChangelistUser(changelist
, p4User
)
2084 if not submitted
or self
.shelve
:
2086 print ("Reverting shelved files.")
2088 print ("Submission cancelled, undoing p4 changes.")
2089 for f
in editedFiles | filesToDelete
:
2091 for f
in filesToAdd
:
2098 # Export git tags as p4 labels. Create a p4 label and then tag
2100 def exportGitTags(self
, gitTags
):
2101 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
2102 if len(validLabelRegexp
) == 0:
2103 validLabelRegexp
= defaultLabelRegexp
2104 m
= re
.compile(validLabelRegexp
)
2106 for name
in gitTags
:
2108 if not m
.match(name
):
2110 print("tag %s does not match regexp %s" % (name
, validLabelRegexp
))
2113 # Get the p4 commit this corresponds to
2114 logMessage
= extractLogMessageFromGitCommit(name
)
2115 values
= extractSettingsGitLog(logMessage
)
2117 if 'change' not in values
:
2118 # a tag pointing to something not sent to p4; ignore
2120 print("git tag %s does not give a p4 commit" % name
)
2123 changelist
= values
['change']
2125 # Get the tag details.
2129 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
2132 if re
.match(r
'tag\s+', l
):
2134 elif re
.match(r
'\s*$', l
):
2141 body
= ["lightweight tag imported by git p4\n"]
2143 # Create the label - use the same view as the client spec we are using
2144 clientSpec
= getClientSpec()
2146 labelTemplate
= "Label: %s\n" % name
2147 labelTemplate
+= "Description:\n"
2149 labelTemplate
+= "\t" + b
+ "\n"
2150 labelTemplate
+= "View:\n"
2151 for depot_side
in clientSpec
.mappings
:
2152 labelTemplate
+= "\t%s\n" % depot_side
2155 print("Would create p4 label %s for tag" % name
)
2156 elif self
.prepare_p4_only
:
2157 print("Not creating p4 label %s for tag due to option" \
2158 " --prepare-p4-only" % name
)
2160 p4_write_pipe(["label", "-i"], labelTemplate
)
2163 p4_system(["tag", "-l", name
] +
2164 ["%s@%s" % (depot_side
, changelist
) for depot_side
in clientSpec
.mappings
])
2167 print("created p4 label for tag %s" % name
)
2169 def run(self
, args
):
2171 self
.master
= currentGitBranch()
2172 elif len(args
) == 1:
2173 self
.master
= args
[0]
2174 if not branchExists(self
.master
):
2175 die("Branch %s does not exist" % self
.master
)
2179 for i
in self
.update_shelve
:
2181 sys
.exit("invalid changelist %d" % i
)
2184 allowSubmit
= gitConfig("git-p4.allowSubmit")
2185 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
2186 die("%s is not in git-p4.allowSubmit" % self
.master
)
2188 [upstream
, settings
] = findUpstreamBranchPoint()
2189 self
.depotPath
= settings
['depot-paths'][0]
2190 if len(self
.origin
) == 0:
2191 self
.origin
= upstream
2193 if len(self
.update_shelve
) > 0:
2196 if self
.preserveUser
:
2197 if not self
.canChangeChangelists():
2198 die("Cannot preserve user names without p4 super-user or admin permissions")
2200 # if not set from the command line, try the config file
2201 if self
.conflict_behavior
is None:
2202 val
= gitConfig("git-p4.conflict")
2204 if val
not in self
.conflict_behavior_choices
:
2205 die("Invalid value '%s' for config git-p4.conflict" % val
)
2208 self
.conflict_behavior
= val
2211 print("Origin branch is " + self
.origin
)
2213 if len(self
.depotPath
) == 0:
2214 print("Internal error: cannot locate perforce depot path from existing branches")
2217 self
.useClientSpec
= False
2218 if gitConfigBool("git-p4.useclientspec"):
2219 self
.useClientSpec
= True
2220 if self
.useClientSpec
:
2221 self
.clientSpecDirs
= getClientSpec()
2223 # Check for the existence of P4 branches
2224 branchesDetected
= (len(p4BranchesInGit().keys()) > 1)
2226 if self
.useClientSpec
and not branchesDetected
:
2227 # all files are relative to the client spec
2228 self
.clientPath
= getClientRoot()
2230 self
.clientPath
= p4Where(self
.depotPath
)
2232 if self
.clientPath
== "":
2233 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
2235 print("Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
))
2236 self
.oldWorkingDirectory
= os
.getcwd()
2238 # ensure the clientPath exists
2239 new_client_dir
= False
2240 if not os
.path
.exists(self
.clientPath
):
2241 new_client_dir
= True
2242 os
.makedirs(self
.clientPath
)
2244 chdir(self
.clientPath
, is_client_path
=True)
2246 print("Would synchronize p4 checkout in %s" % self
.clientPath
)
2248 print("Synchronizing p4 checkout...")
2250 # old one was destroyed, and maybe nobody told p4
2251 p4_sync("...", "-f")
2258 committish
= self
.master
2262 if self
.commit
!= "":
2263 if self
.commit
.find("..") != -1:
2264 limits_ish
= self
.commit
.split("..")
2265 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish
[0], limits_ish
[1])]):
2266 commits
.append(line
.strip())
2269 commits
.append(self
.commit
)
2271 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self
.origin
, committish
)]):
2272 commits
.append(line
.strip())
2275 if self
.preserveUser
or gitConfigBool("git-p4.skipUserNameCheck"):
2276 self
.checkAuthorship
= False
2278 self
.checkAuthorship
= True
2280 if self
.preserveUser
:
2281 self
.checkValidP4Users(commits
)
2284 # Build up a set of options to be passed to diff when
2285 # submitting each commit to p4.
2287 if self
.detectRenames
:
2288 # command-line -M arg
2289 self
.diffOpts
= "-M"
2291 # If not explicitly set check the config variable
2292 detectRenames
= gitConfig("git-p4.detectRenames")
2294 if detectRenames
.lower() == "false" or detectRenames
== "":
2296 elif detectRenames
.lower() == "true":
2297 self
.diffOpts
= "-M"
2299 self
.diffOpts
= "-M%s" % detectRenames
2301 # no command-line arg for -C or --find-copies-harder, just
2303 detectCopies
= gitConfig("git-p4.detectCopies")
2304 if detectCopies
.lower() == "false" or detectCopies
== "":
2306 elif detectCopies
.lower() == "true":
2307 self
.diffOpts
+= " -C"
2309 self
.diffOpts
+= " -C%s" % detectCopies
2311 if gitConfigBool("git-p4.detectCopiesHarder"):
2312 self
.diffOpts
+= " --find-copies-harder"
2314 num_shelves
= len(self
.update_shelve
)
2315 if num_shelves
> 0 and num_shelves
!= len(commits
):
2316 sys
.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2317 (len(commits
), num_shelves
))
2319 hooks_path
= gitConfig("core.hooksPath")
2320 if len(hooks_path
) <= 0:
2321 hooks_path
= os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), "hooks")
2323 hook_file
= os
.path
.join(hooks_path
, "p4-pre-submit")
2324 if os
.path
.isfile(hook_file
) and os
.access(hook_file
, os
.X_OK
) and subprocess
.call([hook_file
]) != 0:
2328 # Apply the commits, one at a time. On failure, ask if should
2329 # continue to try the rest of the patches, or quit.
2332 print("Would apply")
2334 last
= len(commits
) - 1
2335 for i
, commit
in enumerate(commits
):
2337 print(" ", read_pipe(["git", "show", "-s",
2338 "--format=format:%h %s", commit
]))
2341 ok
= self
.applyCommit(commit
)
2343 applied
.append(commit
)
2345 if self
.prepare_p4_only
and i
< last
:
2346 print("Processing only the first commit due to option" \
2347 " --prepare-p4-only")
2352 # prompt for what to do, or use the option/variable
2353 if self
.conflict_behavior
== "ask":
2354 print("What do you want to do?")
2355 response
= raw_input("[s]kip this commit but apply"
2356 " the rest, or [q]uit? ")
2359 elif self
.conflict_behavior
== "skip":
2361 elif self
.conflict_behavior
== "quit":
2364 die("Unknown conflict_behavior '%s'" %
2365 self
.conflict_behavior
)
2367 if response
[0] == "s":
2368 print("Skipping this commit, but applying the rest")
2370 if response
[0] == "q":
2377 chdir(self
.oldWorkingDirectory
)
2378 shelved_applied
= "shelved" if self
.shelve
else "applied"
2381 elif self
.prepare_p4_only
:
2383 elif len(commits
) == len(applied
):
2384 print("All commits {0}!".format(shelved_applied
))
2388 sync
.branch
= self
.branch
2389 if self
.disable_p4sync
:
2390 sync
.sync_origin_only()
2394 if not self
.disable_rebase
:
2399 if len(applied
) == 0:
2400 print("No commits {0}.".format(shelved_applied
))
2402 print("{0} only the commits marked with '*':".format(shelved_applied
.capitalize()))
2408 print(star
, read_pipe(["git", "show", "-s",
2409 "--format=format:%h %s", c
]))
2410 print("You will have to do 'git p4 sync' and rebase.")
2412 if gitConfigBool("git-p4.exportLabels"):
2413 self
.exportLabels
= True
2415 if self
.exportLabels
:
2416 p4Labels
= getP4Labels(self
.depotPath
)
2417 gitTags
= getGitTags()
2419 missingGitTags
= gitTags
- p4Labels
2420 self
.exportGitTags(missingGitTags
)
2422 # exit with error unless everything applied perfectly
2423 if len(commits
) != len(applied
):
2429 """Represent a p4 view ("p4 help views"), and map files in a
2430 repo according to the view."""
2432 def __init__(self
, client_name
):
2434 self
.client_prefix
= "//%s/" % client_name
2435 # cache results of "p4 where" to lookup client file locations
2436 self
.client_spec_path_cache
= {}
2438 def append(self
, view_line
):
2439 """Parse a view line, splitting it into depot and client
2440 sides. Append to self.mappings, preserving order. This
2441 is only needed for tag creation."""
2443 # Split the view line into exactly two words. P4 enforces
2444 # structure on these lines that simplifies this quite a bit.
2446 # Either or both words may be double-quoted.
2447 # Single quotes do not matter.
2448 # Double-quote marks cannot occur inside the words.
2449 # A + or - prefix is also inside the quotes.
2450 # There are no quotes unless they contain a space.
2451 # The line is already white-space stripped.
2452 # The two words are separated by a single space.
2454 if view_line
[0] == '"':
2455 # First word is double quoted. Find its end.
2456 close_quote_index
= view_line
.find('"', 1)
2457 if close_quote_index
<= 0:
2458 die("No first-word closing quote found: %s" % view_line
)
2459 depot_side
= view_line
[1:close_quote_index
]
2460 # skip closing quote and space
2461 rhs_index
= close_quote_index
+ 1 + 1
2463 space_index
= view_line
.find(" ")
2464 if space_index
<= 0:
2465 die("No word-splitting space found: %s" % view_line
)
2466 depot_side
= view_line
[0:space_index
]
2467 rhs_index
= space_index
+ 1
2469 # prefix + means overlay on previous mapping
2470 if depot_side
.startswith("+"):
2471 depot_side
= depot_side
[1:]
2473 # prefix - means exclude this path, leave out of mappings
2475 if depot_side
.startswith("-"):
2477 depot_side
= depot_side
[1:]
2480 self
.mappings
.append(depot_side
)
2482 def convert_client_path(self
, clientFile
):
2483 # chop off //client/ part to make it relative
2484 if not clientFile
.startswith(self
.client_prefix
):
2485 die("No prefix '%s' on clientFile '%s'" %
2486 (self
.client_prefix
, clientFile
))
2487 return clientFile
[len(self
.client_prefix
):]
2489 def update_client_spec_path_cache(self
, files
):
2490 """ Caching file paths by "p4 where" batch query """
2492 # List depot file paths exclude that already cached
2493 fileArgs
= [f
['path'] for f
in files
if f
['path'] not in self
.client_spec_path_cache
]
2495 if len(fileArgs
) == 0:
2496 return # All files in cache
2498 where_result
= p4CmdList(["-x", "-", "where"], stdin
=fileArgs
)
2499 for res
in where_result
:
2500 if "code" in res
and res
["code"] == "error":
2501 # assume error is "... file(s) not in client view"
2503 if "clientFile" not in res
:
2504 die("No clientFile in 'p4 where' output")
2506 # it will list all of them, but only one not unmap-ped
2508 if gitConfigBool("core.ignorecase"):
2509 res
['depotFile'] = res
['depotFile'].lower()
2510 self
.client_spec_path_cache
[res
['depotFile']] = self
.convert_client_path(res
["clientFile"])
2512 # not found files or unmap files set to ""
2513 for depotFile
in fileArgs
:
2514 if gitConfigBool("core.ignorecase"):
2515 depotFile
= depotFile
.lower()
2516 if depotFile
not in self
.client_spec_path_cache
:
2517 self
.client_spec_path_cache
[depotFile
] = ""
2519 def map_in_client(self
, depot_path
):
2520 """Return the relative location in the client where this
2521 depot file should live. Returns "" if the file should
2522 not be mapped in the client."""
2524 if gitConfigBool("core.ignorecase"):
2525 depot_path
= depot_path
.lower()
2527 if depot_path
in self
.client_spec_path_cache
:
2528 return self
.client_spec_path_cache
[depot_path
]
2530 die( "Error: %s is not found in client spec path" % depot_path
)
2533 class P4Sync(Command
, P4UserMap
):
2536 Command
.__init
__(self
)
2537 P4UserMap
.__init
__(self
)
2539 optparse
.make_option("--branch", dest
="branch"),
2540 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
2541 optparse
.make_option("--changesfile", dest
="changesFile"),
2542 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
2543 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
2544 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2545 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
2546 help="Import into refs/heads/ , not refs/remotes"),
2547 optparse
.make_option("--max-changes", dest
="maxChanges",
2548 help="Maximum number of changes to import"),
2549 optparse
.make_option("--changes-block-size", dest
="changes_block_size", type="int",
2550 help="Internal block size to use when iteratively calling p4 changes"),
2551 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
2552 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2553 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
2554 help="Only sync files that are included in the Perforce Client Spec"),
2555 optparse
.make_option("-/", dest
="cloneExclude",
2556 action
="append", type="string",
2557 help="exclude depot path"),
2559 self
.description
= """Imports from Perforce into a git repository.\n
2561 //depot/my/project/ -- to import the current head
2562 //depot/my/project/@all -- to import everything
2563 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2565 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2567 self
.usage
+= " //depot/path[@revRange]"
2569 self
.createdBranches
= set()
2570 self
.committedChanges
= set()
2572 self
.detectBranches
= False
2573 self
.detectLabels
= False
2574 self
.importLabels
= False
2575 self
.changesFile
= ""
2576 self
.syncWithOrigin
= True
2577 self
.importIntoRemotes
= True
2578 self
.maxChanges
= ""
2579 self
.changes_block_size
= None
2580 self
.keepRepoPath
= False
2581 self
.depotPaths
= None
2582 self
.p4BranchesInGit
= []
2583 self
.cloneExclude
= []
2584 self
.useClientSpec
= False
2585 self
.useClientSpec_from_options
= False
2586 self
.clientSpecDirs
= None
2587 self
.tempBranches
= []
2588 self
.tempBranchLocation
= "refs/git-p4-tmp"
2589 self
.largeFileSystem
= None
2590 self
.suppress_meta_comment
= False
2592 if gitConfig('git-p4.largeFileSystem'):
2593 largeFileSystemConstructor
= globals()[gitConfig('git-p4.largeFileSystem')]
2594 self
.largeFileSystem
= largeFileSystemConstructor(
2595 lambda git_mode
, relPath
, contents
: self
.writeToGitStream(git_mode
, relPath
, contents
)
2598 if gitConfig("git-p4.syncFromOrigin") == "false":
2599 self
.syncWithOrigin
= False
2601 self
.depotPaths
= []
2602 self
.changeRange
= ""
2603 self
.previousDepotPaths
= []
2604 self
.hasOrigin
= False
2606 # map from branch depot path to parent branch
2607 self
.knownBranches
= {}
2608 self
.initialParents
= {}
2610 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
2613 # Force a checkpoint in fast-import and wait for it to finish
2614 def checkpoint(self
):
2615 self
.gitStream
.write("checkpoint\n\n")
2616 self
.gitStream
.write("progress checkpoint\n\n")
2617 out
= self
.gitOutput
.readline()
2619 print("checkpoint finished: " + out
)
2621 def extractFilesFromCommit(self
, commit
, shelved
=False, shelved_cl
= 0):
2622 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
2623 for path
in self
.cloneExclude
]
2626 while "depotFile%s" % fnum
in commit
:
2627 path
= commit
["depotFile%s" % fnum
]
2629 if [p
for p
in self
.cloneExclude
2630 if p4PathStartsWith(path
, p
)]:
2633 found
= [p
for p
in self
.depotPaths
2634 if p4PathStartsWith(path
, p
)]
2641 file["rev"] = commit
["rev%s" % fnum
]
2642 file["action"] = commit
["action%s" % fnum
]
2643 file["type"] = commit
["type%s" % fnum
]
2645 file["shelved_cl"] = int(shelved_cl
)
2650 def extractJobsFromCommit(self
, commit
):
2653 while "job%s" % jnum
in commit
:
2654 job
= commit
["job%s" % jnum
]
2659 def stripRepoPath(self
, path
, prefixes
):
2660 """When streaming files, this is called to map a p4 depot path
2661 to where it should go in git. The prefixes are either
2662 self.depotPaths, or self.branchPrefixes in the case of
2663 branch detection."""
2665 if self
.useClientSpec
:
2666 # branch detection moves files up a level (the branch name)
2667 # from what client spec interpretation gives
2668 path
= self
.clientSpecDirs
.map_in_client(path
)
2669 if self
.detectBranches
:
2670 for b
in self
.knownBranches
:
2671 if path
.startswith(b
+ "/"):
2672 path
= path
[len(b
)+1:]
2674 elif self
.keepRepoPath
:
2675 # Preserve everything in relative path name except leading
2676 # //depot/; just look at first prefix as they all should
2677 # be in the same depot.
2678 depot
= re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])
2679 if p4PathStartsWith(path
, depot
):
2680 path
= path
[len(depot
):]
2684 if p4PathStartsWith(path
, p
):
2685 path
= path
[len(p
):]
2688 path
= wildcard_decode(path
)
2691 def splitFilesIntoBranches(self
, commit
):
2692 """Look at each depotFile in the commit to figure out to what
2693 branch it belongs."""
2695 if self
.clientSpecDirs
:
2696 files
= self
.extractFilesFromCommit(commit
)
2697 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2701 while "depotFile%s" % fnum
in commit
:
2702 path
= commit
["depotFile%s" % fnum
]
2703 found
= [p
for p
in self
.depotPaths
2704 if p4PathStartsWith(path
, p
)]
2711 file["rev"] = commit
["rev%s" % fnum
]
2712 file["action"] = commit
["action%s" % fnum
]
2713 file["type"] = commit
["type%s" % fnum
]
2716 # start with the full relative path where this file would
2718 if self
.useClientSpec
:
2719 relPath
= self
.clientSpecDirs
.map_in_client(path
)
2721 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
2723 for branch
in self
.knownBranches
.keys():
2724 # add a trailing slash so that a commit into qt/4.2foo
2725 # doesn't end up in qt/4.2, e.g.
2726 if relPath
.startswith(branch
+ "/"):
2727 if branch
not in branches
:
2728 branches
[branch
] = []
2729 branches
[branch
].append(file)
2734 def writeToGitStream(self
, gitMode
, relPath
, contents
):
2735 self
.gitStream
.write('M %s inline %s\n' % (gitMode
, relPath
))
2736 self
.gitStream
.write('data %d\n' % sum(len(d
) for d
in contents
))
2738 self
.gitStream
.write(d
)
2739 self
.gitStream
.write('\n')
2741 def encodeWithUTF8(self
, path
):
2743 path
.decode('ascii')
2746 if gitConfig('git-p4.pathEncoding'):
2747 encoding
= gitConfig('git-p4.pathEncoding')
2748 path
= path
.decode(encoding
, 'replace').encode('utf8', 'replace')
2750 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding
, path
))
2753 # output one file from the P4 stream
2754 # - helper for streamP4Files
2756 def streamOneP4File(self
, file, contents
):
2757 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
2758 relPath
= self
.encodeWithUTF8(relPath
)
2760 if 'fileSize' in self
.stream_file
:
2761 size
= int(self
.stream_file
['fileSize'])
2763 size
= 0 # deleted files don't get a fileSize apparently
2764 sys
.stdout
.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath
, size
/1024/1024))
2767 (type_base
, type_mods
) = split_p4_type(file["type"])
2770 if "x" in type_mods
:
2772 if type_base
== "symlink":
2774 # p4 print on a symlink sometimes contains "target\n";
2775 # if it does, remove the newline
2776 data
= ''.join(contents
)
2778 # Some version of p4 allowed creating a symlink that pointed
2779 # to nothing. This causes p4 errors when checking out such
2780 # a change, and errors here too. Work around it by ignoring
2781 # the bad symlink; hopefully a future change fixes it.
2782 print("\nIgnoring empty symlink in %s" % file['depotFile'])
2784 elif data
[-1] == '\n':
2785 contents
= [data
[:-1]]
2789 if type_base
== "utf16":
2790 # p4 delivers different text in the python output to -G
2791 # than it does when using "print -o", or normal p4 client
2792 # operations. utf16 is converted to ascii or utf8, perhaps.
2793 # But ascii text saved as -t utf16 is completely mangled.
2794 # Invoke print -o to get the real contents.
2796 # On windows, the newlines will always be mangled by print, so put
2797 # them back too. This is not needed to the cygwin windows version,
2798 # just the native "NT" type.
2801 text
= p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2802 except Exception as e
:
2803 if 'Translation of file content failed' in str(e
):
2804 type_base
= 'binary'
2808 if p4_version_string().find('/NT') >= 0:
2809 text
= text
.replace('\r\n', '\n')
2812 if type_base
== "apple":
2813 # Apple filetype files will be streamed as a concatenation of
2814 # its appledouble header and the contents. This is useless
2815 # on both macs and non-macs. If using "print -q -o xx", it
2816 # will create "xx" with the data, and "%xx" with the header.
2817 # This is also not very useful.
2819 # Ideally, someday, this script can learn how to generate
2820 # appledouble files directly and import those to git, but
2821 # non-mac machines can never find a use for apple filetype.
2822 print("\nIgnoring apple filetype file %s" % file['depotFile'])
2825 # Note that we do not try to de-mangle keywords on utf16 files,
2826 # even though in theory somebody may want that.
2827 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
2829 regexp
= re
.compile(pattern
, re
.VERBOSE
)
2830 text
= ''.join(contents
)
2831 text
= regexp
.sub(r
'$\1$', text
)
2834 if self
.largeFileSystem
:
2835 (git_mode
, contents
) = self
.largeFileSystem
.processContent(git_mode
, relPath
, contents
)
2837 self
.writeToGitStream(git_mode
, relPath
, contents
)
2839 def streamOneP4Deletion(self
, file):
2840 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
2841 relPath
= self
.encodeWithUTF8(relPath
)
2843 sys
.stdout
.write("delete %s\n" % relPath
)
2845 self
.gitStream
.write("D %s\n" % relPath
)
2847 if self
.largeFileSystem
and self
.largeFileSystem
.isLargeFile(relPath
):
2848 self
.largeFileSystem
.removeLargeFile(relPath
)
2850 # handle another chunk of streaming data
2851 def streamP4FilesCb(self
, marshalled
):
2853 # catch p4 errors and complain
2855 if "code" in marshalled
:
2856 if marshalled
["code"] == "error":
2857 if "data" in marshalled
:
2858 err
= marshalled
["data"].rstrip()
2860 if not err
and 'fileSize' in self
.stream_file
:
2861 required_bytes
= int((4 * int(self
.stream_file
["fileSize"])) - calcDiskFree())
2862 if required_bytes
> 0:
2863 err
= 'Not enough space left on %s! Free at least %i MB.' % (
2864 os
.getcwd(), required_bytes
/1024/1024
2869 if self
.stream_have_file_info
:
2870 if "depotFile" in self
.stream_file
:
2871 f
= self
.stream_file
["depotFile"]
2872 # force a failure in fast-import, else an empty
2873 # commit will be made
2874 self
.gitStream
.write("\n")
2875 self
.gitStream
.write("die-now\n")
2876 self
.gitStream
.close()
2877 # ignore errors, but make sure it exits first
2878 self
.importProcess
.wait()
2880 die("Error from p4 print for %s: %s" % (f
, err
))
2882 die("Error from p4 print: %s" % err
)
2884 if 'depotFile' in marshalled
and self
.stream_have_file_info
:
2885 # start of a new file - output the old one first
2886 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2887 self
.stream_file
= {}
2888 self
.stream_contents
= []
2889 self
.stream_have_file_info
= False
2891 # pick up the new file information... for the
2892 # 'data' field we need to append to our array
2893 for k
in marshalled
.keys():
2895 if 'streamContentSize' not in self
.stream_file
:
2896 self
.stream_file
['streamContentSize'] = 0
2897 self
.stream_file
['streamContentSize'] += len(marshalled
['data'])
2898 self
.stream_contents
.append(marshalled
['data'])
2900 self
.stream_file
[k
] = marshalled
[k
]
2903 'streamContentSize' in self
.stream_file
and
2904 'fileSize' in self
.stream_file
and
2905 'depotFile' in self
.stream_file
):
2906 size
= int(self
.stream_file
["fileSize"])
2908 progress
= 100*self
.stream_file
['streamContentSize']/size
2909 sys
.stdout
.write('\r%s %d%% (%i MB)' % (self
.stream_file
['depotFile'], progress
, int(size
/1024/1024)))
2912 self
.stream_have_file_info
= True
2914 # Stream directly from "p4 files" into "git fast-import"
2915 def streamP4Files(self
, files
):
2921 filesForCommit
.append(f
)
2922 if f
['action'] in self
.delete_actions
:
2923 filesToDelete
.append(f
)
2925 filesToRead
.append(f
)
2928 for f
in filesToDelete
:
2929 self
.streamOneP4Deletion(f
)
2931 if len(filesToRead
) > 0:
2932 self
.stream_file
= {}
2933 self
.stream_contents
= []
2934 self
.stream_have_file_info
= False
2936 # curry self argument
2937 def streamP4FilesCbSelf(entry
):
2938 self
.streamP4FilesCb(entry
)
2941 for f
in filesToRead
:
2942 if 'shelved_cl' in f
:
2943 # Handle shelved CLs using the "p4 print file@=N" syntax to print
2945 fileArg
= '%s@=%d' % (f
['path'], f
['shelved_cl'])
2947 fileArg
= '%s#%s' % (f
['path'], f
['rev'])
2949 fileArgs
.append(fileArg
)
2951 p4CmdList(["-x", "-", "print"],
2953 cb
=streamP4FilesCbSelf
)
2956 if 'depotFile' in self
.stream_file
:
2957 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2959 def make_email(self
, userid
):
2960 if userid
in self
.users
:
2961 return self
.users
[userid
]
2963 return "%s <a@b>" % userid
2965 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
2966 """ Stream a p4 tag.
2967 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2971 print("writing tag %s for commit %s" % (labelName
, commit
))
2972 gitStream
.write("tag %s\n" % labelName
)
2973 gitStream
.write("from %s\n" % commit
)
2975 if 'Owner' in labelDetails
:
2976 owner
= labelDetails
["Owner"]
2980 # Try to use the owner of the p4 label, or failing that,
2981 # the current p4 user id.
2983 email
= self
.make_email(owner
)
2985 email
= self
.make_email(self
.p4UserId())
2986 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2988 gitStream
.write("tagger %s\n" % tagger
)
2990 print("labelDetails=",labelDetails
)
2991 if 'Description' in labelDetails
:
2992 description
= labelDetails
['Description']
2994 description
= 'Label from git p4'
2996 gitStream
.write("data %d\n" % len(description
))
2997 gitStream
.write(description
)
2998 gitStream
.write("\n")
3000 def inClientSpec(self
, path
):
3001 if not self
.clientSpecDirs
:
3003 inClientSpec
= self
.clientSpecDirs
.map_in_client(path
)
3004 if not inClientSpec
and self
.verbose
:
3005 print('Ignoring file outside of client spec: {0}'.format(path
))
3008 def hasBranchPrefix(self
, path
):
3009 if not self
.branchPrefixes
:
3011 hasPrefix
= [p
for p
in self
.branchPrefixes
3012 if p4PathStartsWith(path
, p
)]
3013 if not hasPrefix
and self
.verbose
:
3014 print('Ignoring file outside of prefix: {0}'.format(path
))
3017 def commit(self
, details
, files
, branch
, parent
= "", allow_empty
=False):
3018 epoch
= details
["time"]
3019 author
= details
["user"]
3020 jobs
= self
.extractJobsFromCommit(details
)
3023 print('commit into {0}'.format(branch
))
3025 if self
.clientSpecDirs
:
3026 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
3028 files
= [f
for f
in files
3029 if self
.inClientSpec(f
['path']) and self
.hasBranchPrefix(f
['path'])]
3031 if gitConfigBool('git-p4.keepEmptyCommits'):
3034 if not files
and not allow_empty
:
3035 print('Ignoring revision {0} as it would produce an empty commit.'
3036 .format(details
['change']))
3039 self
.gitStream
.write("commit %s\n" % branch
)
3040 self
.gitStream
.write("mark :%s\n" % details
["change"])
3041 self
.committedChanges
.add(int(details
["change"]))
3043 if author
not in self
.users
:
3044 self
.getUserMapFromPerforceServer()
3045 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
3047 self
.gitStream
.write("committer %s\n" % committer
)
3049 self
.gitStream
.write("data <<EOT\n")
3050 self
.gitStream
.write(details
["desc"])
3052 self
.gitStream
.write("\nJobs: %s" % (' '.join(jobs
)))
3054 if not self
.suppress_meta_comment
:
3055 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3056 (','.join(self
.branchPrefixes
), details
["change"]))
3057 if len(details
['options']) > 0:
3058 self
.gitStream
.write(": options = %s" % details
['options'])
3059 self
.gitStream
.write("]\n")
3061 self
.gitStream
.write("EOT\n\n")
3065 print("parent %s" % parent
)
3066 self
.gitStream
.write("from %s\n" % parent
)
3068 self
.streamP4Files(files
)
3069 self
.gitStream
.write("\n")
3071 change
= int(details
["change"])
3073 if change
in self
.labels
:
3074 label
= self
.labels
[change
]
3075 labelDetails
= label
[0]
3076 labelRevisions
= label
[1]
3078 print("Change %s is labelled %s" % (change
, labelDetails
))
3080 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
3081 for p
in self
.branchPrefixes
])
3083 if len(files
) == len(labelRevisions
):
3087 if info
["action"] in self
.delete_actions
:
3089 cleanedFiles
[info
["depotFile"]] = info
["rev"]
3091 if cleanedFiles
== labelRevisions
:
3092 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
3096 print("Tag %s does not match with change %s: files do not match."
3097 % (labelDetails
["label"], change
))
3101 print("Tag %s does not match with change %s: file count is different."
3102 % (labelDetails
["label"], change
))
3104 # Build a dictionary of changelists and labels, for "detect-labels" option.
3105 def getLabels(self
):
3108 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
3109 if len(l
) > 0 and not self
.silent
:
3110 print("Finding files belonging to labels in %s" % self
.depotPaths
)
3113 label
= output
["label"]
3117 print("Querying files for label %s" % label
)
3118 for file in p4CmdList(["files"] +
3119 ["%s...@%s" % (p
, label
)
3120 for p
in self
.depotPaths
]):
3121 revisions
[file["depotFile"]] = file["rev"]
3122 change
= int(file["change"])
3123 if change
> newestChange
:
3124 newestChange
= change
3126 self
.labels
[newestChange
] = [output
, revisions
]
3129 print("Label changes: %s" % self
.labels
.keys())
3131 # Import p4 labels as git tags. A direct mapping does not
3132 # exist, so assume that if all the files are at the same revision
3133 # then we can use that, or it's something more complicated we should
3135 def importP4Labels(self
, stream
, p4Labels
):
3137 print("import p4 labels: " + ' '.join(p4Labels
))
3139 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
3140 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
3141 if len(validLabelRegexp
) == 0:
3142 validLabelRegexp
= defaultLabelRegexp
3143 m
= re
.compile(validLabelRegexp
)
3145 for name
in p4Labels
:
3148 if not m
.match(name
):
3150 print("label %s does not match regexp %s" % (name
,validLabelRegexp
))
3153 if name
in ignoredP4Labels
:
3156 labelDetails
= p4CmdList(['label', "-o", name
])[0]
3158 # get the most recent changelist for each file in this label
3159 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
3160 for p
in self
.depotPaths
])
3162 if 'change' in change
:
3163 # find the corresponding git commit; take the oldest commit
3164 changelist
= int(change
['change'])
3165 if changelist
in self
.committedChanges
:
3166 gitCommit
= ":%d" % changelist
# use a fast-import mark
3169 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
3170 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
], ignore_error
=True)
3171 if len(gitCommit
) == 0:
3172 print("importing label %s: could not find git commit for changelist %d" % (name
, changelist
))
3175 gitCommit
= gitCommit
.strip()
3178 # Convert from p4 time format
3180 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
3182 print("Could not convert label time %s" % labelDetails
['Update'])
3185 when
= int(time
.mktime(tmwhen
))
3186 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
3188 print("p4 label %s mapped to git commit %s" % (name
, gitCommit
))
3191 print("Label %s has no changelists - possibly deleted?" % name
)
3194 # We can't import this label; don't try again as it will get very
3195 # expensive repeatedly fetching all the files for labels that will
3196 # never be imported. If the label is moved in the future, the
3197 # ignore will need to be removed manually.
3198 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
3200 def guessProjectName(self
):
3201 for p
in self
.depotPaths
:
3204 p
= p
[p
.strip().rfind("/") + 1:]
3205 if not p
.endswith("/"):
3209 def getBranchMapping(self
):
3210 lostAndFoundBranches
= set()
3212 user
= gitConfig("git-p4.branchUser")
3214 command
= "branches -u %s" % user
3216 command
= "branches"
3218 for info
in p4CmdList(command
):
3219 details
= p4Cmd(["branch", "-o", info
["branch"]])
3221 while "View%s" % viewIdx
in details
:
3222 paths
= details
["View%s" % viewIdx
].split(" ")
3223 viewIdx
= viewIdx
+ 1
3224 # require standard //depot/foo/... //depot/bar/... mapping
3225 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
3228 destination
= paths
[1]
3230 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
3231 source
= source
[len(self
.depotPaths
[0]):-4]
3232 destination
= destination
[len(self
.depotPaths
[0]):-4]
3234 if destination
in self
.knownBranches
:
3236 print("p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
))
3237 print("but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
))
3240 self
.knownBranches
[destination
] = source
3242 lostAndFoundBranches
.discard(destination
)
3244 if source
not in self
.knownBranches
:
3245 lostAndFoundBranches
.add(source
)
3247 # Perforce does not strictly require branches to be defined, so we also
3248 # check git config for a branch list.
3250 # Example of branch definition in git config file:
3252 # branchList=main:branchA
3253 # branchList=main:branchB
3254 # branchList=branchA:branchC
3255 configBranches
= gitConfigList("git-p4.branchList")
3256 for branch
in configBranches
:
3258 (source
, destination
) = branch
.split(":")
3259 self
.knownBranches
[destination
] = source
3261 lostAndFoundBranches
.discard(destination
)
3263 if source
not in self
.knownBranches
:
3264 lostAndFoundBranches
.add(source
)
3267 for branch
in lostAndFoundBranches
:
3268 self
.knownBranches
[branch
] = branch
3270 def getBranchMappingFromGitBranches(self
):
3271 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3272 for branch
in branches
.keys():
3273 if branch
== "master":
3276 branch
= branch
[len(self
.projectName
):]
3277 self
.knownBranches
[branch
] = branch
3279 def updateOptionDict(self
, d
):
3281 if self
.keepRepoPath
:
3282 option_keys
['keepRepoPath'] = 1
3284 d
["options"] = ' '.join(sorted(option_keys
.keys()))
3286 def readOptions(self
, d
):
3287 self
.keepRepoPath
= ('options' in d
3288 and ('keepRepoPath' in d
['options']))
3290 def gitRefForBranch(self
, branch
):
3291 if branch
== "main":
3292 return self
.refPrefix
+ "master"
3294 if len(branch
) <= 0:
3297 return self
.refPrefix
+ self
.projectName
+ branch
3299 def gitCommitByP4Change(self
, ref
, change
):
3301 print("looking in ref " + ref
+ " for change %s using bisect..." % change
)
3304 latestCommit
= parseRevision(ref
)
3308 print("trying: earliest %s latest %s" % (earliestCommit
, latestCommit
))
3309 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
3314 log
= extractLogMessageFromGitCommit(next
)
3315 settings
= extractSettingsGitLog(log
)
3316 currentChange
= int(settings
['change'])
3318 print("current change %s" % currentChange
)
3320 if currentChange
== change
:
3322 print("found %s" % next
)
3325 if currentChange
< change
:
3326 earliestCommit
= "^%s" % next
3328 latestCommit
= "%s" % next
3332 def importNewBranch(self
, branch
, maxChange
):
3333 # make fast-import flush all changes to disk and update the refs using the checkpoint
3334 # command so that we can try to find the branch parent in the git history
3335 self
.gitStream
.write("checkpoint\n\n");
3336 self
.gitStream
.flush();
3337 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3338 range = "@1,%s" % maxChange
3339 #print "prefix" + branchPrefix
3340 changes
= p4ChangesForPaths([branchPrefix
], range, self
.changes_block_size
)
3341 if len(changes
) <= 0:
3343 firstChange
= changes
[0]
3344 #print "first change in branch: %s" % firstChange
3345 sourceBranch
= self
.knownBranches
[branch
]
3346 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
3347 sourceRef
= self
.gitRefForBranch(sourceBranch
)
3348 #print "source " + sourceBranch
3350 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
3351 #print "branch parent: %s" % branchParentChange
3352 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
3353 if len(gitParent
) > 0:
3354 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
3355 #print "parent git commit: %s" % gitParent
3357 self
.importChanges(changes
)
3360 def searchParent(self
, parent
, branch
, target
):
3362 for blob
in read_pipe_lines(["git", "rev-list", "--reverse",
3363 "--no-merges", parent
]):
3365 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
3368 print("Found parent of %s in commit %s" % (branch
, blob
))
3375 def importChanges(self
, changes
, origin_revision
=0):
3377 for change
in changes
:
3378 description
= p4_describe(change
)
3379 self
.updateOptionDict(description
)
3382 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
3387 if self
.detectBranches
:
3388 branches
= self
.splitFilesIntoBranches(description
)
3389 for branch
in branches
.keys():
3391 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3392 self
.branchPrefixes
= [ branchPrefix
]
3396 filesForCommit
= branches
[branch
]
3399 print("branch is %s" % branch
)
3401 self
.updatedBranches
.add(branch
)
3403 if branch
not in self
.createdBranches
:
3404 self
.createdBranches
.add(branch
)
3405 parent
= self
.knownBranches
[branch
]
3406 if parent
== branch
:
3409 fullBranch
= self
.projectName
+ branch
3410 if fullBranch
not in self
.p4BranchesInGit
:
3412 print("\n Importing new branch %s" % fullBranch
);
3413 if self
.importNewBranch(branch
, change
- 1):
3415 self
.p4BranchesInGit
.append(fullBranch
)
3417 print("\n Resuming with change %s" % change
);
3420 print("parent determined through known branches: %s" % parent
)
3422 branch
= self
.gitRefForBranch(branch
)
3423 parent
= self
.gitRefForBranch(parent
)
3426 print("looking for initial parent for %s; current parent is %s" % (branch
, parent
))
3428 if len(parent
) == 0 and branch
in self
.initialParents
:
3429 parent
= self
.initialParents
[branch
]
3430 del self
.initialParents
[branch
]
3434 tempBranch
= "%s/%d" % (self
.tempBranchLocation
, change
)
3436 print("Creating temporary branch: " + tempBranch
)
3437 self
.commit(description
, filesForCommit
, tempBranch
)
3438 self
.tempBranches
.append(tempBranch
)
3440 blob
= self
.searchParent(parent
, branch
, tempBranch
)
3442 self
.commit(description
, filesForCommit
, branch
, blob
)
3445 print("Parent of %s not found. Committing into head of %s" % (branch
, parent
))
3446 self
.commit(description
, filesForCommit
, branch
, parent
)
3448 files
= self
.extractFilesFromCommit(description
)
3449 self
.commit(description
, files
, self
.branch
,
3451 # only needed once, to connect to the previous commit
3452 self
.initialParent
= ""
3454 print(self
.gitError
.read())
3457 def sync_origin_only(self
):
3458 if self
.syncWithOrigin
:
3459 self
.hasOrigin
= originP4BranchesExist()
3462 print('Syncing with origin first, using "git fetch origin"')
3463 system("git fetch origin")
3465 def importHeadRevision(self
, revision
):
3466 print("Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
))
3469 details
["user"] = "git perforce import user"
3470 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
3471 % (' '.join(self
.depotPaths
), revision
))
3472 details
["change"] = revision
3476 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
3478 for info
in p4CmdList(["files"] + fileArgs
):
3480 if 'code' in info
and info
['code'] == 'error':
3481 sys
.stderr
.write("p4 returned an error: %s\n"
3483 if info
['data'].find("must refer to client") >= 0:
3484 sys
.stderr
.write("This particular p4 error is misleading.\n")
3485 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
3486 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
3488 if 'p4ExitCode' in info
:
3489 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
3493 change
= int(info
["change"])
3494 if change
> newestRevision
:
3495 newestRevision
= change
3497 if info
["action"] in self
.delete_actions
:
3498 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3499 #fileCnt = fileCnt + 1
3502 for prop
in ["depotFile", "rev", "action", "type" ]:
3503 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
3505 fileCnt
= fileCnt
+ 1
3507 details
["change"] = newestRevision
3509 # Use time from top-most change so that all git p4 clones of
3510 # the same p4 repo have the same commit SHA1s.
3511 res
= p4_describe(newestRevision
)
3512 details
["time"] = res
["time"]
3514 self
.updateOptionDict(details
)
3516 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
3518 print("IO error with git fast-import. Is your git version recent enough?")
3519 print(self
.gitError
.read())
3521 def openStreams(self
):
3522 self
.importProcess
= subprocess
.Popen(["git", "fast-import"],
3523 stdin
=subprocess
.PIPE
,
3524 stdout
=subprocess
.PIPE
,
3525 stderr
=subprocess
.PIPE
);
3526 self
.gitOutput
= self
.importProcess
.stdout
3527 self
.gitStream
= self
.importProcess
.stdin
3528 self
.gitError
= self
.importProcess
.stderr
3530 def closeStreams(self
):
3531 self
.gitStream
.close()
3532 if self
.importProcess
.wait() != 0:
3533 die("fast-import failed: %s" % self
.gitError
.read())
3534 self
.gitOutput
.close()
3535 self
.gitError
.close()
3537 def run(self
, args
):
3538 if self
.importIntoRemotes
:
3539 self
.refPrefix
= "refs/remotes/p4/"
3541 self
.refPrefix
= "refs/heads/p4/"
3543 self
.sync_origin_only()
3545 branch_arg_given
= bool(self
.branch
)
3546 if len(self
.branch
) == 0:
3547 self
.branch
= self
.refPrefix
+ "master"
3548 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
3549 system("git update-ref %s refs/heads/p4" % self
.branch
)
3550 system("git branch -D p4")
3552 # accept either the command-line option, or the configuration variable
3553 if self
.useClientSpec
:
3554 # will use this after clone to set the variable
3555 self
.useClientSpec_from_options
= True
3557 if gitConfigBool("git-p4.useclientspec"):
3558 self
.useClientSpec
= True
3559 if self
.useClientSpec
:
3560 self
.clientSpecDirs
= getClientSpec()
3562 # TODO: should always look at previous commits,
3563 # merge with previous imports, if possible.
3566 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
3568 # branches holds mapping from branch name to sha1
3569 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3571 # restrict to just this one, disabling detect-branches
3572 if branch_arg_given
:
3573 short
= self
.branch
.split("/")[-1]
3574 if short
in branches
:
3575 self
.p4BranchesInGit
= [ short
]
3577 self
.p4BranchesInGit
= branches
.keys()
3579 if len(self
.p4BranchesInGit
) > 1:
3581 print("Importing from/into multiple branches")
3582 self
.detectBranches
= True
3583 for branch
in branches
.keys():
3584 self
.initialParents
[self
.refPrefix
+ branch
] = \
3588 print("branches: %s" % self
.p4BranchesInGit
)
3591 for branch
in self
.p4BranchesInGit
:
3592 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
3594 settings
= extractSettingsGitLog(logMsg
)
3596 self
.readOptions(settings
)
3597 if ('depot-paths' in settings
3598 and 'change' in settings
):
3599 change
= int(settings
['change']) + 1
3600 p4Change
= max(p4Change
, change
)
3602 depotPaths
= sorted(settings
['depot-paths'])
3603 if self
.previousDepotPaths
== []:
3604 self
.previousDepotPaths
= depotPaths
3607 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
3608 prev_list
= prev
.split("/")
3609 cur_list
= cur
.split("/")
3610 for i
in range(0, min(len(cur_list
), len(prev_list
))):
3611 if cur_list
[i
] != prev_list
[i
]:
3615 paths
.append ("/".join(cur_list
[:i
+ 1]))
3617 self
.previousDepotPaths
= paths
3620 self
.depotPaths
= sorted(self
.previousDepotPaths
)
3621 self
.changeRange
= "@%s,#head" % p4Change
3622 if not self
.silent
and not self
.detectBranches
:
3623 print("Performing incremental import into %s git branch" % self
.branch
)
3625 # accept multiple ref name abbreviations:
3626 # refs/foo/bar/branch -> use it exactly
3627 # p4/branch -> prepend refs/remotes/ or refs/heads/
3628 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3629 if not self
.branch
.startswith("refs/"):
3630 if self
.importIntoRemotes
:
3631 prepend
= "refs/remotes/"
3633 prepend
= "refs/heads/"
3634 if not self
.branch
.startswith("p4/"):
3636 self
.branch
= prepend
+ self
.branch
3638 if len(args
) == 0 and self
.depotPaths
:
3640 print("Depot paths: %s" % ' '.join(self
.depotPaths
))
3642 if self
.depotPaths
and self
.depotPaths
!= args
:
3643 print("previous import used depot path %s and now %s was specified. "
3644 "This doesn't work!" % (' '.join (self
.depotPaths
),
3648 self
.depotPaths
= sorted(args
)
3653 # Make sure no revision specifiers are used when --changesfile
3655 bad_changesfile
= False
3656 if len(self
.changesFile
) > 0:
3657 for p
in self
.depotPaths
:
3658 if p
.find("@") >= 0 or p
.find("#") >= 0:
3659 bad_changesfile
= True
3662 die("Option --changesfile is incompatible with revision specifiers")
3665 for p
in self
.depotPaths
:
3666 if p
.find("@") != -1:
3667 atIdx
= p
.index("@")
3668 self
.changeRange
= p
[atIdx
:]
3669 if self
.changeRange
== "@all":
3670 self
.changeRange
= ""
3671 elif ',' not in self
.changeRange
:
3672 revision
= self
.changeRange
3673 self
.changeRange
= ""
3675 elif p
.find("#") != -1:
3676 hashIdx
= p
.index("#")
3677 revision
= p
[hashIdx
:]
3679 elif self
.previousDepotPaths
== []:
3680 # pay attention to changesfile, if given, else import
3681 # the entire p4 tree at the head revision
3682 if len(self
.changesFile
) == 0:
3685 p
= re
.sub ("\.\.\.$", "", p
)
3686 if not p
.endswith("/"):
3691 self
.depotPaths
= newPaths
3693 # --detect-branches may change this for each branch
3694 self
.branchPrefixes
= self
.depotPaths
3696 self
.loadUserMapFromCache()
3698 if self
.detectLabels
:
3701 if self
.detectBranches
:
3702 ## FIXME - what's a P4 projectName ?
3703 self
.projectName
= self
.guessProjectName()
3706 self
.getBranchMappingFromGitBranches()
3708 self
.getBranchMapping()
3710 print("p4-git branches: %s" % self
.p4BranchesInGit
)
3711 print("initial parents: %s" % self
.initialParents
)
3712 for b
in self
.p4BranchesInGit
:
3716 b
= b
[len(self
.projectName
):]
3717 self
.createdBranches
.add(b
)
3722 self
.importHeadRevision(revision
)
3726 if len(self
.changesFile
) > 0:
3727 output
= open(self
.changesFile
).readlines()
3730 changeSet
.add(int(line
))
3732 for change
in changeSet
:
3733 changes
.append(change
)
3737 # catch "git p4 sync" with no new branches, in a repo that
3738 # does not have any existing p4 branches
3740 if not self
.p4BranchesInGit
:
3741 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3743 # The default branch is master, unless --branch is used to
3744 # specify something else. Make sure it exists, or complain
3745 # nicely about how to use --branch.
3746 if not self
.detectBranches
:
3747 if not branch_exists(self
.branch
):
3748 if branch_arg_given
:
3749 die("Error: branch %s does not exist." % self
.branch
)
3751 die("Error: no branch %s; perhaps specify one with --branch." %
3755 print("Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
3757 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
, self
.changes_block_size
)
3759 if len(self
.maxChanges
) > 0:
3760 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
3762 if len(changes
) == 0:
3764 print("No changes to import!")
3766 if not self
.silent
and not self
.detectBranches
:
3767 print("Import destination: %s" % self
.branch
)
3769 self
.updatedBranches
= set()
3771 if not self
.detectBranches
:
3773 # start a new branch
3774 self
.initialParent
= ""
3776 # build on a previous revision
3777 self
.initialParent
= parseRevision(self
.branch
)
3779 self
.importChanges(changes
)
3783 if len(self
.updatedBranches
) > 0:
3784 sys
.stdout
.write("Updated branches: ")
3785 for b
in self
.updatedBranches
:
3786 sys
.stdout
.write("%s " % b
)
3787 sys
.stdout
.write("\n")
3789 if gitConfigBool("git-p4.importLabels"):
3790 self
.importLabels
= True
3792 if self
.importLabels
:
3793 p4Labels
= getP4Labels(self
.depotPaths
)
3794 gitTags
= getGitTags()
3796 missingP4Labels
= p4Labels
- gitTags
3797 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
3801 # Cleanup temporary branches created during import
3802 if self
.tempBranches
!= []:
3803 for branch
in self
.tempBranches
:
3804 read_pipe("git update-ref -d %s" % branch
)
3805 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
3807 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3808 # a convenient shortcut refname "p4".
3809 if self
.importIntoRemotes
:
3810 head_ref
= self
.refPrefix
+ "HEAD"
3811 if not gitBranchExists(head_ref
) and gitBranchExists(self
.branch
):
3812 system(["git", "symbolic-ref", head_ref
, self
.branch
])
3816 class P4Rebase(Command
):
3818 Command
.__init
__(self
)
3820 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
3822 self
.importLabels
= False
3823 self
.description
= ("Fetches the latest revision from perforce and "
3824 + "rebases the current work (branch) against it")
3826 def run(self
, args
):
3828 sync
.importLabels
= self
.importLabels
3831 return self
.rebase()
3834 if os
.system("git update-index --refresh") != 0:
3835 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3836 if len(read_pipe("git diff-index HEAD --")) > 0:
3837 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3839 [upstream
, settings
] = findUpstreamBranchPoint()
3840 if len(upstream
) == 0:
3841 die("Cannot find upstream branchpoint for rebase")
3843 # the branchpoint may be p4/foo~3, so strip off the parent
3844 upstream
= re
.sub("~[0-9]+$", "", upstream
)
3846 print("Rebasing the current branch onto %s" % upstream
)
3847 oldHead
= read_pipe("git rev-parse HEAD").strip()
3848 system("git rebase %s" % upstream
)
3849 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead
)
3852 class P4Clone(P4Sync
):
3854 P4Sync
.__init
__(self
)
3855 self
.description
= "Creates a new git repository and imports from Perforce into it"
3856 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
3858 optparse
.make_option("--destination", dest
="cloneDestination",
3859 action
='store', default
=None,
3860 help="where to leave result of the clone"),
3861 optparse
.make_option("--bare", dest
="cloneBare",
3862 action
="store_true", default
=False),
3864 self
.cloneDestination
= None
3865 self
.needsGit
= False
3866 self
.cloneBare
= False
3868 def defaultDestination(self
, args
):
3869 ## TODO: use common prefix of args?
3871 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
3872 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
3873 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
3874 depotDir
= re
.sub(r
"/$", "", depotDir
)
3875 return os
.path
.split(depotDir
)[1]
3877 def run(self
, args
):
3881 if self
.keepRepoPath
and not self
.cloneDestination
:
3882 sys
.stderr
.write("Must specify destination for --keep-path\n")
3887 if not self
.cloneDestination
and len(depotPaths
) > 1:
3888 self
.cloneDestination
= depotPaths
[-1]
3889 depotPaths
= depotPaths
[:-1]
3891 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
3892 for p
in depotPaths
:
3893 if not p
.startswith("//"):
3894 sys
.stderr
.write('Depot paths must start with "//": %s\n' % p
)
3897 if not self
.cloneDestination
:
3898 self
.cloneDestination
= self
.defaultDestination(args
)
3900 print("Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
))
3902 if not os
.path
.exists(self
.cloneDestination
):
3903 os
.makedirs(self
.cloneDestination
)
3904 chdir(self
.cloneDestination
)
3906 init_cmd
= [ "git", "init" ]
3908 init_cmd
.append("--bare")
3909 retcode
= subprocess
.call(init_cmd
)
3911 raise CalledProcessError(retcode
, init_cmd
)
3913 if not P4Sync
.run(self
, depotPaths
):
3916 # create a master branch and check out a work tree
3917 if gitBranchExists(self
.branch
):
3918 system([ "git", "branch", "master", self
.branch
])
3919 if not self
.cloneBare
:
3920 system([ "git", "checkout", "-f" ])
3922 print('Not checking out any branch, use ' \
3923 '"git checkout -q -b master <branch>"')
3925 # auto-set this variable if invoked with --use-client-spec
3926 if self
.useClientSpec_from_options
:
3927 system("git config --bool git-p4.useclientspec true")
3931 class P4Unshelve(Command
):
3933 Command
.__init
__(self
)
3935 self
.origin
= "HEAD"
3936 self
.description
= "Unshelve a P4 changelist into a git commit"
3937 self
.usage
= "usage: %prog [options] changelist"
3939 optparse
.make_option("--origin", dest
="origin",
3940 help="Use this base revision instead of the default (%s)" % self
.origin
),
3942 self
.verbose
= False
3943 self
.noCommit
= False
3944 self
.destbranch
= "refs/remotes/p4-unshelved"
3946 def renameBranch(self
, branch_name
):
3947 """ Rename the existing branch to branch_name.N
3951 for i
in range(0,1000):
3952 backup_branch_name
= "{0}.{1}".format(branch_name
, i
)
3953 if not gitBranchExists(backup_branch_name
):
3954 gitUpdateRef(backup_branch_name
, branch_name
) # copy ref to backup
3955 gitDeleteRef(branch_name
)
3957 print("renamed old unshelve branch to {0}".format(backup_branch_name
))
3961 sys
.exit("gave up trying to rename existing branch {0}".format(sync
.branch
))
3963 def findLastP4Revision(self
, starting_point
):
3964 """ Look back from starting_point for the first commit created by git-p4
3965 to find the P4 commit we are based on, and the depot-paths.
3968 for parent
in (range(65535)):
3969 log
= extractLogMessageFromGitCommit("{0}^{1}".format(starting_point
, parent
))
3970 settings
= extractSettingsGitLog(log
)
3971 if 'change' in settings
:
3974 sys
.exit("could not find git-p4 commits in {0}".format(self
.origin
))
3976 def createShelveParent(self
, change
, branch_name
, sync
, origin
):
3977 """ Create a commit matching the parent of the shelved changelist 'change'
3979 parent_description
= p4_describe(change
, shelved
=True)
3980 parent_description
['desc'] = 'parent for shelved changelist {}\n'.format(change
)
3981 files
= sync
.extractFilesFromCommit(parent_description
, shelved
=False, shelved_cl
=change
)
3985 # if it was added in the shelved changelist, it won't exist in the parent
3986 if f
['action'] in self
.add_actions
:
3989 # if it was deleted in the shelved changelist it must not be deleted
3990 # in the parent - we might even need to create it if the origin branch
3992 if f
['action'] in self
.delete_actions
:
3995 parent_files
.append(f
)
3997 sync
.commit(parent_description
, parent_files
, branch_name
,
3998 parent
=origin
, allow_empty
=True)
3999 print("created parent commit for {0} based on {1} in {2}".format(
4000 change
, self
.origin
, branch_name
))
4002 def run(self
, args
):
4006 if not gitBranchExists(self
.origin
):
4007 sys
.exit("origin branch {0} does not exist".format(self
.origin
))
4012 # only one change at a time
4015 # if the target branch already exists, rename it
4016 branch_name
= "{0}/{1}".format(self
.destbranch
, change
)
4017 if gitBranchExists(branch_name
):
4018 self
.renameBranch(branch_name
)
4019 sync
.branch
= branch_name
4021 sync
.verbose
= self
.verbose
4022 sync
.suppress_meta_comment
= True
4024 settings
= self
.findLastP4Revision(self
.origin
)
4025 sync
.depotPaths
= settings
['depot-paths']
4026 sync
.branchPrefixes
= sync
.depotPaths
4029 sync
.loadUserMapFromCache()
4032 # create a commit for the parent of the shelved changelist
4033 self
.createShelveParent(change
, branch_name
, sync
, self
.origin
)
4035 # create the commit for the shelved changelist itself
4036 description
= p4_describe(change
, True)
4037 files
= sync
.extractFilesFromCommit(description
, True, change
)
4039 sync
.commit(description
, files
, branch_name
, "")
4042 print("unshelved changelist {0} into {1}".format(change
, branch_name
))
4046 class P4Branches(Command
):
4048 Command
.__init
__(self
)
4050 self
.description
= ("Shows the git branches that hold imports and their "
4051 + "corresponding perforce depot paths")
4052 self
.verbose
= False
4054 def run(self
, args
):
4055 if originP4BranchesExist():
4056 createOrUpdateBranchesFromOrigin()
4058 cmdline
= "git rev-parse --symbolic "
4059 cmdline
+= " --remotes"
4061 for line
in read_pipe_lines(cmdline
):
4064 if not line
.startswith('p4/') or line
== "p4/HEAD":
4068 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
4069 settings
= extractSettingsGitLog(log
)
4071 print("%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"]))
4074 class HelpFormatter(optparse
.IndentedHelpFormatter
):
4076 optparse
.IndentedHelpFormatter
.__init
__(self
)
4078 def format_description(self
, description
):
4080 return description
+ "\n"
4084 def printUsage(commands
):
4085 print("usage: %s <command> [options]" % sys
.argv
[0])
4087 print("valid commands: %s" % ", ".join(commands
))
4089 print("Try %s <command> --help for command specific help." % sys
.argv
[0])
4094 "submit" : P4Submit
,
4095 "commit" : P4Submit
,
4097 "rebase" : P4Rebase
,
4099 "rollback" : P4RollBack
,
4100 "branches" : P4Branches
,
4101 "unshelve" : P4Unshelve
,
4106 if len(sys
.argv
[1:]) == 0:
4107 printUsage(commands
.keys())
4110 cmdName
= sys
.argv
[1]
4112 klass
= commands
[cmdName
]
4115 print("unknown command %s" % cmdName
)
4117 printUsage(commands
.keys())
4120 options
= cmd
.options
4121 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
4125 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
4127 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
4129 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
4131 description
= cmd
.description
,
4132 formatter
= HelpFormatter())
4134 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
4136 verbose
= cmd
.verbose
4138 if cmd
.gitdir
== None:
4139 cmd
.gitdir
= os
.path
.abspath(".git")
4140 if not isValidGitDir(cmd
.gitdir
):
4141 # "rev-parse --git-dir" without arguments will try $PWD/.git
4142 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
4143 if os
.path
.exists(cmd
.gitdir
):
4144 cdup
= read_pipe("git rev-parse --show-cdup").strip()
4148 if not isValidGitDir(cmd
.gitdir
):
4149 if isValidGitDir(cmd
.gitdir
+ "/.git"):
4150 cmd
.gitdir
+= "/.git"
4152 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
4154 # so git commands invoked from the P4 workspace will succeed
4155 os
.environ
["GIT_DIR"] = cmd
.gitdir
4157 if not cmd
.run(args
):
4162 if __name__
== '__main__':