3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 # pylint: disable=invalid-name,missing-docstring,too-many-arguments,broad-except
11 # pylint: disable=no-self-use,wrong-import-position,consider-iterating-dictionary
12 # pylint: disable=wrong-import-order,unused-import,too-few-public-methods
13 # pylint: disable=too-many-lines,ungrouped-imports,fixme,too-many-locals
14 # pylint: disable=line-too-long,bad-whitespace,superfluous-parens
15 # pylint: disable=too-many-statements,too-many-instance-attributes
16 # pylint: disable=too-many-branches,too-many-nested-blocks
19 if sys
.version_info
.major
< 3 and sys
.version_info
.minor
< 7:
20 sys
.stderr
.write("git-p4: requires Python 2.7 or later.\n")
39 # On python2.7 where raw_input() and input() are both availble,
40 # we want raw_input's semantics, but aliased to input for python3
42 # support basestring in python3
44 if raw_input and input:
51 # Only labels/tags matching this will be imported/exported
52 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
54 # The block size is reduced automatically if required
55 defaultBlockSize
= 1<<20
57 p4_access_checked
= False
59 def p4_build_cmd(cmd
):
60 """Build a suitable p4 command line.
62 This consolidates building and returning a p4 command line into one
63 location. It means that hooking into the environment, or other configuration
64 can be done more easily.
68 user
= gitConfig("git-p4.user")
70 real_cmd
+= ["-u",user
]
72 password
= gitConfig("git-p4.password")
74 real_cmd
+= ["-P", password
]
76 port
= gitConfig("git-p4.port")
78 real_cmd
+= ["-p", port
]
80 host
= gitConfig("git-p4.host")
82 real_cmd
+= ["-H", host
]
84 client
= gitConfig("git-p4.client")
86 real_cmd
+= ["-c", client
]
88 retries
= gitConfigInt("git-p4.retries")
90 # Perform 3 retries by default
93 # Provide a way to not pass this option by setting git-p4.retries to 0
94 real_cmd
+= ["-r", str(retries
)]
96 if not isinstance(cmd
, list):
97 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
101 # now check that we can actually talk to the server
102 global p4_access_checked
103 if not p4_access_checked
:
104 p4_access_checked
= True # suppress access checks in p4_check_access itself
110 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
111 This won't automatically add ".git" to a directory.
113 d
= read_pipe(["git", "--git-dir", path
, "rev-parse", "--git-dir"], True).strip()
114 if not d
or len(d
) == 0:
119 def chdir(path
, is_client_path
=False):
120 """Do chdir to the given path, and set the PWD environment
121 variable for use by P4. It does not look at getcwd() output.
122 Since we're not using the shell, it is necessary to set the
123 PWD environment variable explicitly.
125 Normally, expand the path to force it to be absolute. This
126 addresses the use of relative path names inside P4 settings,
127 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
128 as given; it looks for .p4config using PWD.
130 If is_client_path, the path was handed to us directly by p4,
131 and may be a symbolic link. Do not call os.getcwd() in this
132 case, because it will cause p4 to think that PWD is not inside
137 if not is_client_path
:
139 os
.environ
['PWD'] = path
142 """Return free space in bytes on the disk of the given dirname."""
143 if platform
.system() == 'Windows':
144 free_bytes
= ctypes
.c_ulonglong(0)
145 ctypes
.windll
.kernel32
.GetDiskFreeSpaceExW(ctypes
.c_wchar_p(os
.getcwd()), None, None, ctypes
.pointer(free_bytes
))
146 return free_bytes
.value
148 st
= os
.statvfs(os
.getcwd())
149 return st
.f_bavail
* st
.f_frsize
152 """ Terminate execution. Make sure that any running child processes have been wait()ed for before
158 sys
.stderr
.write(msg
+ "\n")
161 def prompt(prompt_text
):
162 """ Prompt the user to choose one of the choices
164 Choices are identified in the prompt_text by square brackets around
165 a single letter option.
167 choices
= set(m
.group(1) for m
in re
.finditer(r
"\[(.)\]", prompt_text
))
170 sys
.stdout
.write(prompt_text
)
172 response
=sys
.stdin
.readline().strip().lower()
175 response
= response
[0]
176 if response
in choices
:
179 # We need different encoding/decoding strategies for text data being passed
180 # around in pipes depending on python version
182 # For python3, always encode and decode as appropriate
183 def decode_text_stream(s
):
184 return s
.decode() if isinstance(s
, bytes
) else s
185 def encode_text_stream(s
):
186 return s
.encode() if isinstance(s
, str) else s
188 # For python2.7, pass read strings as-is, but also allow writing unicode
189 def decode_text_stream(s
):
191 def encode_text_stream(s
):
192 return s
.encode('utf_8') if isinstance(s
, unicode) else s
194 def decode_path(path
):
195 """Decode a given string (bytes or otherwise) using configured path encoding options
197 encoding
= gitConfig('git-p4.pathEncoding') or 'utf_8'
199 return path
.decode(encoding
, errors
='replace') if isinstance(path
, bytes
) else path
204 path
= path
.decode(encoding
, errors
='replace')
206 print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding
, path
))
209 def run_git_hook(cmd
, param
=[]):
210 """Execute a hook if the hook exists."""
212 sys
.stderr
.write("Looking for hook: %s\n" % cmd
)
215 hooks_path
= gitConfig("core.hooksPath")
216 if len(hooks_path
) <= 0:
217 hooks_path
= os
.path
.join(os
.environ
["GIT_DIR"], "hooks")
219 if not isinstance(param
, list):
222 # resolve hook file name, OS depdenent
223 hook_file
= os
.path
.join(hooks_path
, cmd
)
224 if platform
.system() == 'Windows':
225 if not os
.path
.isfile(hook_file
):
226 # look for the file with an extension
227 files
= glob
.glob(hook_file
+ ".*")
231 hook_file
= files
.pop()
232 while hook_file
.upper().endswith(".SAMPLE"):
233 # The file is a sample hook. We don't want it
235 hook_file
= files
.pop()
239 if not os
.path
.isfile(hook_file
) or not os
.access(hook_file
, os
.X_OK
):
242 return run_hook_command(hook_file
, param
) == 0
244 def run_hook_command(cmd
, param
):
245 """Executes a git hook command
246 cmd = the command line file to be executed. This can be
247 a file that is run by OS association.
249 param = a list of parameters to pass to the cmd command
251 On windows, the extension is checked to see if it should
252 be run with the Git for Windows Bash shell. If there
253 is no file extension, the file is deemed a bash shell
254 and will be handed off to sh.exe. Otherwise, Windows
255 will be called with the shell to handle the file assocation.
257 For non Windows operating systems, the file is called
262 if platform
.system() == 'Windows':
263 (root
,ext
) = os
.path
.splitext(cmd
)
265 exe_path
= os
.environ
.get("EXEPATH")
269 exe_path
= os
.path
.join(exe_path
, "bin")
270 cli
= [os
.path
.join(exe_path
, "SH.EXE")] + cli
273 return subprocess
.call(cli
, shell
=use_shell
)
276 def write_pipe(c
, stdin
):
278 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
280 expand
= not isinstance(c
, list)
281 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
283 val
= pipe
.write(stdin
)
286 die('Command failed: %s' % str(c
))
290 def p4_write_pipe(c
, stdin
):
291 real_cmd
= p4_build_cmd(c
)
292 if bytes
is not str and isinstance(stdin
, str):
293 stdin
= encode_text_stream(stdin
)
294 return write_pipe(real_cmd
, stdin
)
296 def read_pipe_full(c
):
297 """ Read output from command. Returns a tuple
298 of the return status, stdout text and stderr
302 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
304 expand
= not isinstance(c
, list)
305 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, shell
=expand
)
306 (out
, err
) = p
.communicate()
307 return (p
.returncode
, out
, decode_text_stream(err
))
309 def read_pipe(c
, ignore_error
=False, raw
=False):
310 """ Read output from command. Returns the output text on
311 success. On failure, terminates execution, unless
312 ignore_error is True, when it returns an empty string.
314 If raw is True, do not attempt to decode output text.
316 (retcode
, out
, err
) = read_pipe_full(c
)
321 die('Command failed: %s\nError: %s' % (str(c
), err
))
323 out
= decode_text_stream(out
)
326 def read_pipe_text(c
):
327 """ Read output from a command with trailing whitespace stripped.
328 On error, returns None.
330 (retcode
, out
, err
) = read_pipe_full(c
)
334 return decode_text_stream(out
).rstrip()
336 def p4_read_pipe(c
, ignore_error
=False, raw
=False):
337 real_cmd
= p4_build_cmd(c
)
338 return read_pipe(real_cmd
, ignore_error
, raw
=raw
)
340 def read_pipe_lines(c
):
342 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
344 expand
= not isinstance(c
, list)
345 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
347 val
= [decode_text_stream(line
) for line
in pipe
.readlines()]
348 if pipe
.close() or p
.wait():
349 die('Command failed: %s' % str(c
))
352 def p4_read_pipe_lines(c
):
353 """Specifically invoke p4 on the command supplied. """
354 real_cmd
= p4_build_cmd(c
)
355 return read_pipe_lines(real_cmd
)
357 def p4_has_command(cmd
):
358 """Ask p4 for help on this command. If it returns an error, the
359 command does not exist in this version of p4."""
360 real_cmd
= p4_build_cmd(["help", cmd
])
361 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
362 stderr
=subprocess
.PIPE
)
364 return p
.returncode
== 0
366 def p4_has_move_command():
367 """See if the move command exists, that it supports -k, and that
368 it has not been administratively disabled. The arguments
369 must be correct, but the filenames do not have to exist. Use
370 ones with wildcards so even if they exist, it will fail."""
372 if not p4_has_command("move"):
374 cmd
= p4_build_cmd(["move", "-k", "@from", "@to"])
375 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
376 (out
, err
) = p
.communicate()
377 err
= decode_text_stream(err
)
378 # return code will be 1 in either case
379 if err
.find("Invalid option") >= 0:
381 if err
.find("disabled") >= 0:
383 # assume it failed because @... was invalid changelist
386 def system(cmd
, ignore_error
=False):
387 expand
= not isinstance(cmd
, list)
389 sys
.stderr
.write("executing %s\n" % str(cmd
))
390 retcode
= subprocess
.call(cmd
, shell
=expand
)
391 if retcode
and not ignore_error
:
392 raise CalledProcessError(retcode
, cmd
)
397 """Specifically invoke p4 as the system command. """
398 real_cmd
= p4_build_cmd(cmd
)
399 expand
= not isinstance(real_cmd
, list)
400 retcode
= subprocess
.call(real_cmd
, shell
=expand
)
402 raise CalledProcessError(retcode
, real_cmd
)
404 def die_bad_access(s
):
405 die("failure accessing depot: {0}".format(s
.rstrip()))
407 def p4_check_access(min_expiration
=1):
408 """ Check if we can access Perforce - account still logged in
410 results
= p4CmdList(["login", "-s"])
412 if len(results
) == 0:
413 # should never get here: always get either some results, or a p4ExitCode
414 assert("could not parse response from perforce")
418 if 'p4ExitCode' in result
:
419 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
420 die_bad_access("could not run p4")
422 code
= result
.get("code")
424 # we get here if we couldn't connect and there was nothing to unmarshal
425 die_bad_access("could not connect")
428 expiry
= result
.get("TicketExpiration")
431 if expiry
> min_expiration
:
435 die_bad_access("perforce ticket expires in {0} seconds".format(expiry
))
438 # account without a timeout - all ok
441 elif code
== "error":
442 data
= result
.get("data")
444 die_bad_access("p4 error: {0}".format(data
))
446 die_bad_access("unknown error")
450 die_bad_access("unknown error code {0}".format(code
))
452 _p4_version_string
= None
453 def p4_version_string():
454 """Read the version string, showing just the last line, which
455 hopefully is the interesting version bit.
458 Perforce - The Fast Software Configuration Management System.
459 Copyright 1995-2011 Perforce Software. All rights reserved.
460 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
462 global _p4_version_string
463 if not _p4_version_string
:
464 a
= p4_read_pipe_lines(["-V"])
465 _p4_version_string
= a
[-1].rstrip()
466 return _p4_version_string
468 def p4_integrate(src
, dest
):
469 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
471 def p4_sync(f
, *options
):
472 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
475 # forcibly add file names with wildcards
476 if wildcard_present(f
):
477 p4_system(["add", "-f", f
])
479 p4_system(["add", f
])
482 p4_system(["delete", wildcard_encode(f
)])
484 def p4_edit(f
, *options
):
485 p4_system(["edit"] + list(options
) + [wildcard_encode(f
)])
488 p4_system(["revert", wildcard_encode(f
)])
490 def p4_reopen(type, f
):
491 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
493 def p4_reopen_in_change(changelist
, files
):
494 cmd
= ["reopen", "-c", str(changelist
)] + files
497 def p4_move(src
, dest
):
498 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
500 def p4_last_change():
501 results
= p4CmdList(["changes", "-m", "1"], skip_info
=True)
502 return int(results
[0]['change'])
504 def p4_describe(change
, shelved
=False):
505 """Make sure it returns a valid result by checking for
506 the presence of field "time". Return a dict of the
509 cmd
= ["describe", "-s"]
514 ds
= p4CmdList(cmd
, skip_info
=True)
516 die("p4 describe -s %d did not return 1 result: %s" % (change
, str(ds
)))
520 if "p4ExitCode" in d
:
521 die("p4 describe -s %d exited with %d: %s" % (change
, d
["p4ExitCode"],
524 if d
["code"] == "error":
525 die("p4 describe -s %d returned error code: %s" % (change
, str(d
)))
528 die("p4 describe -s %d returned no \"time\": %s" % (change
, str(d
)))
533 # Canonicalize the p4 type and return a tuple of the
534 # base type, plus any modifiers. See "p4 help filetypes"
535 # for a list and explanation.
537 def split_p4_type(p4type
):
539 p4_filetypes_historical
= {
540 "ctempobj": "binary+Sw",
546 "tempobj": "binary+FSw",
547 "ubinary": "binary+F",
548 "uresource": "resource+F",
549 "uxbinary": "binary+Fx",
550 "xbinary": "binary+x",
552 "xtempobj": "binary+Swx",
554 "xunicode": "unicode+x",
557 if p4type
in p4_filetypes_historical
:
558 p4type
= p4_filetypes_historical
[p4type
]
560 s
= p4type
.split("+")
568 # return the raw p4 type of a file (text, text+ko, etc)
571 results
= p4CmdList(["fstat", "-T", "headType", wildcard_encode(f
)])
572 return results
[0]['headType']
575 # Given a type base and modifier, return a regexp matching
576 # the keywords that can be expanded in the file
578 def p4_keywords_regexp_for_type(base
, type_mods
):
579 if base
in ("text", "unicode", "binary"):
581 if "ko" in type_mods
:
583 elif "k" in type_mods
:
584 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
588 \$ # Starts with a dollar, followed by...
589 (%s) # one of the keywords, followed by...
590 (:[^$\n]+)? # possibly an old expansion, followed by...
598 # Given a file, return a regexp matching the possible
599 # RCS keywords that will be expanded, or None for files
600 # with kw expansion turned off.
602 def p4_keywords_regexp_for_file(file):
603 if not os
.path
.exists(file):
606 (type_base
, type_mods
) = split_p4_type(p4_type(file))
607 return p4_keywords_regexp_for_type(type_base
, type_mods
)
609 def setP4ExecBit(file, mode
):
610 # Reopens an already open file and changes the execute bit to match
611 # the execute bit setting in the passed in mode.
615 if not isModeExec(mode
):
616 p4Type
= getP4OpenedType(file)
617 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
618 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
619 if p4Type
[-1] == "+":
620 p4Type
= p4Type
[0:-1]
622 p4_reopen(p4Type
, file)
624 def getP4OpenedType(file):
625 # Returns the perforce file type for the given file.
627 result
= p4_read_pipe(["opened", wildcard_encode(file)])
628 match
= re
.match(".*\((.+)\)( \*exclusive\*)?\r?$", result
)
630 return match
.group(1)
632 die("Could not determine file type for %s (result: '%s')" % (file, result
))
634 # Return the set of all p4 labels
635 def getP4Labels(depotPaths
):
637 if not isinstance(depotPaths
, list):
638 depotPaths
= [depotPaths
]
640 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
646 # Return the set of all git tags
649 for line
in read_pipe_lines(["git", "tag"]):
654 _diff_tree_pattern
= None
656 def parseDiffTreeEntry(entry
):
657 """Parses a single diff tree entry into its component elements.
659 See git-diff-tree(1) manpage for details about the format of the diff
660 output. This method returns a dictionary with the following elements:
662 src_mode - The mode of the source file
663 dst_mode - The mode of the destination file
664 src_sha1 - The sha1 for the source file
665 dst_sha1 - The sha1 fr the destination file
666 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
667 status_score - The score for the status (applicable for 'C' and 'R'
668 statuses). This is None if there is no score.
669 src - The path for the source file.
670 dst - The path for the destination file. This is only present for
671 copy or renames. If it is not present, this is None.
673 If the pattern is not matched, None is returned."""
675 global _diff_tree_pattern
676 if not _diff_tree_pattern
:
677 _diff_tree_pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
679 match
= _diff_tree_pattern
.match(entry
)
682 'src_mode': match
.group(1),
683 'dst_mode': match
.group(2),
684 'src_sha1': match
.group(3),
685 'dst_sha1': match
.group(4),
686 'status': match
.group(5),
687 'status_score': match
.group(6),
688 'src': match
.group(7),
689 'dst': match
.group(10)
693 def isModeExec(mode
):
694 # Returns True if the given git mode represents an executable file,
696 return mode
[-3:] == "755"
698 class P4Exception(Exception):
699 """ Base class for exceptions from the p4 client """
700 def __init__(self
, exit_code
):
701 self
.p4ExitCode
= exit_code
703 class P4ServerException(P4Exception
):
704 """ Base class for exceptions where we get some kind of marshalled up result from the server """
705 def __init__(self
, exit_code
, p4_result
):
706 super(P4ServerException
, self
).__init
__(exit_code
)
707 self
.p4_result
= p4_result
708 self
.code
= p4_result
[0]['code']
709 self
.data
= p4_result
[0]['data']
711 class P4RequestSizeException(P4ServerException
):
712 """ One of the maxresults or maxscanrows errors """
713 def __init__(self
, exit_code
, p4_result
, limit
):
714 super(P4RequestSizeException
, self
).__init
__(exit_code
, p4_result
)
717 class P4CommandException(P4Exception
):
718 """ Something went wrong calling p4 which means we have to give up """
719 def __init__(self
, msg
):
725 def isModeExecChanged(src_mode
, dst_mode
):
726 return isModeExec(src_mode
) != isModeExec(dst_mode
)
728 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None, skip_info
=False,
729 errors_as_exceptions
=False):
731 if not isinstance(cmd
, list):
738 cmd
= p4_build_cmd(cmd
)
740 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
742 # Use a temporary file to avoid deadlocks without
743 # subprocess.communicate(), which would put another copy
744 # of stdout into memory.
746 if stdin
is not None:
747 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
748 if not isinstance(stdin
, list):
749 stdin_file
.write(stdin
)
752 stdin_file
.write(encode_text_stream(i
))
753 stdin_file
.write(b
'\n')
757 p4
= subprocess
.Popen(cmd
,
760 stdout
=subprocess
.PIPE
)
765 entry
= marshal
.load(p4
.stdout
)
767 # Decode unmarshalled dict to use str keys and values, except for:
768 # - `data` which may contain arbitrary binary data
769 # - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text
771 for key
, value
in entry
.items():
773 if isinstance(value
, bytes
) and not (key
in ('data', 'path', 'clientFile') or key
.startswith('depotFile')):
774 value
= value
.decode()
775 decoded_entry
[key
] = value
776 # Parse out data if it's an error response
777 if decoded_entry
.get('code') == 'error' and 'data' in decoded_entry
:
778 decoded_entry
['data'] = decoded_entry
['data'].decode()
779 entry
= decoded_entry
781 if 'code' in entry
and entry
['code'] == 'info':
791 if errors_as_exceptions
:
793 data
= result
[0].get('data')
795 m
= re
.search('Too many rows scanned \(over (\d+)\)', data
)
797 m
= re
.search('Request too large \(over (\d+)\)', data
)
800 limit
= int(m
.group(1))
801 raise P4RequestSizeException(exitCode
, result
, limit
)
803 raise P4ServerException(exitCode
, result
)
805 raise P4Exception(exitCode
)
808 entry
["p4ExitCode"] = exitCode
814 list = p4CmdList(cmd
)
820 def p4Where(depotPath
):
821 if not depotPath
.endswith("/"):
823 depotPathLong
= depotPath
+ "..."
824 outputList
= p4CmdList(["where", depotPathLong
])
826 for entry
in outputList
:
827 if "depotFile" in entry
:
828 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
829 # The base path always ends with "/...".
830 entry_path
= decode_path(entry
['depotFile'])
831 if entry_path
.find(depotPath
) == 0 and entry_path
[-4:] == "/...":
834 elif "data" in entry
:
835 data
= entry
.get("data")
836 space
= data
.find(" ")
837 if data
[:space
] == depotPath
:
842 if output
["code"] == "error":
846 clientPath
= decode_path(output
['path'])
847 elif "data" in output
:
848 data
= output
.get("data")
849 lastSpace
= data
.rfind(b
" ")
850 clientPath
= decode_path(data
[lastSpace
+ 1:])
852 if clientPath
.endswith("..."):
853 clientPath
= clientPath
[:-3]
856 def currentGitBranch():
857 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
859 def isValidGitDir(path
):
860 return git_dir(path
) != None
862 def parseRevision(ref
):
863 return read_pipe("git rev-parse %s" % ref
).strip()
865 def branchExists(ref
):
866 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
870 def extractLogMessageFromGitCommit(commit
):
873 ## fixme: title is first line of commit, not 1st paragraph.
875 for log
in read_pipe_lines(["git", "cat-file", "commit", commit
]):
884 def extractSettingsGitLog(log
):
886 for line
in log
.split("\n"):
888 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
892 assignments
= m
.group(1).split (':')
893 for a
in assignments
:
895 key
= vals
[0].strip()
896 val
= ('='.join (vals
[1:])).strip()
897 if val
.endswith ('\"') and val
.startswith('"'):
902 paths
= values
.get("depot-paths")
904 paths
= values
.get("depot-path")
906 values
['depot-paths'] = paths
.split(',')
909 def gitBranchExists(branch
):
910 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
911 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
912 return proc
.wait() == 0;
914 def gitUpdateRef(ref
, newvalue
):
915 subprocess
.check_call(["git", "update-ref", ref
, newvalue
])
917 def gitDeleteRef(ref
):
918 subprocess
.check_call(["git", "update-ref", "-d", ref
])
922 def gitConfig(key
, typeSpecifier
=None):
923 if key
not in _gitConfig
:
924 cmd
= [ "git", "config" ]
926 cmd
+= [ typeSpecifier
]
928 s
= read_pipe(cmd
, ignore_error
=True)
929 _gitConfig
[key
] = s
.strip()
930 return _gitConfig
[key
]
932 def gitConfigBool(key
):
933 """Return a bool, using git config --bool. It is True only if the
934 variable is set to true, and False if set to false or not present
937 if key
not in _gitConfig
:
938 _gitConfig
[key
] = gitConfig(key
, '--bool') == "true"
939 return _gitConfig
[key
]
941 def gitConfigInt(key
):
942 if key
not in _gitConfig
:
943 cmd
= [ "git", "config", "--int", key
]
944 s
= read_pipe(cmd
, ignore_error
=True)
947 _gitConfig
[key
] = int(gitConfig(key
, '--int'))
949 _gitConfig
[key
] = None
950 return _gitConfig
[key
]
952 def gitConfigList(key
):
953 if key
not in _gitConfig
:
954 s
= read_pipe(["git", "config", "--get-all", key
], ignore_error
=True)
955 _gitConfig
[key
] = s
.strip().splitlines()
956 if _gitConfig
[key
] == ['']:
958 return _gitConfig
[key
]
960 def p4BranchesInGit(branchesAreInRemotes
=True):
961 """Find all the branches whose names start with "p4/", looking
962 in remotes or heads as specified by the argument. Return
963 a dictionary of { branch: revision } for each one found.
964 The branch names are the short names, without any
969 cmdline
= "git rev-parse --symbolic "
970 if branchesAreInRemotes
:
971 cmdline
+= "--remotes"
973 cmdline
+= "--branches"
975 for line
in read_pipe_lines(cmdline
):
979 if not line
.startswith('p4/'):
981 # special symbolic ref to p4/master
982 if line
== "p4/HEAD":
985 # strip off p4/ prefix
986 branch
= line
[len("p4/"):]
988 branches
[branch
] = parseRevision(line
)
992 def branch_exists(branch
):
993 """Make sure that the given ref name really exists."""
995 cmd
= [ "git", "rev-parse", "--symbolic", "--verify", branch
]
996 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
997 out
, _
= p
.communicate()
998 out
= decode_text_stream(out
)
1001 # expect exactly one line of output: the branch name
1002 return out
.rstrip() == branch
1004 def findUpstreamBranchPoint(head
= "HEAD"):
1005 branches
= p4BranchesInGit()
1006 # map from depot-path to branch name
1007 branchByDepotPath
= {}
1008 for branch
in branches
.keys():
1009 tip
= branches
[branch
]
1010 log
= extractLogMessageFromGitCommit(tip
)
1011 settings
= extractSettingsGitLog(log
)
1012 if "depot-paths" in settings
:
1013 paths
= ",".join(settings
["depot-paths"])
1014 branchByDepotPath
[paths
] = "remotes/p4/" + branch
1018 while parent
< 65535:
1019 commit
= head
+ "~%s" % parent
1020 log
= extractLogMessageFromGitCommit(commit
)
1021 settings
= extractSettingsGitLog(log
)
1022 if "depot-paths" in settings
:
1023 paths
= ",".join(settings
["depot-paths"])
1024 if paths
in branchByDepotPath
:
1025 return [branchByDepotPath
[paths
], settings
]
1029 return ["", settings
]
1031 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
1033 print("Creating/updating branch(es) in %s based on origin branch(es)"
1036 originPrefix
= "origin/p4/"
1038 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
1040 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
1043 headName
= line
[len(originPrefix
):]
1044 remoteHead
= localRefPrefix
+ headName
1047 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
1048 if ('depot-paths' not in original
1049 or 'change' not in original
):
1053 if not gitBranchExists(remoteHead
):
1055 print("creating %s" % remoteHead
)
1058 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
1059 if 'change' in settings
:
1060 if settings
['depot-paths'] == original
['depot-paths']:
1061 originP4Change
= int(original
['change'])
1062 p4Change
= int(settings
['change'])
1063 if originP4Change
> p4Change
:
1064 print("%s (%s) is newer than %s (%s). "
1065 "Updating p4 branch from origin."
1066 % (originHead
, originP4Change
,
1067 remoteHead
, p4Change
))
1070 print("Ignoring: %s was imported from %s while "
1071 "%s was imported from %s"
1072 % (originHead
, ','.join(original
['depot-paths']),
1073 remoteHead
, ','.join(settings
['depot-paths'])))
1076 system("git update-ref %s %s" % (remoteHead
, originHead
))
1078 def originP4BranchesExist():
1079 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
1082 def p4ParseNumericChangeRange(parts
):
1083 changeStart
= int(parts
[0][1:])
1084 if parts
[1] == '#head':
1085 changeEnd
= p4_last_change()
1087 changeEnd
= int(parts
[1])
1089 return (changeStart
, changeEnd
)
1091 def chooseBlockSize(blockSize
):
1095 return defaultBlockSize
1097 def p4ChangesForPaths(depotPaths
, changeRange
, requestedBlockSize
):
1100 # Parse the change range into start and end. Try to find integer
1101 # revision ranges as these can be broken up into blocks to avoid
1102 # hitting server-side limits (maxrows, maxscanresults). But if
1103 # that doesn't work, fall back to using the raw revision specifier
1104 # strings, without using block mode.
1106 if changeRange
is None or changeRange
== '':
1108 changeEnd
= p4_last_change()
1109 block_size
= chooseBlockSize(requestedBlockSize
)
1111 parts
= changeRange
.split(',')
1112 assert len(parts
) == 2
1114 (changeStart
, changeEnd
) = p4ParseNumericChangeRange(parts
)
1115 block_size
= chooseBlockSize(requestedBlockSize
)
1117 changeStart
= parts
[0][1:]
1118 changeEnd
= parts
[1]
1119 if requestedBlockSize
:
1120 die("cannot use --changes-block-size with non-numeric revisions")
1125 # Retrieve changes a block at a time, to prevent running
1126 # into a MaxResults/MaxScanRows error from the server. If
1127 # we _do_ hit one of those errors, turn down the block size
1133 end
= min(changeEnd
, changeStart
+ block_size
)
1134 revisionRange
= "%d,%d" % (changeStart
, end
)
1136 revisionRange
= "%s,%s" % (changeStart
, changeEnd
)
1138 for p
in depotPaths
:
1139 cmd
+= ["%s...@%s" % (p
, revisionRange
)]
1143 result
= p4CmdList(cmd
, errors_as_exceptions
=True)
1144 except P4RequestSizeException
as e
:
1146 block_size
= e
.limit
1147 elif block_size
> e
.limit
:
1148 block_size
= e
.limit
1150 block_size
= max(2, block_size
// 2)
1152 if verbose
: print("block size error, retrying with block size {0}".format(block_size
))
1154 except P4Exception
as e
:
1155 die('Error retrieving changes description ({0})'.format(e
.p4ExitCode
))
1157 # Insert changes in chronological order
1158 for entry
in reversed(result
):
1159 if 'change' not in entry
:
1161 changes
.add(int(entry
['change']))
1166 if end
>= changeEnd
:
1169 changeStart
= end
+ 1
1171 changes
= sorted(changes
)
1174 def p4PathStartsWith(path
, prefix
):
1175 # This method tries to remedy a potential mixed-case issue:
1177 # If UserA adds //depot/DirA/file1
1178 # and UserB adds //depot/dira/file2
1180 # we may or may not have a problem. If you have core.ignorecase=true,
1181 # we treat DirA and dira as the same directory
1182 if gitConfigBool("core.ignorecase"):
1183 return path
.lower().startswith(prefix
.lower())
1184 return path
.startswith(prefix
)
1186 def getClientSpec():
1187 """Look at the p4 client spec, create a View() object that contains
1188 all the mappings, and return it."""
1190 specList
= p4CmdList("client -o")
1191 if len(specList
) != 1:
1192 die('Output from "client -o" is %d lines, expecting 1' %
1195 # dictionary of all client parameters
1198 # the //client/ name
1199 client_name
= entry
["Client"]
1201 # just the keys that start with "View"
1202 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
1204 # hold this new View
1205 view
= View(client_name
)
1207 # append the lines, in order, to the view
1208 for view_num
in range(len(view_keys
)):
1209 k
= "View%d" % view_num
1210 if k
not in view_keys
:
1211 die("Expected view key %s missing" % k
)
1212 view
.append(entry
[k
])
1216 def getClientRoot():
1217 """Grab the client directory."""
1219 output
= p4CmdList("client -o")
1220 if len(output
) != 1:
1221 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
1224 if "Root" not in entry
:
1225 die('Client has no "Root"')
1227 return entry
["Root"]
1230 # P4 wildcards are not allowed in filenames. P4 complains
1231 # if you simply add them, but you can force it with "-f", in
1232 # which case it translates them into %xx encoding internally.
1234 def wildcard_decode(path
):
1235 # Search for and fix just these four characters. Do % last so
1236 # that fixing it does not inadvertently create new %-escapes.
1237 # Cannot have * in a filename in windows; untested as to
1238 # what p4 would do in such a case.
1239 if not platform
.system() == "Windows":
1240 path
= path
.replace("%2A", "*")
1241 path
= path
.replace("%23", "#") \
1242 .replace("%40", "@") \
1243 .replace("%25", "%")
1246 def wildcard_encode(path
):
1247 # do % first to avoid double-encoding the %s introduced here
1248 path
= path
.replace("%", "%25") \
1249 .replace("*", "%2A") \
1250 .replace("#", "%23") \
1251 .replace("@", "%40")
1254 def wildcard_present(path
):
1255 m
= re
.search("[*#@%]", path
)
1256 return m
is not None
1258 class LargeFileSystem(object):
1259 """Base class for large file system support."""
1261 def __init__(self
, writeToGitStream
):
1262 self
.largeFiles
= set()
1263 self
.writeToGitStream
= writeToGitStream
1265 def generatePointer(self
, cloneDestination
, contentFile
):
1266 """Return the content of a pointer file that is stored in Git instead of
1267 the actual content."""
1268 assert False, "Method 'generatePointer' required in " + self
.__class
__.__name
__
1270 def pushFile(self
, localLargeFile
):
1271 """Push the actual content which is not stored in the Git repository to
1273 assert False, "Method 'pushFile' required in " + self
.__class
__.__name
__
1275 def hasLargeFileExtension(self
, relPath
):
1276 return functools
.reduce(
1277 lambda a
, b
: a
or b
,
1278 [relPath
.endswith('.' + e
) for e
in gitConfigList('git-p4.largeFileExtensions')],
1282 def generateTempFile(self
, contents
):
1283 contentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
1285 contentFile
.write(d
)
1287 return contentFile
.name
1289 def exceedsLargeFileThreshold(self
, relPath
, contents
):
1290 if gitConfigInt('git-p4.largeFileThreshold'):
1291 contentsSize
= sum(len(d
) for d
in contents
)
1292 if contentsSize
> gitConfigInt('git-p4.largeFileThreshold'):
1294 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1295 contentsSize
= sum(len(d
) for d
in contents
)
1296 if contentsSize
<= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1298 contentTempFile
= self
.generateTempFile(contents
)
1299 compressedContentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=True)
1300 with zipfile
.ZipFile(compressedContentFile
, mode
='w') as zf
:
1301 zf
.write(contentTempFile
, compress_type
=zipfile
.ZIP_DEFLATED
)
1302 compressedContentsSize
= zf
.infolist()[0].compress_size
1303 os
.remove(contentTempFile
)
1304 if compressedContentsSize
> gitConfigInt('git-p4.largeFileCompressedThreshold'):
1308 def addLargeFile(self
, relPath
):
1309 self
.largeFiles
.add(relPath
)
1311 def removeLargeFile(self
, relPath
):
1312 self
.largeFiles
.remove(relPath
)
1314 def isLargeFile(self
, relPath
):
1315 return relPath
in self
.largeFiles
1317 def processContent(self
, git_mode
, relPath
, contents
):
1318 """Processes the content of git fast import. This method decides if a
1319 file is stored in the large file system and handles all necessary
1321 if self
.exceedsLargeFileThreshold(relPath
, contents
) or self
.hasLargeFileExtension(relPath
):
1322 contentTempFile
= self
.generateTempFile(contents
)
1323 (pointer_git_mode
, contents
, localLargeFile
) = self
.generatePointer(contentTempFile
)
1324 if pointer_git_mode
:
1325 git_mode
= pointer_git_mode
1327 # Move temp file to final location in large file system
1328 largeFileDir
= os
.path
.dirname(localLargeFile
)
1329 if not os
.path
.isdir(largeFileDir
):
1330 os
.makedirs(largeFileDir
)
1331 shutil
.move(contentTempFile
, localLargeFile
)
1332 self
.addLargeFile(relPath
)
1333 if gitConfigBool('git-p4.largeFilePush'):
1334 self
.pushFile(localLargeFile
)
1336 sys
.stderr
.write("%s moved to large file system (%s)\n" % (relPath
, localLargeFile
))
1337 return (git_mode
, contents
)
1339 class MockLFS(LargeFileSystem
):
1340 """Mock large file system for testing."""
1342 def generatePointer(self
, contentFile
):
1343 """The pointer content is the original content prefixed with "pointer-".
1344 The local filename of the large file storage is derived from the file content.
1346 with
open(contentFile
, 'r') as f
:
1349 pointerContents
= 'pointer-' + content
1350 localLargeFile
= os
.path
.join(os
.getcwd(), '.git', 'mock-storage', 'local', content
[:-1])
1351 return (gitMode
, pointerContents
, localLargeFile
)
1353 def pushFile(self
, localLargeFile
):
1354 """The remote filename of the large file storage is the same as the local
1355 one but in a different directory.
1357 remotePath
= os
.path
.join(os
.path
.dirname(localLargeFile
), '..', 'remote')
1358 if not os
.path
.exists(remotePath
):
1359 os
.makedirs(remotePath
)
1360 shutil
.copyfile(localLargeFile
, os
.path
.join(remotePath
, os
.path
.basename(localLargeFile
)))
1362 class GitLFS(LargeFileSystem
):
1363 """Git LFS as backend for the git-p4 large file system.
1364 See https://git-lfs.github.com/ for details."""
1366 def __init__(self
, *args
):
1367 LargeFileSystem
.__init
__(self
, *args
)
1368 self
.baseGitAttributes
= []
1370 def generatePointer(self
, contentFile
):
1371 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1372 mode and content which is stored in the Git repository instead of
1373 the actual content. Return also the new location of the actual
1376 if os
.path
.getsize(contentFile
) == 0:
1377 return (None, '', None)
1379 pointerProcess
= subprocess
.Popen(
1380 ['git', 'lfs', 'pointer', '--file=' + contentFile
],
1381 stdout
=subprocess
.PIPE
1383 pointerFile
= decode_text_stream(pointerProcess
.stdout
.read())
1384 if pointerProcess
.wait():
1385 os
.remove(contentFile
)
1386 die('git-lfs pointer command failed. Did you install the extension?')
1388 # Git LFS removed the preamble in the output of the 'pointer' command
1389 # starting from version 1.2.0. Check for the preamble here to support
1391 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1392 if pointerFile
.startswith('Git LFS pointer for'):
1393 pointerFile
= re
.sub(r
'Git LFS pointer for.*\n\n', '', pointerFile
)
1395 oid
= re
.search(r
'^oid \w+:(\w+)', pointerFile
, re
.MULTILINE
).group(1)
1396 # if someone use external lfs.storage ( not in local repo git )
1397 lfs_path
= gitConfig('lfs.storage')
1400 if not os
.path
.isabs(lfs_path
):
1401 lfs_path
= os
.path
.join(os
.getcwd(), '.git', lfs_path
)
1402 localLargeFile
= os
.path
.join(
1404 'objects', oid
[:2], oid
[2:4],
1407 # LFS Spec states that pointer files should not have the executable bit set.
1409 return (gitMode
, pointerFile
, localLargeFile
)
1411 def pushFile(self
, localLargeFile
):
1412 uploadProcess
= subprocess
.Popen(
1413 ['git', 'lfs', 'push', '--object-id', 'origin', os
.path
.basename(localLargeFile
)]
1415 if uploadProcess
.wait():
1416 die('git-lfs push command failed. Did you define a remote?')
1418 def generateGitAttributes(self
):
1420 self
.baseGitAttributes
+
1424 '# Git LFS (see https://git-lfs.github.com/)\n',
1427 ['*.' + f
.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1428 for f
in sorted(gitConfigList('git-p4.largeFileExtensions'))
1430 ['/' + f
.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1431 for f
in sorted(self
.largeFiles
) if not self
.hasLargeFileExtension(f
)
1435 def addLargeFile(self
, relPath
):
1436 LargeFileSystem
.addLargeFile(self
, relPath
)
1437 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1439 def removeLargeFile(self
, relPath
):
1440 LargeFileSystem
.removeLargeFile(self
, relPath
)
1441 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1443 def processContent(self
, git_mode
, relPath
, contents
):
1444 if relPath
== '.gitattributes':
1445 self
.baseGitAttributes
= contents
1446 return (git_mode
, self
.generateGitAttributes())
1448 return LargeFileSystem
.processContent(self
, git_mode
, relPath
, contents
)
1451 delete_actions
= ( "delete", "move/delete", "purge" )
1452 add_actions
= ( "add", "branch", "move/add" )
1455 self
.usage
= "usage: %prog [options]"
1456 self
.needsGit
= True
1457 self
.verbose
= False
1459 # This is required for the "append" update_shelve action
1460 def ensure_value(self
, attr
, value
):
1461 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
1462 setattr(self
, attr
, value
)
1463 return getattr(self
, attr
)
1467 self
.userMapFromPerforceServer
= False
1468 self
.myP4UserId
= None
1472 return self
.myP4UserId
1474 results
= p4CmdList("user -o")
1477 self
.myP4UserId
= r
['User']
1479 die("Could not find your p4 user id")
1481 def p4UserIsMe(self
, p4User
):
1482 # return True if the given p4 user is actually me
1483 me
= self
.p4UserId()
1484 if not p4User
or p4User
!= me
:
1489 def getUserCacheFilename(self
):
1490 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
1491 return home
+ "/.gitp4-usercache.txt"
1493 def getUserMapFromPerforceServer(self
):
1494 if self
.userMapFromPerforceServer
:
1499 for output
in p4CmdList("users"):
1500 if "User" not in output
:
1502 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
1503 self
.emails
[output
["Email"]] = output
["User"]
1505 mapUserConfigRegex
= re
.compile(r
"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re
.VERBOSE
)
1506 for mapUserConfig
in gitConfigList("git-p4.mapUser"):
1507 mapUser
= mapUserConfigRegex
.findall(mapUserConfig
)
1508 if mapUser
and len(mapUser
[0]) == 3:
1509 user
= mapUser
[0][0]
1510 fullname
= mapUser
[0][1]
1511 email
= mapUser
[0][2]
1512 self
.users
[user
] = fullname
+ " <" + email
+ ">"
1513 self
.emails
[email
] = user
1516 for (key
, val
) in self
.users
.items():
1517 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
1519 open(self
.getUserCacheFilename(), 'w').write(s
)
1520 self
.userMapFromPerforceServer
= True
1522 def loadUserMapFromCache(self
):
1524 self
.userMapFromPerforceServer
= False
1526 cache
= open(self
.getUserCacheFilename(), 'r')
1527 lines
= cache
.readlines()
1530 entry
= line
.strip().split("\t")
1531 self
.users
[entry
[0]] = entry
[1]
1533 self
.getUserMapFromPerforceServer()
1535 class P4Submit(Command
, P4UserMap
):
1537 conflict_behavior_choices
= ("ask", "skip", "quit")
1540 Command
.__init
__(self
)
1541 P4UserMap
.__init
__(self
)
1543 optparse
.make_option("--origin", dest
="origin"),
1544 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
1545 # preserve the user, requires relevant p4 permissions
1546 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
1547 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
1548 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
1549 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
1550 optparse
.make_option("--conflict", dest
="conflict_behavior",
1551 choices
=self
.conflict_behavior_choices
),
1552 optparse
.make_option("--branch", dest
="branch"),
1553 optparse
.make_option("--shelve", dest
="shelve", action
="store_true",
1554 help="Shelve instead of submit. Shelved files are reverted, "
1555 "restoring the workspace to the state before the shelve"),
1556 optparse
.make_option("--update-shelve", dest
="update_shelve", action
="append", type="int",
1557 metavar
="CHANGELIST",
1558 help="update an existing shelved changelist, implies --shelve, "
1559 "repeat in-order for multiple shelved changelists"),
1560 optparse
.make_option("--commit", dest
="commit", metavar
="COMMIT",
1561 help="submit only the specified commit(s), one commit or xxx..xxx"),
1562 optparse
.make_option("--disable-rebase", dest
="disable_rebase", action
="store_true",
1563 help="Disable rebase after submit is completed. Can be useful if you "
1564 "work from a local git branch that is not master"),
1565 optparse
.make_option("--disable-p4sync", dest
="disable_p4sync", action
="store_true",
1566 help="Skip Perforce sync of p4/master after submit or shelve"),
1567 optparse
.make_option("--no-verify", dest
="no_verify", action
="store_true",
1568 help="Bypass p4-pre-submit and p4-changelist hooks"),
1570 self
.description
= """Submit changes from git to the perforce depot.\n
1571 The `p4-pre-submit` hook is executed if it exists and is executable. It
1572 can be bypassed with the `--no-verify` command line option. The hook takes
1573 no parameters and nothing from standard input. Exiting with a non-zero status
1574 from this script prevents `git-p4 submit` from launching.
1576 One usage scenario is to run unit tests in the hook.
1578 The `p4-prepare-changelist` hook is executed right after preparing the default
1579 changelist message and before the editor is started. It takes one parameter,
1580 the name of the file that contains the changelist text. Exiting with a non-zero
1581 status from the script will abort the process.
1583 The purpose of the hook is to edit the message file in place, and it is not
1584 supressed by the `--no-verify` option. This hook is called even if
1585 `--prepare-p4-only` is set.
1587 The `p4-changelist` hook is executed after the changelist message has been
1588 edited by the user. It can be bypassed with the `--no-verify` option. It
1589 takes a single parameter, the name of the file that holds the proposed
1590 changelist text. Exiting with a non-zero status causes the command to abort.
1592 The hook is allowed to edit the changelist file and can be used to normalize
1593 the text into some project standard format. It can also be used to refuse the
1594 Submit after inspect the message file.
1596 The `p4-post-changelist` hook is invoked after the submit has successfully
1597 occurred in P4. It takes no parameters and is meant primarily for notification
1598 and cannot affect the outcome of the git p4 submit action.
1601 self
.usage
+= " [name of git branch to submit into perforce depot]"
1603 self
.detectRenames
= False
1604 self
.preserveUser
= gitConfigBool("git-p4.preserveUser")
1605 self
.dry_run
= False
1607 self
.update_shelve
= list()
1609 self
.disable_rebase
= gitConfigBool("git-p4.disableRebase")
1610 self
.disable_p4sync
= gitConfigBool("git-p4.disableP4Sync")
1611 self
.prepare_p4_only
= False
1612 self
.conflict_behavior
= None
1613 self
.isWindows
= (platform
.system() == "Windows")
1614 self
.exportLabels
= False
1615 self
.p4HasMoveCommand
= p4_has_move_command()
1617 self
.no_verify
= False
1619 if gitConfig('git-p4.largeFileSystem'):
1620 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1623 if len(p4CmdList("opened ...")) > 0:
1624 die("You have files opened with perforce! Close them before starting the sync.")
1626 def separate_jobs_from_description(self
, message
):
1627 """Extract and return a possible Jobs field in the commit
1628 message. It goes into a separate section in the p4 change
1631 A jobs line starts with "Jobs:" and looks like a new field
1632 in a form. Values are white-space separated on the same
1633 line or on following lines that start with a tab.
1635 This does not parse and extract the full git commit message
1636 like a p4 form. It just sees the Jobs: line as a marker
1637 to pass everything from then on directly into the p4 form,
1638 but outside the description section.
1640 Return a tuple (stripped log message, jobs string)."""
1642 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
1644 return (message
, None)
1646 jobtext
= message
[m
.start():]
1647 stripped_message
= message
[:m
.start()].rstrip()
1648 return (stripped_message
, jobtext
)
1650 def prepareLogMessage(self
, template
, message
, jobs
):
1651 """Edits the template returned from "p4 change -o" to insert
1652 the message in the Description field, and the jobs text in
1656 inDescriptionSection
= False
1658 for line
in template
.split("\n"):
1659 if line
.startswith("#"):
1660 result
+= line
+ "\n"
1663 if inDescriptionSection
:
1664 if line
.startswith("Files:") or line
.startswith("Jobs:"):
1665 inDescriptionSection
= False
1666 # insert Jobs section
1668 result
+= jobs
+ "\n"
1672 if line
.startswith("Description:"):
1673 inDescriptionSection
= True
1675 for messageLine
in message
.split("\n"):
1676 line
+= "\t" + messageLine
+ "\n"
1678 result
+= line
+ "\n"
1682 def patchRCSKeywords(self
, file, pattern
):
1683 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1684 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
1686 outFile
= os
.fdopen(handle
, "w+")
1687 inFile
= open(file, "r")
1688 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1689 for line
in inFile
.readlines():
1690 line
= regexp
.sub(r
'$\1$', line
)
1694 # Forcibly overwrite the original file
1696 shutil
.move(outFileName
, file)
1698 # cleanup our temporary file
1699 os
.unlink(outFileName
)
1700 print("Failed to strip RCS keywords in %s" % file)
1703 print("Patched up RCS keywords in %s" % file)
1705 def p4UserForCommit(self
,id):
1706 # Return the tuple (perforce user,git email) for a given git commit id
1707 self
.getUserMapFromPerforceServer()
1708 gitEmail
= read_pipe(["git", "log", "--max-count=1",
1709 "--format=%ae", id])
1710 gitEmail
= gitEmail
.strip()
1711 if gitEmail
not in self
.emails
:
1712 return (None,gitEmail
)
1714 return (self
.emails
[gitEmail
],gitEmail
)
1716 def checkValidP4Users(self
,commits
):
1717 # check if any git authors cannot be mapped to p4 users
1719 (user
,email
) = self
.p4UserForCommit(id)
1721 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
1722 if gitConfigBool("git-p4.allowMissingP4Users"):
1725 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
1727 def lastP4Changelist(self
):
1728 # Get back the last changelist number submitted in this client spec. This
1729 # then gets used to patch up the username in the change. If the same
1730 # client spec is being used by multiple processes then this might go
1732 results
= p4CmdList("client -o") # find the current client
1736 client
= r
['Client']
1739 die("could not get client spec")
1740 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
1744 die("Could not get changelist number for last submit - cannot patch up user details")
1746 def modifyChangelistUser(self
, changelist
, newUser
):
1747 # fixup the user field of a changelist after it has been submitted.
1748 changes
= p4CmdList("change -o %s" % changelist
)
1749 if len(changes
) != 1:
1750 die("Bad output from p4 change modifying %s to user %s" %
1751 (changelist
, newUser
))
1754 if c
['User'] == newUser
: return # nothing to do
1756 # p4 does not understand format version 3 and above
1757 input = marshal
.dumps(c
, 2)
1759 result
= p4CmdList("change -f -i", stdin
=input)
1762 if r
['code'] == 'error':
1763 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1765 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1767 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1769 def canChangeChangelists(self
):
1770 # check to see if we have p4 admin or super-user permissions, either of
1771 # which are required to modify changelists.
1772 results
= p4CmdList(["protects", self
.depotPath
])
1775 if r
['perm'] == 'admin':
1777 if r
['perm'] == 'super':
1781 def prepareSubmitTemplate(self
, changelist
=None):
1782 """Run "p4 change -o" to grab a change specification template.
1783 This does not use "p4 -G", as it is nice to keep the submission
1784 template in original order, since a human might edit it.
1786 Remove lines in the Files section that show changes to files
1787 outside the depot path we're committing into."""
1789 [upstream
, settings
] = findUpstreamBranchPoint()
1792 # A Perforce Change Specification.
1794 # Change: The change number. 'new' on a new changelist.
1795 # Date: The date this specification was last modified.
1796 # Client: The client on which the changelist was created. Read-only.
1797 # User: The user who created the changelist.
1798 # Status: Either 'pending' or 'submitted'. Read-only.
1799 # Type: Either 'public' or 'restricted'. Default is 'public'.
1800 # Description: Comments about the changelist. Required.
1801 # Jobs: What opened jobs are to be closed by this changelist.
1802 # You may delete jobs from this list. (New changelists only.)
1803 # Files: What opened files from the default changelist are to be added
1804 # to this changelist. You may delete files from this list.
1805 # (New changelists only.)
1808 inFilesSection
= False
1810 args
= ['change', '-o']
1812 args
.append(str(changelist
))
1813 for entry
in p4CmdList(args
):
1814 if 'code' not in entry
:
1816 if entry
['code'] == 'stat':
1817 change_entry
= entry
1819 if not change_entry
:
1820 die('Failed to decode output of p4 change -o')
1821 for key
, value
in change_entry
.items():
1822 if key
.startswith('File'):
1823 if 'depot-paths' in settings
:
1824 if not [p
for p
in settings
['depot-paths']
1825 if p4PathStartsWith(value
, p
)]:
1828 if not p4PathStartsWith(value
, self
.depotPath
):
1830 files_list
.append(value
)
1832 # Output in the order expected by prepareLogMessage
1833 for key
in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1834 if key
not in change_entry
:
1837 template
+= key
+ ':'
1838 if key
== 'Description':
1840 for field_line
in change_entry
[key
].splitlines():
1841 template
+= '\t'+field_line
+'\n'
1842 if len(files_list
) > 0:
1844 template
+= 'Files:\n'
1845 for path
in files_list
:
1846 template
+= '\t'+path
+'\n'
1849 def edit_template(self
, template_file
):
1850 """Invoke the editor to let the user change the submission
1851 message. Return true if okay to continue with the submit."""
1853 # if configured to skip the editing part, just submit
1854 if gitConfigBool("git-p4.skipSubmitEdit"):
1857 # look at the modification time, to check later if the user saved
1859 mtime
= os
.stat(template_file
).st_mtime
1862 if "P4EDITOR" in os
.environ
and (os
.environ
.get("P4EDITOR") != ""):
1863 editor
= os
.environ
.get("P4EDITOR")
1865 editor
= read_pipe("git var GIT_EDITOR").strip()
1866 system(["sh", "-c", ('%s "$@"' % editor
), editor
, template_file
])
1868 # If the file was not saved, prompt to see if this patch should
1869 # be skipped. But skip this verification step if configured so.
1870 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1873 # modification time updated means user saved the file
1874 if os
.stat(template_file
).st_mtime
> mtime
:
1877 response
= prompt("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1883 def get_diff_description(self
, editedFiles
, filesToAdd
, symlinks
):
1885 if "P4DIFF" in os
.environ
:
1886 del(os
.environ
["P4DIFF"])
1888 for editedFile
in editedFiles
:
1889 diff
+= p4_read_pipe(['diff', '-du',
1890 wildcard_encode(editedFile
)])
1894 for newFile
in filesToAdd
:
1895 newdiff
+= "==== new file ====\n"
1896 newdiff
+= "--- /dev/null\n"
1897 newdiff
+= "+++ %s\n" % newFile
1899 is_link
= os
.path
.islink(newFile
)
1900 expect_link
= newFile
in symlinks
1902 if is_link
and expect_link
:
1903 newdiff
+= "+%s\n" % os
.readlink(newFile
)
1905 f
= open(newFile
, "r")
1907 for line
in f
.readlines():
1908 newdiff
+= "+" + line
1909 except UnicodeDecodeError:
1910 pass # Found non-text data and skip, since diff description should only include text
1913 return (diff
+ newdiff
).replace('\r\n', '\n')
1915 def applyCommit(self
, id):
1916 """Apply one commit, return True if it succeeded."""
1918 print("Applying", read_pipe(["git", "show", "-s",
1919 "--format=format:%h %s", id]))
1921 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1923 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1925 filesToChangeType
= set()
1926 filesToDelete
= set()
1928 pureRenameCopy
= set()
1930 filesToChangeExecBit
= {}
1934 diff
= parseDiffTreeEntry(line
)
1935 modifier
= diff
['status']
1937 all_files
.append(path
)
1941 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1942 filesToChangeExecBit
[path
] = diff
['dst_mode']
1943 editedFiles
.add(path
)
1944 elif modifier
== "A":
1945 filesToAdd
.add(path
)
1946 filesToChangeExecBit
[path
] = diff
['dst_mode']
1947 if path
in filesToDelete
:
1948 filesToDelete
.remove(path
)
1950 dst_mode
= int(diff
['dst_mode'], 8)
1951 if dst_mode
== 0o120000:
1954 elif modifier
== "D":
1955 filesToDelete
.add(path
)
1956 if path
in filesToAdd
:
1957 filesToAdd
.remove(path
)
1958 elif modifier
== "C":
1959 src
, dest
= diff
['src'], diff
['dst']
1960 all_files
.append(dest
)
1961 p4_integrate(src
, dest
)
1962 pureRenameCopy
.add(dest
)
1963 if diff
['src_sha1'] != diff
['dst_sha1']:
1965 pureRenameCopy
.discard(dest
)
1966 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1968 pureRenameCopy
.discard(dest
)
1969 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1971 # turn off read-only attribute
1972 os
.chmod(dest
, stat
.S_IWRITE
)
1974 editedFiles
.add(dest
)
1975 elif modifier
== "R":
1976 src
, dest
= diff
['src'], diff
['dst']
1977 all_files
.append(dest
)
1978 if self
.p4HasMoveCommand
:
1979 p4_edit(src
) # src must be open before move
1980 p4_move(src
, dest
) # opens for (move/delete, move/add)
1982 p4_integrate(src
, dest
)
1983 if diff
['src_sha1'] != diff
['dst_sha1']:
1986 pureRenameCopy
.add(dest
)
1987 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1988 if not self
.p4HasMoveCommand
:
1989 p4_edit(dest
) # with move: already open, writable
1990 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1991 if not self
.p4HasMoveCommand
:
1993 os
.chmod(dest
, stat
.S_IWRITE
)
1995 filesToDelete
.add(src
)
1996 editedFiles
.add(dest
)
1997 elif modifier
== "T":
1998 filesToChangeType
.add(path
)
2000 die("unknown modifier %s for %s" % (modifier
, path
))
2002 diffcmd
= "git diff-tree --full-index -p \"%s\"" % (id)
2003 patchcmd
= diffcmd
+ " | git apply "
2004 tryPatchCmd
= patchcmd
+ "--check -"
2005 applyPatchCmd
= patchcmd
+ "--check --apply -"
2006 patch_succeeded
= True
2009 print("TryPatch: %s" % tryPatchCmd
)
2011 if os
.system(tryPatchCmd
) != 0:
2012 fixed_rcs_keywords
= False
2013 patch_succeeded
= False
2014 print("Unfortunately applying the change failed!")
2016 # Patch failed, maybe it's just RCS keyword woes. Look through
2017 # the patch to see if that's possible.
2018 if gitConfigBool("git-p4.attemptRCSCleanup"):
2022 for file in editedFiles | filesToDelete
:
2023 # did this file's delta contain RCS keywords?
2024 pattern
= p4_keywords_regexp_for_file(file)
2027 # this file is a possibility...look for RCS keywords.
2028 regexp
= re
.compile(pattern
, re
.VERBOSE
)
2029 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
2030 if regexp
.search(line
):
2032 print("got keyword match on %s in %s in %s" % (pattern
, line
, file))
2033 kwfiles
[file] = pattern
2036 for file in kwfiles
:
2038 print("zapping %s with %s" % (line
,pattern
))
2039 # File is being deleted, so not open in p4. Must
2040 # disable the read-only bit on windows.
2041 if self
.isWindows
and file not in editedFiles
:
2042 os
.chmod(file, stat
.S_IWRITE
)
2043 self
.patchRCSKeywords(file, kwfiles
[file])
2044 fixed_rcs_keywords
= True
2046 if fixed_rcs_keywords
:
2047 print("Retrying the patch with RCS keywords cleaned up")
2048 if os
.system(tryPatchCmd
) == 0:
2049 patch_succeeded
= True
2050 print("Patch succeesed this time with RCS keywords cleaned")
2052 if not patch_succeeded
:
2053 for f
in editedFiles
:
2058 # Apply the patch for real, and do add/delete/+x handling.
2060 system(applyPatchCmd
)
2062 for f
in filesToChangeType
:
2063 p4_edit(f
, "-t", "auto")
2064 for f
in filesToAdd
:
2066 for f
in filesToDelete
:
2070 # Set/clear executable bits
2071 for f
in filesToChangeExecBit
.keys():
2072 mode
= filesToChangeExecBit
[f
]
2073 setP4ExecBit(f
, mode
)
2076 if len(self
.update_shelve
) > 0:
2077 update_shelve
= self
.update_shelve
.pop(0)
2078 p4_reopen_in_change(update_shelve
, all_files
)
2081 # Build p4 change description, starting with the contents
2082 # of the git commit message.
2084 logMessage
= extractLogMessageFromGitCommit(id)
2085 logMessage
= logMessage
.strip()
2086 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
2088 template
= self
.prepareSubmitTemplate(update_shelve
)
2089 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
2091 if self
.preserveUser
:
2092 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
2094 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
2095 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
2096 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
2097 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
2099 separatorLine
= "######## everything below this line is just the diff #######\n"
2100 if not self
.prepare_p4_only
:
2101 submitTemplate
+= separatorLine
2102 submitTemplate
+= self
.get_diff_description(editedFiles
, filesToAdd
, symlinks
)
2104 (handle
, fileName
) = tempfile
.mkstemp()
2105 tmpFile
= os
.fdopen(handle
, "w+b")
2107 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
2108 tmpFile
.write(encode_text_stream(submitTemplate
))
2114 # Allow the hook to edit the changelist text before presenting it
2116 if not run_git_hook("p4-prepare-changelist", [fileName
]):
2119 if self
.prepare_p4_only
:
2121 # Leave the p4 tree prepared, and the submit template around
2122 # and let the user decide what to do next
2126 print("P4 workspace prepared for submission.")
2127 print("To submit or revert, go to client workspace")
2128 print(" " + self
.clientPath
)
2130 print("To submit, use \"p4 submit\" to write a new description,")
2131 print("or \"p4 submit -i <%s\" to use the one prepared by" \
2132 " \"git p4\"." % fileName
)
2133 print("You can delete the file \"%s\" when finished." % fileName
)
2135 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
2136 print("To preserve change ownership by user %s, you must\n" \
2137 "do \"p4 change -f <change>\" after submitting and\n" \
2138 "edit the User field.")
2140 print("After submitting, renamed files must be re-synced.")
2141 print("Invoke \"p4 sync -f\" on each of these files:")
2142 for f
in pureRenameCopy
:
2146 print("To revert the changes, use \"p4 revert ...\", and delete")
2147 print("the submit template file \"%s\"" % fileName
)
2149 print("Since the commit adds new files, they must be deleted:")
2150 for f
in filesToAdd
:
2156 if self
.edit_template(fileName
):
2157 if not self
.no_verify
:
2158 if not run_git_hook("p4-changelist", [fileName
]):
2159 print("The p4-changelist hook failed.")
2163 # read the edited message and submit
2164 tmpFile
= open(fileName
, "rb")
2165 message
= decode_text_stream(tmpFile
.read())
2168 message
= message
.replace("\r\n", "\n")
2169 if message
.find(separatorLine
) != -1:
2170 submitTemplate
= message
[:message
.index(separatorLine
)]
2172 submitTemplate
= message
2174 if len(submitTemplate
.strip()) == 0:
2175 print("Changelist is empty, aborting this changelist.")
2180 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate
)
2182 p4_write_pipe(['shelve', '-i'], submitTemplate
)
2184 p4_write_pipe(['submit', '-i'], submitTemplate
)
2185 # The rename/copy happened by applying a patch that created a
2186 # new file. This leaves it writable, which confuses p4.
2187 for f
in pureRenameCopy
:
2190 if self
.preserveUser
:
2192 # Get last changelist number. Cannot easily get it from
2193 # the submit command output as the output is
2195 changelist
= self
.lastP4Changelist()
2196 self
.modifyChangelistUser(changelist
, p4User
)
2200 run_git_hook("p4-post-changelist")
2202 # Revert changes if we skip this patch
2203 if not submitted
or self
.shelve
:
2205 print ("Reverting shelved files.")
2207 print ("Submission cancelled, undoing p4 changes.")
2209 for f
in editedFiles | filesToDelete
:
2211 for f
in filesToAdd
:
2215 if not self
.prepare_p4_only
:
2219 # Export git tags as p4 labels. Create a p4 label and then tag
2221 def exportGitTags(self
, gitTags
):
2222 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
2223 if len(validLabelRegexp
) == 0:
2224 validLabelRegexp
= defaultLabelRegexp
2225 m
= re
.compile(validLabelRegexp
)
2227 for name
in gitTags
:
2229 if not m
.match(name
):
2231 print("tag %s does not match regexp %s" % (name
, validLabelRegexp
))
2234 # Get the p4 commit this corresponds to
2235 logMessage
= extractLogMessageFromGitCommit(name
)
2236 values
= extractSettingsGitLog(logMessage
)
2238 if 'change' not in values
:
2239 # a tag pointing to something not sent to p4; ignore
2241 print("git tag %s does not give a p4 commit" % name
)
2244 changelist
= values
['change']
2246 # Get the tag details.
2250 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
2253 if re
.match(r
'tag\s+', l
):
2255 elif re
.match(r
'\s*$', l
):
2262 body
= ["lightweight tag imported by git p4\n"]
2264 # Create the label - use the same view as the client spec we are using
2265 clientSpec
= getClientSpec()
2267 labelTemplate
= "Label: %s\n" % name
2268 labelTemplate
+= "Description:\n"
2270 labelTemplate
+= "\t" + b
+ "\n"
2271 labelTemplate
+= "View:\n"
2272 for depot_side
in clientSpec
.mappings
:
2273 labelTemplate
+= "\t%s\n" % depot_side
2276 print("Would create p4 label %s for tag" % name
)
2277 elif self
.prepare_p4_only
:
2278 print("Not creating p4 label %s for tag due to option" \
2279 " --prepare-p4-only" % name
)
2281 p4_write_pipe(["label", "-i"], labelTemplate
)
2284 p4_system(["tag", "-l", name
] +
2285 ["%s@%s" % (depot_side
, changelist
) for depot_side
in clientSpec
.mappings
])
2288 print("created p4 label for tag %s" % name
)
2290 def run(self
, args
):
2292 self
.master
= currentGitBranch()
2293 elif len(args
) == 1:
2294 self
.master
= args
[0]
2295 if not branchExists(self
.master
):
2296 die("Branch %s does not exist" % self
.master
)
2300 for i
in self
.update_shelve
:
2302 sys
.exit("invalid changelist %d" % i
)
2305 allowSubmit
= gitConfig("git-p4.allowSubmit")
2306 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
2307 die("%s is not in git-p4.allowSubmit" % self
.master
)
2309 [upstream
, settings
] = findUpstreamBranchPoint()
2310 self
.depotPath
= settings
['depot-paths'][0]
2311 if len(self
.origin
) == 0:
2312 self
.origin
= upstream
2314 if len(self
.update_shelve
) > 0:
2317 if self
.preserveUser
:
2318 if not self
.canChangeChangelists():
2319 die("Cannot preserve user names without p4 super-user or admin permissions")
2321 # if not set from the command line, try the config file
2322 if self
.conflict_behavior
is None:
2323 val
= gitConfig("git-p4.conflict")
2325 if val
not in self
.conflict_behavior_choices
:
2326 die("Invalid value '%s' for config git-p4.conflict" % val
)
2329 self
.conflict_behavior
= val
2332 print("Origin branch is " + self
.origin
)
2334 if len(self
.depotPath
) == 0:
2335 print("Internal error: cannot locate perforce depot path from existing branches")
2338 self
.useClientSpec
= False
2339 if gitConfigBool("git-p4.useclientspec"):
2340 self
.useClientSpec
= True
2341 if self
.useClientSpec
:
2342 self
.clientSpecDirs
= getClientSpec()
2344 # Check for the existence of P4 branches
2345 branchesDetected
= (len(p4BranchesInGit().keys()) > 1)
2347 if self
.useClientSpec
and not branchesDetected
:
2348 # all files are relative to the client spec
2349 self
.clientPath
= getClientRoot()
2351 self
.clientPath
= p4Where(self
.depotPath
)
2353 if self
.clientPath
== "":
2354 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
2356 print("Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
))
2357 self
.oldWorkingDirectory
= os
.getcwd()
2359 # ensure the clientPath exists
2360 new_client_dir
= False
2361 if not os
.path
.exists(self
.clientPath
):
2362 new_client_dir
= True
2363 os
.makedirs(self
.clientPath
)
2365 chdir(self
.clientPath
, is_client_path
=True)
2367 print("Would synchronize p4 checkout in %s" % self
.clientPath
)
2369 print("Synchronizing p4 checkout...")
2371 # old one was destroyed, and maybe nobody told p4
2372 p4_sync("...", "-f")
2379 committish
= self
.master
2383 if self
.commit
!= "":
2384 if self
.commit
.find("..") != -1:
2385 limits_ish
= self
.commit
.split("..")
2386 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish
[0], limits_ish
[1])]):
2387 commits
.append(line
.strip())
2390 commits
.append(self
.commit
)
2392 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self
.origin
, committish
)]):
2393 commits
.append(line
.strip())
2396 if self
.preserveUser
or gitConfigBool("git-p4.skipUserNameCheck"):
2397 self
.checkAuthorship
= False
2399 self
.checkAuthorship
= True
2401 if self
.preserveUser
:
2402 self
.checkValidP4Users(commits
)
2405 # Build up a set of options to be passed to diff when
2406 # submitting each commit to p4.
2408 if self
.detectRenames
:
2409 # command-line -M arg
2410 self
.diffOpts
= "-M"
2412 # If not explicitly set check the config variable
2413 detectRenames
= gitConfig("git-p4.detectRenames")
2415 if detectRenames
.lower() == "false" or detectRenames
== "":
2417 elif detectRenames
.lower() == "true":
2418 self
.diffOpts
= "-M"
2420 self
.diffOpts
= "-M%s" % detectRenames
2422 # no command-line arg for -C or --find-copies-harder, just
2424 detectCopies
= gitConfig("git-p4.detectCopies")
2425 if detectCopies
.lower() == "false" or detectCopies
== "":
2427 elif detectCopies
.lower() == "true":
2428 self
.diffOpts
+= " -C"
2430 self
.diffOpts
+= " -C%s" % detectCopies
2432 if gitConfigBool("git-p4.detectCopiesHarder"):
2433 self
.diffOpts
+= " --find-copies-harder"
2435 num_shelves
= len(self
.update_shelve
)
2436 if num_shelves
> 0 and num_shelves
!= len(commits
):
2437 sys
.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2438 (len(commits
), num_shelves
))
2440 if not self
.no_verify
:
2442 if not run_git_hook("p4-pre-submit"):
2443 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nYou can skip " \
2444 "this pre-submission check by adding\nthe command line option '--no-verify', " \
2445 "however,\nthis will also skip the p4-changelist hook as well.")
2447 except Exception as e
:
2448 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nThe hook failed "\
2449 "with the error '{0}'".format(e
.message
) )
2453 # Apply the commits, one at a time. On failure, ask if should
2454 # continue to try the rest of the patches, or quit.
2457 print("Would apply")
2459 last
= len(commits
) - 1
2460 for i
, commit
in enumerate(commits
):
2462 print(" ", read_pipe(["git", "show", "-s",
2463 "--format=format:%h %s", commit
]))
2466 ok
= self
.applyCommit(commit
)
2468 applied
.append(commit
)
2469 if self
.prepare_p4_only
:
2471 print("Processing only the first commit due to option" \
2472 " --prepare-p4-only")
2476 # prompt for what to do, or use the option/variable
2477 if self
.conflict_behavior
== "ask":
2478 print("What do you want to do?")
2479 response
= prompt("[s]kip this commit but apply the rest, or [q]uit? ")
2480 elif self
.conflict_behavior
== "skip":
2482 elif self
.conflict_behavior
== "quit":
2485 die("Unknown conflict_behavior '%s'" %
2486 self
.conflict_behavior
)
2489 print("Skipping this commit, but applying the rest")
2494 chdir(self
.oldWorkingDirectory
)
2495 shelved_applied
= "shelved" if self
.shelve
else "applied"
2498 elif self
.prepare_p4_only
:
2500 elif len(commits
) == len(applied
):
2501 print("All commits {0}!".format(shelved_applied
))
2505 sync
.branch
= self
.branch
2506 if self
.disable_p4sync
:
2507 sync
.sync_origin_only()
2511 if not self
.disable_rebase
:
2516 if len(applied
) == 0:
2517 print("No commits {0}.".format(shelved_applied
))
2519 print("{0} only the commits marked with '*':".format(shelved_applied
.capitalize()))
2525 print(star
, read_pipe(["git", "show", "-s",
2526 "--format=format:%h %s", c
]))
2527 print("You will have to do 'git p4 sync' and rebase.")
2529 if gitConfigBool("git-p4.exportLabels"):
2530 self
.exportLabels
= True
2532 if self
.exportLabels
:
2533 p4Labels
= getP4Labels(self
.depotPath
)
2534 gitTags
= getGitTags()
2536 missingGitTags
= gitTags
- p4Labels
2537 self
.exportGitTags(missingGitTags
)
2539 # exit with error unless everything applied perfectly
2540 if len(commits
) != len(applied
):
2546 """Represent a p4 view ("p4 help views"), and map files in a
2547 repo according to the view."""
2549 def __init__(self
, client_name
):
2551 self
.client_prefix
= "//%s/" % client_name
2552 # cache results of "p4 where" to lookup client file locations
2553 self
.client_spec_path_cache
= {}
2555 def append(self
, view_line
):
2556 """Parse a view line, splitting it into depot and client
2557 sides. Append to self.mappings, preserving order. This
2558 is only needed for tag creation."""
2560 # Split the view line into exactly two words. P4 enforces
2561 # structure on these lines that simplifies this quite a bit.
2563 # Either or both words may be double-quoted.
2564 # Single quotes do not matter.
2565 # Double-quote marks cannot occur inside the words.
2566 # A + or - prefix is also inside the quotes.
2567 # There are no quotes unless they contain a space.
2568 # The line is already white-space stripped.
2569 # The two words are separated by a single space.
2571 if view_line
[0] == '"':
2572 # First word is double quoted. Find its end.
2573 close_quote_index
= view_line
.find('"', 1)
2574 if close_quote_index
<= 0:
2575 die("No first-word closing quote found: %s" % view_line
)
2576 depot_side
= view_line
[1:close_quote_index
]
2577 # skip closing quote and space
2578 rhs_index
= close_quote_index
+ 1 + 1
2580 space_index
= view_line
.find(" ")
2581 if space_index
<= 0:
2582 die("No word-splitting space found: %s" % view_line
)
2583 depot_side
= view_line
[0:space_index
]
2584 rhs_index
= space_index
+ 1
2586 # prefix + means overlay on previous mapping
2587 if depot_side
.startswith("+"):
2588 depot_side
= depot_side
[1:]
2590 # prefix - means exclude this path, leave out of mappings
2592 if depot_side
.startswith("-"):
2594 depot_side
= depot_side
[1:]
2597 self
.mappings
.append(depot_side
)
2599 def convert_client_path(self
, clientFile
):
2600 # chop off //client/ part to make it relative
2601 if not decode_path(clientFile
).startswith(self
.client_prefix
):
2602 die("No prefix '%s' on clientFile '%s'" %
2603 (self
.client_prefix
, clientFile
))
2604 return clientFile
[len(self
.client_prefix
):]
2606 def update_client_spec_path_cache(self
, files
):
2607 """ Caching file paths by "p4 where" batch query """
2609 # List depot file paths exclude that already cached
2610 fileArgs
= [f
['path'] for f
in files
if decode_path(f
['path']) not in self
.client_spec_path_cache
]
2612 if len(fileArgs
) == 0:
2613 return # All files in cache
2615 where_result
= p4CmdList(["-x", "-", "where"], stdin
=fileArgs
)
2616 for res
in where_result
:
2617 if "code" in res
and res
["code"] == "error":
2618 # assume error is "... file(s) not in client view"
2620 if "clientFile" not in res
:
2621 die("No clientFile in 'p4 where' output")
2623 # it will list all of them, but only one not unmap-ped
2625 depot_path
= decode_path(res
['depotFile'])
2626 if gitConfigBool("core.ignorecase"):
2627 depot_path
= depot_path
.lower()
2628 self
.client_spec_path_cache
[depot_path
] = self
.convert_client_path(res
["clientFile"])
2630 # not found files or unmap files set to ""
2631 for depotFile
in fileArgs
:
2632 depotFile
= decode_path(depotFile
)
2633 if gitConfigBool("core.ignorecase"):
2634 depotFile
= depotFile
.lower()
2635 if depotFile
not in self
.client_spec_path_cache
:
2636 self
.client_spec_path_cache
[depotFile
] = b
''
2638 def map_in_client(self
, depot_path
):
2639 """Return the relative location in the client where this
2640 depot file should live. Returns "" if the file should
2641 not be mapped in the client."""
2643 if gitConfigBool("core.ignorecase"):
2644 depot_path
= depot_path
.lower()
2646 if depot_path
in self
.client_spec_path_cache
:
2647 return self
.client_spec_path_cache
[depot_path
]
2649 die( "Error: %s is not found in client spec path" % depot_path
)
2652 def cloneExcludeCallback(option
, opt_str
, value
, parser
):
2653 # prepend "/" because the first "/" was consumed as part of the option itself.
2654 # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2655 parser
.values
.cloneExclude
+= ["/" + re
.sub(r
"\.\.\.$", "", value
)]
2657 class P4Sync(Command
, P4UserMap
):
2660 Command
.__init
__(self
)
2661 P4UserMap
.__init
__(self
)
2663 optparse
.make_option("--branch", dest
="branch"),
2664 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
2665 optparse
.make_option("--changesfile", dest
="changesFile"),
2666 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
2667 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
2668 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2669 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
2670 help="Import into refs/heads/ , not refs/remotes"),
2671 optparse
.make_option("--max-changes", dest
="maxChanges",
2672 help="Maximum number of changes to import"),
2673 optparse
.make_option("--changes-block-size", dest
="changes_block_size", type="int",
2674 help="Internal block size to use when iteratively calling p4 changes"),
2675 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
2676 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2677 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
2678 help="Only sync files that are included in the Perforce Client Spec"),
2679 optparse
.make_option("-/", dest
="cloneExclude",
2680 action
="callback", callback
=cloneExcludeCallback
, type="string",
2681 help="exclude depot path"),
2683 self
.description
= """Imports from Perforce into a git repository.\n
2685 //depot/my/project/ -- to import the current head
2686 //depot/my/project/@all -- to import everything
2687 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2689 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2691 self
.usage
+= " //depot/path[@revRange]"
2693 self
.createdBranches
= set()
2694 self
.committedChanges
= set()
2696 self
.detectBranches
= False
2697 self
.detectLabels
= False
2698 self
.importLabels
= False
2699 self
.changesFile
= ""
2700 self
.syncWithOrigin
= True
2701 self
.importIntoRemotes
= True
2702 self
.maxChanges
= ""
2703 self
.changes_block_size
= None
2704 self
.keepRepoPath
= False
2705 self
.depotPaths
= None
2706 self
.p4BranchesInGit
= []
2707 self
.cloneExclude
= []
2708 self
.useClientSpec
= False
2709 self
.useClientSpec_from_options
= False
2710 self
.clientSpecDirs
= None
2711 self
.tempBranches
= []
2712 self
.tempBranchLocation
= "refs/git-p4-tmp"
2713 self
.largeFileSystem
= None
2714 self
.suppress_meta_comment
= False
2716 if gitConfig('git-p4.largeFileSystem'):
2717 largeFileSystemConstructor
= globals()[gitConfig('git-p4.largeFileSystem')]
2718 self
.largeFileSystem
= largeFileSystemConstructor(
2719 lambda git_mode
, relPath
, contents
: self
.writeToGitStream(git_mode
, relPath
, contents
)
2722 if gitConfig("git-p4.syncFromOrigin") == "false":
2723 self
.syncWithOrigin
= False
2725 self
.depotPaths
= []
2726 self
.changeRange
= ""
2727 self
.previousDepotPaths
= []
2728 self
.hasOrigin
= False
2730 # map from branch depot path to parent branch
2731 self
.knownBranches
= {}
2732 self
.initialParents
= {}
2734 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
2737 # Force a checkpoint in fast-import and wait for it to finish
2738 def checkpoint(self
):
2739 self
.gitStream
.write("checkpoint\n\n")
2740 self
.gitStream
.write("progress checkpoint\n\n")
2741 self
.gitStream
.flush()
2742 out
= self
.gitOutput
.readline()
2744 print("checkpoint finished: " + out
)
2746 def isPathWanted(self
, path
):
2747 for p
in self
.cloneExclude
:
2749 if p4PathStartsWith(path
, p
):
2751 # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2752 elif path
.lower() == p
.lower():
2754 for p
in self
.depotPaths
:
2755 if p4PathStartsWith(path
, decode_path(p
)):
2759 def extractFilesFromCommit(self
, commit
, shelved
=False, shelved_cl
= 0):
2762 while "depotFile%s" % fnum
in commit
:
2763 path
= commit
["depotFile%s" % fnum
]
2764 found
= self
.isPathWanted(decode_path(path
))
2771 file["rev"] = commit
["rev%s" % fnum
]
2772 file["action"] = commit
["action%s" % fnum
]
2773 file["type"] = commit
["type%s" % fnum
]
2775 file["shelved_cl"] = int(shelved_cl
)
2780 def extractJobsFromCommit(self
, commit
):
2783 while "job%s" % jnum
in commit
:
2784 job
= commit
["job%s" % jnum
]
2789 def stripRepoPath(self
, path
, prefixes
):
2790 """When streaming files, this is called to map a p4 depot path
2791 to where it should go in git. The prefixes are either
2792 self.depotPaths, or self.branchPrefixes in the case of
2793 branch detection."""
2795 if self
.useClientSpec
:
2796 # branch detection moves files up a level (the branch name)
2797 # from what client spec interpretation gives
2798 path
= decode_path(self
.clientSpecDirs
.map_in_client(path
))
2799 if self
.detectBranches
:
2800 for b
in self
.knownBranches
:
2801 if p4PathStartsWith(path
, b
+ "/"):
2802 path
= path
[len(b
)+1:]
2804 elif self
.keepRepoPath
:
2805 # Preserve everything in relative path name except leading
2806 # //depot/; just look at first prefix as they all should
2807 # be in the same depot.
2808 depot
= re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])
2809 if p4PathStartsWith(path
, depot
):
2810 path
= path
[len(depot
):]
2814 if p4PathStartsWith(path
, p
):
2815 path
= path
[len(p
):]
2818 path
= wildcard_decode(path
)
2821 def splitFilesIntoBranches(self
, commit
):
2822 """Look at each depotFile in the commit to figure out to what
2823 branch it belongs."""
2825 if self
.clientSpecDirs
:
2826 files
= self
.extractFilesFromCommit(commit
)
2827 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2831 while "depotFile%s" % fnum
in commit
:
2832 raw_path
= commit
["depotFile%s" % fnum
]
2833 path
= decode_path(raw_path
)
2834 found
= self
.isPathWanted(path
)
2840 file["path"] = raw_path
2841 file["rev"] = commit
["rev%s" % fnum
]
2842 file["action"] = commit
["action%s" % fnum
]
2843 file["type"] = commit
["type%s" % fnum
]
2846 # start with the full relative path where this file would
2848 if self
.useClientSpec
:
2849 relPath
= decode_path(self
.clientSpecDirs
.map_in_client(path
))
2851 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
2853 for branch
in self
.knownBranches
.keys():
2854 # add a trailing slash so that a commit into qt/4.2foo
2855 # doesn't end up in qt/4.2, e.g.
2856 if p4PathStartsWith(relPath
, branch
+ "/"):
2857 if branch
not in branches
:
2858 branches
[branch
] = []
2859 branches
[branch
].append(file)
2864 def writeToGitStream(self
, gitMode
, relPath
, contents
):
2865 self
.gitStream
.write(encode_text_stream(u
'M {} inline {}\n'.format(gitMode
, relPath
)))
2866 self
.gitStream
.write('data %d\n' % sum(len(d
) for d
in contents
))
2868 self
.gitStream
.write(d
)
2869 self
.gitStream
.write('\n')
2871 def encodeWithUTF8(self
, path
):
2873 path
.decode('ascii')
2876 if gitConfig('git-p4.pathEncoding'):
2877 encoding
= gitConfig('git-p4.pathEncoding')
2878 path
= path
.decode(encoding
, 'replace').encode('utf8', 'replace')
2880 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding
, path
))
2883 # output one file from the P4 stream
2884 # - helper for streamP4Files
2886 def streamOneP4File(self
, file, contents
):
2887 file_path
= file['depotFile']
2888 relPath
= self
.stripRepoPath(decode_path(file_path
), self
.branchPrefixes
)
2891 if 'fileSize' in self
.stream_file
:
2892 size
= int(self
.stream_file
['fileSize'])
2894 size
= 0 # deleted files don't get a fileSize apparently
2895 sys
.stdout
.write('\r%s --> %s (%i MB)\n' % (file_path
, relPath
, size
/1024/1024))
2898 (type_base
, type_mods
) = split_p4_type(file["type"])
2901 if "x" in type_mods
:
2903 if type_base
== "symlink":
2905 # p4 print on a symlink sometimes contains "target\n";
2906 # if it does, remove the newline
2907 data
= ''.join(decode_text_stream(c
) for c
in contents
)
2909 # Some version of p4 allowed creating a symlink that pointed
2910 # to nothing. This causes p4 errors when checking out such
2911 # a change, and errors here too. Work around it by ignoring
2912 # the bad symlink; hopefully a future change fixes it.
2913 print("\nIgnoring empty symlink in %s" % file_path
)
2915 elif data
[-1] == '\n':
2916 contents
= [data
[:-1]]
2920 if type_base
== "utf16":
2921 # p4 delivers different text in the python output to -G
2922 # than it does when using "print -o", or normal p4 client
2923 # operations. utf16 is converted to ascii or utf8, perhaps.
2924 # But ascii text saved as -t utf16 is completely mangled.
2925 # Invoke print -o to get the real contents.
2927 # On windows, the newlines will always be mangled by print, so put
2928 # them back too. This is not needed to the cygwin windows version,
2929 # just the native "NT" type.
2932 text
= p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (decode_path(file['depotFile']), file['change'])], raw
=True)
2933 except Exception as e
:
2934 if 'Translation of file content failed' in str(e
):
2935 type_base
= 'binary'
2939 if p4_version_string().find('/NT') >= 0:
2940 text
= text
.replace(b
'\r\n', b
'\n')
2943 if type_base
== "apple":
2944 # Apple filetype files will be streamed as a concatenation of
2945 # its appledouble header and the contents. This is useless
2946 # on both macs and non-macs. If using "print -q -o xx", it
2947 # will create "xx" with the data, and "%xx" with the header.
2948 # This is also not very useful.
2950 # Ideally, someday, this script can learn how to generate
2951 # appledouble files directly and import those to git, but
2952 # non-mac machines can never find a use for apple filetype.
2953 print("\nIgnoring apple filetype file %s" % file['depotFile'])
2956 # Note that we do not try to de-mangle keywords on utf16 files,
2957 # even though in theory somebody may want that.
2958 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
2960 regexp
= re
.compile(pattern
, re
.VERBOSE
)
2961 text
= ''.join(decode_text_stream(c
) for c
in contents
)
2962 text
= regexp
.sub(r
'$\1$', text
)
2963 contents
= [ encode_text_stream(text
) ]
2965 if self
.largeFileSystem
:
2966 (git_mode
, contents
) = self
.largeFileSystem
.processContent(git_mode
, relPath
, contents
)
2968 self
.writeToGitStream(git_mode
, relPath
, contents
)
2970 def streamOneP4Deletion(self
, file):
2971 relPath
= self
.stripRepoPath(decode_path(file['path']), self
.branchPrefixes
)
2973 sys
.stdout
.write("delete %s\n" % relPath
)
2975 self
.gitStream
.write(encode_text_stream(u
'D {}\n'.format(relPath
)))
2977 if self
.largeFileSystem
and self
.largeFileSystem
.isLargeFile(relPath
):
2978 self
.largeFileSystem
.removeLargeFile(relPath
)
2980 # handle another chunk of streaming data
2981 def streamP4FilesCb(self
, marshalled
):
2983 # catch p4 errors and complain
2985 if "code" in marshalled
:
2986 if marshalled
["code"] == "error":
2987 if "data" in marshalled
:
2988 err
= marshalled
["data"].rstrip()
2990 if not err
and 'fileSize' in self
.stream_file
:
2991 required_bytes
= int((4 * int(self
.stream_file
["fileSize"])) - calcDiskFree())
2992 if required_bytes
> 0:
2993 err
= 'Not enough space left on %s! Free at least %i MB.' % (
2994 os
.getcwd(), required_bytes
/1024/1024
2999 if self
.stream_have_file_info
:
3000 if "depotFile" in self
.stream_file
:
3001 f
= self
.stream_file
["depotFile"]
3002 # force a failure in fast-import, else an empty
3003 # commit will be made
3004 self
.gitStream
.write("\n")
3005 self
.gitStream
.write("die-now\n")
3006 self
.gitStream
.close()
3007 # ignore errors, but make sure it exits first
3008 self
.importProcess
.wait()
3010 die("Error from p4 print for %s: %s" % (f
, err
))
3012 die("Error from p4 print: %s" % err
)
3014 if 'depotFile' in marshalled
and self
.stream_have_file_info
:
3015 # start of a new file - output the old one first
3016 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
3017 self
.stream_file
= {}
3018 self
.stream_contents
= []
3019 self
.stream_have_file_info
= False
3021 # pick up the new file information... for the
3022 # 'data' field we need to append to our array
3023 for k
in marshalled
.keys():
3025 if 'streamContentSize' not in self
.stream_file
:
3026 self
.stream_file
['streamContentSize'] = 0
3027 self
.stream_file
['streamContentSize'] += len(marshalled
['data'])
3028 self
.stream_contents
.append(marshalled
['data'])
3030 self
.stream_file
[k
] = marshalled
[k
]
3033 'streamContentSize' in self
.stream_file
and
3034 'fileSize' in self
.stream_file
and
3035 'depotFile' in self
.stream_file
):
3036 size
= int(self
.stream_file
["fileSize"])
3038 progress
= 100*self
.stream_file
['streamContentSize']/size
3039 sys
.stdout
.write('\r%s %d%% (%i MB)' % (self
.stream_file
['depotFile'], progress
, int(size
/1024/1024)))
3042 self
.stream_have_file_info
= True
3044 # Stream directly from "p4 files" into "git fast-import"
3045 def streamP4Files(self
, files
):
3051 filesForCommit
.append(f
)
3052 if f
['action'] in self
.delete_actions
:
3053 filesToDelete
.append(f
)
3055 filesToRead
.append(f
)
3058 for f
in filesToDelete
:
3059 self
.streamOneP4Deletion(f
)
3061 if len(filesToRead
) > 0:
3062 self
.stream_file
= {}
3063 self
.stream_contents
= []
3064 self
.stream_have_file_info
= False
3066 # curry self argument
3067 def streamP4FilesCbSelf(entry
):
3068 self
.streamP4FilesCb(entry
)
3071 for f
in filesToRead
:
3072 if 'shelved_cl' in f
:
3073 # Handle shelved CLs using the "p4 print file@=N" syntax to print
3075 fileArg
= f
['path'] + encode_text_stream('@={}'.format(f
['shelved_cl']))
3077 fileArg
= f
['path'] + encode_text_stream('#{}'.format(f
['rev']))
3079 fileArgs
.append(fileArg
)
3081 p4CmdList(["-x", "-", "print"],
3083 cb
=streamP4FilesCbSelf
)
3086 if 'depotFile' in self
.stream_file
:
3087 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
3089 def make_email(self
, userid
):
3090 if userid
in self
.users
:
3091 return self
.users
[userid
]
3093 return "%s <a@b>" % userid
3095 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
3096 """ Stream a p4 tag.
3097 commit is either a git commit, or a fast-import mark, ":<p4commit>"
3101 print("writing tag %s for commit %s" % (labelName
, commit
))
3102 gitStream
.write("tag %s\n" % labelName
)
3103 gitStream
.write("from %s\n" % commit
)
3105 if 'Owner' in labelDetails
:
3106 owner
= labelDetails
["Owner"]
3110 # Try to use the owner of the p4 label, or failing that,
3111 # the current p4 user id.
3113 email
= self
.make_email(owner
)
3115 email
= self
.make_email(self
.p4UserId())
3116 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
3118 gitStream
.write("tagger %s\n" % tagger
)
3120 print("labelDetails=",labelDetails
)
3121 if 'Description' in labelDetails
:
3122 description
= labelDetails
['Description']
3124 description
= 'Label from git p4'
3126 gitStream
.write("data %d\n" % len(description
))
3127 gitStream
.write(description
)
3128 gitStream
.write("\n")
3130 def inClientSpec(self
, path
):
3131 if not self
.clientSpecDirs
:
3133 inClientSpec
= self
.clientSpecDirs
.map_in_client(path
)
3134 if not inClientSpec
and self
.verbose
:
3135 print('Ignoring file outside of client spec: {0}'.format(path
))
3138 def hasBranchPrefix(self
, path
):
3139 if not self
.branchPrefixes
:
3141 hasPrefix
= [p
for p
in self
.branchPrefixes
3142 if p4PathStartsWith(path
, p
)]
3143 if not hasPrefix
and self
.verbose
:
3144 print('Ignoring file outside of prefix: {0}'.format(path
))
3147 def findShadowedFiles(self
, files
, change
):
3148 # Perforce allows you commit files and directories with the same name,
3149 # so you could have files //depot/foo and //depot/foo/bar both checked
3150 # in. A p4 sync of a repository in this state fails. Deleting one of
3151 # the files recovers the repository.
3153 # Git will not allow the broken state to exist and only the most recent
3154 # of the conflicting names is left in the repository. When one of the
3155 # conflicting files is deleted we need to re-add the other one to make
3156 # sure the git repository recovers in the same way as perforce.
3157 deleted
= [f
for f
in files
if f
['action'] in self
.delete_actions
]
3160 path
= decode_path(f
['path'])
3161 to_check
.add(path
+ '/...')
3163 path
= path
.rsplit("/", 1)[0]
3164 if path
== "/" or path
in to_check
:
3167 to_check
= ['%s@%s' % (wildcard_encode(p
), change
) for p
in to_check
3168 if self
.hasBranchPrefix(p
)]
3170 stat_result
= p4CmdList(["-x", "-", "fstat", "-T",
3171 "depotFile,headAction,headRev,headType"], stdin
=to_check
)
3172 for record
in stat_result
:
3173 if record
['code'] != 'stat':
3175 if record
['headAction'] in self
.delete_actions
:
3179 'path': record
['depotFile'],
3180 'rev': record
['headRev'],
3181 'type': record
['headType']})
3183 def commit(self
, details
, files
, branch
, parent
= "", allow_empty
=False):
3184 epoch
= details
["time"]
3185 author
= details
["user"]
3186 jobs
= self
.extractJobsFromCommit(details
)
3189 print('commit into {0}'.format(branch
))
3191 files
= [f
for f
in files
3192 if self
.hasBranchPrefix(decode_path(f
['path']))]
3193 self
.findShadowedFiles(files
, details
['change'])
3195 if self
.clientSpecDirs
:
3196 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
3198 files
= [f
for f
in files
if self
.inClientSpec(decode_path(f
['path']))]
3200 if gitConfigBool('git-p4.keepEmptyCommits'):
3203 if not files
and not allow_empty
:
3204 print('Ignoring revision {0} as it would produce an empty commit.'
3205 .format(details
['change']))
3208 self
.gitStream
.write("commit %s\n" % branch
)
3209 self
.gitStream
.write("mark :%s\n" % details
["change"])
3210 self
.committedChanges
.add(int(details
["change"]))
3212 if author
not in self
.users
:
3213 self
.getUserMapFromPerforceServer()
3214 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
3216 self
.gitStream
.write("committer %s\n" % committer
)
3218 self
.gitStream
.write("data <<EOT\n")
3219 self
.gitStream
.write(details
["desc"])
3221 self
.gitStream
.write("\nJobs: %s" % (' '.join(jobs
)))
3223 if not self
.suppress_meta_comment
:
3224 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3225 (','.join(self
.branchPrefixes
), details
["change"]))
3226 if len(details
['options']) > 0:
3227 self
.gitStream
.write(": options = %s" % details
['options'])
3228 self
.gitStream
.write("]\n")
3230 self
.gitStream
.write("EOT\n\n")
3234 print("parent %s" % parent
)
3235 self
.gitStream
.write("from %s\n" % parent
)
3237 self
.streamP4Files(files
)
3238 self
.gitStream
.write("\n")
3240 change
= int(details
["change"])
3242 if change
in self
.labels
:
3243 label
= self
.labels
[change
]
3244 labelDetails
= label
[0]
3245 labelRevisions
= label
[1]
3247 print("Change %s is labelled %s" % (change
, labelDetails
))
3249 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
3250 for p
in self
.branchPrefixes
])
3252 if len(files
) == len(labelRevisions
):
3256 if info
["action"] in self
.delete_actions
:
3258 cleanedFiles
[info
["depotFile"]] = info
["rev"]
3260 if cleanedFiles
== labelRevisions
:
3261 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
3265 print("Tag %s does not match with change %s: files do not match."
3266 % (labelDetails
["label"], change
))
3270 print("Tag %s does not match with change %s: file count is different."
3271 % (labelDetails
["label"], change
))
3273 # Build a dictionary of changelists and labels, for "detect-labels" option.
3274 def getLabels(self
):
3277 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
3278 if len(l
) > 0 and not self
.silent
:
3279 print("Finding files belonging to labels in %s" % self
.depotPaths
)
3282 label
= output
["label"]
3286 print("Querying files for label %s" % label
)
3287 for file in p4CmdList(["files"] +
3288 ["%s...@%s" % (p
, label
)
3289 for p
in self
.depotPaths
]):
3290 revisions
[file["depotFile"]] = file["rev"]
3291 change
= int(file["change"])
3292 if change
> newestChange
:
3293 newestChange
= change
3295 self
.labels
[newestChange
] = [output
, revisions
]
3298 print("Label changes: %s" % self
.labels
.keys())
3300 # Import p4 labels as git tags. A direct mapping does not
3301 # exist, so assume that if all the files are at the same revision
3302 # then we can use that, or it's something more complicated we should
3304 def importP4Labels(self
, stream
, p4Labels
):
3306 print("import p4 labels: " + ' '.join(p4Labels
))
3308 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
3309 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
3310 if len(validLabelRegexp
) == 0:
3311 validLabelRegexp
= defaultLabelRegexp
3312 m
= re
.compile(validLabelRegexp
)
3314 for name
in p4Labels
:
3317 if not m
.match(name
):
3319 print("label %s does not match regexp %s" % (name
,validLabelRegexp
))
3322 if name
in ignoredP4Labels
:
3325 labelDetails
= p4CmdList(['label', "-o", name
])[0]
3327 # get the most recent changelist for each file in this label
3328 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
3329 for p
in self
.depotPaths
])
3331 if 'change' in change
:
3332 # find the corresponding git commit; take the oldest commit
3333 changelist
= int(change
['change'])
3334 if changelist
in self
.committedChanges
:
3335 gitCommit
= ":%d" % changelist
# use a fast-import mark
3338 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
3339 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
], ignore_error
=True)
3340 if len(gitCommit
) == 0:
3341 print("importing label %s: could not find git commit for changelist %d" % (name
, changelist
))
3344 gitCommit
= gitCommit
.strip()
3347 # Convert from p4 time format
3349 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
3351 print("Could not convert label time %s" % labelDetails
['Update'])
3354 when
= int(time
.mktime(tmwhen
))
3355 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
3357 print("p4 label %s mapped to git commit %s" % (name
, gitCommit
))
3360 print("Label %s has no changelists - possibly deleted?" % name
)
3363 # We can't import this label; don't try again as it will get very
3364 # expensive repeatedly fetching all the files for labels that will
3365 # never be imported. If the label is moved in the future, the
3366 # ignore will need to be removed manually.
3367 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
3369 def guessProjectName(self
):
3370 for p
in self
.depotPaths
:
3373 p
= p
[p
.strip().rfind("/") + 1:]
3374 if not p
.endswith("/"):
3378 def getBranchMapping(self
):
3379 lostAndFoundBranches
= set()
3381 user
= gitConfig("git-p4.branchUser")
3383 command
= "branches -u %s" % user
3385 command
= "branches"
3387 for info
in p4CmdList(command
):
3388 details
= p4Cmd(["branch", "-o", info
["branch"]])
3390 while "View%s" % viewIdx
in details
:
3391 paths
= details
["View%s" % viewIdx
].split(" ")
3392 viewIdx
= viewIdx
+ 1
3393 # require standard //depot/foo/... //depot/bar/... mapping
3394 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
3397 destination
= paths
[1]
3399 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
3400 source
= source
[len(self
.depotPaths
[0]):-4]
3401 destination
= destination
[len(self
.depotPaths
[0]):-4]
3403 if destination
in self
.knownBranches
:
3405 print("p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
))
3406 print("but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
))
3409 self
.knownBranches
[destination
] = source
3411 lostAndFoundBranches
.discard(destination
)
3413 if source
not in self
.knownBranches
:
3414 lostAndFoundBranches
.add(source
)
3416 # Perforce does not strictly require branches to be defined, so we also
3417 # check git config for a branch list.
3419 # Example of branch definition in git config file:
3421 # branchList=main:branchA
3422 # branchList=main:branchB
3423 # branchList=branchA:branchC
3424 configBranches
= gitConfigList("git-p4.branchList")
3425 for branch
in configBranches
:
3427 (source
, destination
) = branch
.split(":")
3428 self
.knownBranches
[destination
] = source
3430 lostAndFoundBranches
.discard(destination
)
3432 if source
not in self
.knownBranches
:
3433 lostAndFoundBranches
.add(source
)
3436 for branch
in lostAndFoundBranches
:
3437 self
.knownBranches
[branch
] = branch
3439 def getBranchMappingFromGitBranches(self
):
3440 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3441 for branch
in branches
.keys():
3442 if branch
== "master":
3445 branch
= branch
[len(self
.projectName
):]
3446 self
.knownBranches
[branch
] = branch
3448 def updateOptionDict(self
, d
):
3450 if self
.keepRepoPath
:
3451 option_keys
['keepRepoPath'] = 1
3453 d
["options"] = ' '.join(sorted(option_keys
.keys()))
3455 def readOptions(self
, d
):
3456 self
.keepRepoPath
= ('options' in d
3457 and ('keepRepoPath' in d
['options']))
3459 def gitRefForBranch(self
, branch
):
3460 if branch
== "main":
3461 return self
.refPrefix
+ "master"
3463 if len(branch
) <= 0:
3466 return self
.refPrefix
+ self
.projectName
+ branch
3468 def gitCommitByP4Change(self
, ref
, change
):
3470 print("looking in ref " + ref
+ " for change %s using bisect..." % change
)
3473 latestCommit
= parseRevision(ref
)
3477 print("trying: earliest %s latest %s" % (earliestCommit
, latestCommit
))
3478 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
3483 log
= extractLogMessageFromGitCommit(next
)
3484 settings
= extractSettingsGitLog(log
)
3485 currentChange
= int(settings
['change'])
3487 print("current change %s" % currentChange
)
3489 if currentChange
== change
:
3491 print("found %s" % next
)
3494 if currentChange
< change
:
3495 earliestCommit
= "^%s" % next
3497 if next
== latestCommit
:
3498 die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref
, change
))
3499 latestCommit
= "%s^@" % next
3503 def importNewBranch(self
, branch
, maxChange
):
3504 # make fast-import flush all changes to disk and update the refs using the checkpoint
3505 # command so that we can try to find the branch parent in the git history
3506 self
.gitStream
.write("checkpoint\n\n");
3507 self
.gitStream
.flush();
3508 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3509 range = "@1,%s" % maxChange
3510 #print "prefix" + branchPrefix
3511 changes
= p4ChangesForPaths([branchPrefix
], range, self
.changes_block_size
)
3512 if len(changes
) <= 0:
3514 firstChange
= changes
[0]
3515 #print "first change in branch: %s" % firstChange
3516 sourceBranch
= self
.knownBranches
[branch
]
3517 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
3518 sourceRef
= self
.gitRefForBranch(sourceBranch
)
3519 #print "source " + sourceBranch
3521 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
3522 #print "branch parent: %s" % branchParentChange
3523 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
3524 if len(gitParent
) > 0:
3525 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
3526 #print "parent git commit: %s" % gitParent
3528 self
.importChanges(changes
)
3531 def searchParent(self
, parent
, branch
, target
):
3532 targetTree
= read_pipe(["git", "rev-parse",
3533 "{}^{{tree}}".format(target
)]).strip()
3534 for line
in read_pipe_lines(["git", "rev-list", "--format=%H %T",
3535 "--no-merges", parent
]):
3536 if line
.startswith("commit "):
3538 commit
, tree
= line
.strip().split(" ")
3539 if tree
== targetTree
:
3541 print("Found parent of %s in commit %s" % (branch
, commit
))
3545 def importChanges(self
, changes
, origin_revision
=0):
3547 for change
in changes
:
3548 description
= p4_describe(change
)
3549 self
.updateOptionDict(description
)
3552 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
3557 if self
.detectBranches
:
3558 branches
= self
.splitFilesIntoBranches(description
)
3559 for branch
in branches
.keys():
3561 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3562 self
.branchPrefixes
= [ branchPrefix
]
3566 filesForCommit
= branches
[branch
]
3569 print("branch is %s" % branch
)
3571 self
.updatedBranches
.add(branch
)
3573 if branch
not in self
.createdBranches
:
3574 self
.createdBranches
.add(branch
)
3575 parent
= self
.knownBranches
[branch
]
3576 if parent
== branch
:
3579 fullBranch
= self
.projectName
+ branch
3580 if fullBranch
not in self
.p4BranchesInGit
:
3582 print("\n Importing new branch %s" % fullBranch
);
3583 if self
.importNewBranch(branch
, change
- 1):
3585 self
.p4BranchesInGit
.append(fullBranch
)
3587 print("\n Resuming with change %s" % change
);
3590 print("parent determined through known branches: %s" % parent
)
3592 branch
= self
.gitRefForBranch(branch
)
3593 parent
= self
.gitRefForBranch(parent
)
3596 print("looking for initial parent for %s; current parent is %s" % (branch
, parent
))
3598 if len(parent
) == 0 and branch
in self
.initialParents
:
3599 parent
= self
.initialParents
[branch
]
3600 del self
.initialParents
[branch
]
3604 tempBranch
= "%s/%d" % (self
.tempBranchLocation
, change
)
3606 print("Creating temporary branch: " + tempBranch
)
3607 self
.commit(description
, filesForCommit
, tempBranch
)
3608 self
.tempBranches
.append(tempBranch
)
3610 blob
= self
.searchParent(parent
, branch
, tempBranch
)
3612 self
.commit(description
, filesForCommit
, branch
, blob
)
3615 print("Parent of %s not found. Committing into head of %s" % (branch
, parent
))
3616 self
.commit(description
, filesForCommit
, branch
, parent
)
3618 files
= self
.extractFilesFromCommit(description
)
3619 self
.commit(description
, files
, self
.branch
,
3621 # only needed once, to connect to the previous commit
3622 self
.initialParent
= ""
3624 print(self
.gitError
.read())
3627 def sync_origin_only(self
):
3628 if self
.syncWithOrigin
:
3629 self
.hasOrigin
= originP4BranchesExist()
3632 print('Syncing with origin first, using "git fetch origin"')
3633 system("git fetch origin")
3635 def importHeadRevision(self
, revision
):
3636 print("Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
))
3639 details
["user"] = "git perforce import user"
3640 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
3641 % (' '.join(self
.depotPaths
), revision
))
3642 details
["change"] = revision
3646 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
3648 for info
in p4CmdList(["files"] + fileArgs
):
3650 if 'code' in info
and info
['code'] == 'error':
3651 sys
.stderr
.write("p4 returned an error: %s\n"
3653 if info
['data'].find("must refer to client") >= 0:
3654 sys
.stderr
.write("This particular p4 error is misleading.\n")
3655 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
3656 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
3658 if 'p4ExitCode' in info
:
3659 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
3663 change
= int(info
["change"])
3664 if change
> newestRevision
:
3665 newestRevision
= change
3667 if info
["action"] in self
.delete_actions
:
3668 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3669 #fileCnt = fileCnt + 1
3672 for prop
in ["depotFile", "rev", "action", "type" ]:
3673 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
3675 fileCnt
= fileCnt
+ 1
3677 details
["change"] = newestRevision
3679 # Use time from top-most change so that all git p4 clones of
3680 # the same p4 repo have the same commit SHA1s.
3681 res
= p4_describe(newestRevision
)
3682 details
["time"] = res
["time"]
3684 self
.updateOptionDict(details
)
3686 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
3687 except IOError as err
:
3688 print("IO error with git fast-import. Is your git version recent enough?")
3689 print("IO error details: {}".format(err
))
3690 print(self
.gitError
.read())
3693 def importRevisions(self
, args
, branch_arg_given
):
3696 if len(self
.changesFile
) > 0:
3697 with
open(self
.changesFile
) as f
:
3698 output
= f
.readlines()
3701 changeSet
.add(int(line
))
3703 for change
in changeSet
:
3704 changes
.append(change
)
3708 # catch "git p4 sync" with no new branches, in a repo that
3709 # does not have any existing p4 branches
3711 if not self
.p4BranchesInGit
:
3712 raise P4CommandException("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3714 # The default branch is master, unless --branch is used to
3715 # specify something else. Make sure it exists, or complain
3716 # nicely about how to use --branch.
3717 if not self
.detectBranches
:
3718 if not branch_exists(self
.branch
):
3719 if branch_arg_given
:
3720 raise P4CommandException("Error: branch %s does not exist." % self
.branch
)
3722 raise P4CommandException("Error: no branch %s; perhaps specify one with --branch." %
3726 print("Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
3728 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
, self
.changes_block_size
)
3730 if len(self
.maxChanges
) > 0:
3731 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
3733 if len(changes
) == 0:
3735 print("No changes to import!")
3737 if not self
.silent
and not self
.detectBranches
:
3738 print("Import destination: %s" % self
.branch
)
3740 self
.updatedBranches
= set()
3742 if not self
.detectBranches
:
3744 # start a new branch
3745 self
.initialParent
= ""
3747 # build on a previous revision
3748 self
.initialParent
= parseRevision(self
.branch
)
3750 self
.importChanges(changes
)
3754 if len(self
.updatedBranches
) > 0:
3755 sys
.stdout
.write("Updated branches: ")
3756 for b
in self
.updatedBranches
:
3757 sys
.stdout
.write("%s " % b
)
3758 sys
.stdout
.write("\n")
3760 def openStreams(self
):
3761 self
.importProcess
= subprocess
.Popen(["git", "fast-import"],
3762 stdin
=subprocess
.PIPE
,
3763 stdout
=subprocess
.PIPE
,
3764 stderr
=subprocess
.PIPE
);
3765 self
.gitOutput
= self
.importProcess
.stdout
3766 self
.gitStream
= self
.importProcess
.stdin
3767 self
.gitError
= self
.importProcess
.stderr
3769 if bytes
is not str:
3770 # Wrap gitStream.write() so that it can be called using `str` arguments
3771 def make_encoded_write(write
):
3772 def encoded_write(s
):
3773 return write(s
.encode() if isinstance(s
, str) else s
)
3774 return encoded_write
3776 self
.gitStream
.write
= make_encoded_write(self
.gitStream
.write
)
3778 def closeStreams(self
):
3779 if self
.gitStream
is None:
3781 self
.gitStream
.close()
3782 if self
.importProcess
.wait() != 0:
3783 die("fast-import failed: %s" % self
.gitError
.read())
3784 self
.gitOutput
.close()
3785 self
.gitError
.close()
3786 self
.gitStream
= None
3788 def run(self
, args
):
3789 if self
.importIntoRemotes
:
3790 self
.refPrefix
= "refs/remotes/p4/"
3792 self
.refPrefix
= "refs/heads/p4/"
3794 self
.sync_origin_only()
3796 branch_arg_given
= bool(self
.branch
)
3797 if len(self
.branch
) == 0:
3798 self
.branch
= self
.refPrefix
+ "master"
3799 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
3800 system("git update-ref %s refs/heads/p4" % self
.branch
)
3801 system("git branch -D p4")
3803 # accept either the command-line option, or the configuration variable
3804 if self
.useClientSpec
:
3805 # will use this after clone to set the variable
3806 self
.useClientSpec_from_options
= True
3808 if gitConfigBool("git-p4.useclientspec"):
3809 self
.useClientSpec
= True
3810 if self
.useClientSpec
:
3811 self
.clientSpecDirs
= getClientSpec()
3813 # TODO: should always look at previous commits,
3814 # merge with previous imports, if possible.
3817 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
3819 # branches holds mapping from branch name to sha1
3820 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3822 # restrict to just this one, disabling detect-branches
3823 if branch_arg_given
:
3824 short
= self
.branch
.split("/")[-1]
3825 if short
in branches
:
3826 self
.p4BranchesInGit
= [ short
]
3828 self
.p4BranchesInGit
= branches
.keys()
3830 if len(self
.p4BranchesInGit
) > 1:
3832 print("Importing from/into multiple branches")
3833 self
.detectBranches
= True
3834 for branch
in branches
.keys():
3835 self
.initialParents
[self
.refPrefix
+ branch
] = \
3839 print("branches: %s" % self
.p4BranchesInGit
)
3842 for branch
in self
.p4BranchesInGit
:
3843 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
3845 settings
= extractSettingsGitLog(logMsg
)
3847 self
.readOptions(settings
)
3848 if ('depot-paths' in settings
3849 and 'change' in settings
):
3850 change
= int(settings
['change']) + 1
3851 p4Change
= max(p4Change
, change
)
3853 depotPaths
= sorted(settings
['depot-paths'])
3854 if self
.previousDepotPaths
== []:
3855 self
.previousDepotPaths
= depotPaths
3858 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
3859 prev_list
= prev
.split("/")
3860 cur_list
= cur
.split("/")
3861 for i
in range(0, min(len(cur_list
), len(prev_list
))):
3862 if cur_list
[i
] != prev_list
[i
]:
3866 paths
.append ("/".join(cur_list
[:i
+ 1]))
3868 self
.previousDepotPaths
= paths
3871 self
.depotPaths
= sorted(self
.previousDepotPaths
)
3872 self
.changeRange
= "@%s,#head" % p4Change
3873 if not self
.silent
and not self
.detectBranches
:
3874 print("Performing incremental import into %s git branch" % self
.branch
)
3876 # accept multiple ref name abbreviations:
3877 # refs/foo/bar/branch -> use it exactly
3878 # p4/branch -> prepend refs/remotes/ or refs/heads/
3879 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3880 if not self
.branch
.startswith("refs/"):
3881 if self
.importIntoRemotes
:
3882 prepend
= "refs/remotes/"
3884 prepend
= "refs/heads/"
3885 if not self
.branch
.startswith("p4/"):
3887 self
.branch
= prepend
+ self
.branch
3889 if len(args
) == 0 and self
.depotPaths
:
3891 print("Depot paths: %s" % ' '.join(self
.depotPaths
))
3893 if self
.depotPaths
and self
.depotPaths
!= args
:
3894 print("previous import used depot path %s and now %s was specified. "
3895 "This doesn't work!" % (' '.join (self
.depotPaths
),
3899 self
.depotPaths
= sorted(args
)
3904 # Make sure no revision specifiers are used when --changesfile
3906 bad_changesfile
= False
3907 if len(self
.changesFile
) > 0:
3908 for p
in self
.depotPaths
:
3909 if p
.find("@") >= 0 or p
.find("#") >= 0:
3910 bad_changesfile
= True
3913 die("Option --changesfile is incompatible with revision specifiers")
3916 for p
in self
.depotPaths
:
3917 if p
.find("@") != -1:
3918 atIdx
= p
.index("@")
3919 self
.changeRange
= p
[atIdx
:]
3920 if self
.changeRange
== "@all":
3921 self
.changeRange
= ""
3922 elif ',' not in self
.changeRange
:
3923 revision
= self
.changeRange
3924 self
.changeRange
= ""
3926 elif p
.find("#") != -1:
3927 hashIdx
= p
.index("#")
3928 revision
= p
[hashIdx
:]
3930 elif self
.previousDepotPaths
== []:
3931 # pay attention to changesfile, if given, else import
3932 # the entire p4 tree at the head revision
3933 if len(self
.changesFile
) == 0:
3936 p
= re
.sub ("\.\.\.$", "", p
)
3937 if not p
.endswith("/"):
3942 self
.depotPaths
= newPaths
3944 # --detect-branches may change this for each branch
3945 self
.branchPrefixes
= self
.depotPaths
3947 self
.loadUserMapFromCache()
3949 if self
.detectLabels
:
3952 if self
.detectBranches
:
3953 ## FIXME - what's a P4 projectName ?
3954 self
.projectName
= self
.guessProjectName()
3957 self
.getBranchMappingFromGitBranches()
3959 self
.getBranchMapping()
3961 print("p4-git branches: %s" % self
.p4BranchesInGit
)
3962 print("initial parents: %s" % self
.initialParents
)
3963 for b
in self
.p4BranchesInGit
:
3967 b
= b
[len(self
.projectName
):]
3968 self
.createdBranches
.add(b
)
3978 self
.importHeadRevision(revision
)
3980 self
.importRevisions(args
, branch_arg_given
)
3982 if gitConfigBool("git-p4.importLabels"):
3983 self
.importLabels
= True
3985 if self
.importLabels
:
3986 p4Labels
= getP4Labels(self
.depotPaths
)
3987 gitTags
= getGitTags()
3989 missingP4Labels
= p4Labels
- gitTags
3990 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
3992 except P4CommandException
as e
:
4001 # Cleanup temporary branches created during import
4002 if self
.tempBranches
!= []:
4003 for branch
in self
.tempBranches
:
4004 read_pipe("git update-ref -d %s" % branch
)
4005 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
4007 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
4008 # a convenient shortcut refname "p4".
4009 if self
.importIntoRemotes
:
4010 head_ref
= self
.refPrefix
+ "HEAD"
4011 if not gitBranchExists(head_ref
) and gitBranchExists(self
.branch
):
4012 system(["git", "symbolic-ref", head_ref
, self
.branch
])
4016 class P4Rebase(Command
):
4018 Command
.__init
__(self
)
4020 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
4022 self
.importLabels
= False
4023 self
.description
= ("Fetches the latest revision from perforce and "
4024 + "rebases the current work (branch) against it")
4026 def run(self
, args
):
4028 sync
.importLabels
= self
.importLabels
4031 return self
.rebase()
4034 if os
.system("git update-index --refresh") != 0:
4035 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
4036 if len(read_pipe("git diff-index HEAD --")) > 0:
4037 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
4039 [upstream
, settings
] = findUpstreamBranchPoint()
4040 if len(upstream
) == 0:
4041 die("Cannot find upstream branchpoint for rebase")
4043 # the branchpoint may be p4/foo~3, so strip off the parent
4044 upstream
= re
.sub("~[0-9]+$", "", upstream
)
4046 print("Rebasing the current branch onto %s" % upstream
)
4047 oldHead
= read_pipe("git rev-parse HEAD").strip()
4048 system("git rebase %s" % upstream
)
4049 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead
)
4052 class P4Clone(P4Sync
):
4054 P4Sync
.__init
__(self
)
4055 self
.description
= "Creates a new git repository and imports from Perforce into it"
4056 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
4058 optparse
.make_option("--destination", dest
="cloneDestination",
4059 action
='store', default
=None,
4060 help="where to leave result of the clone"),
4061 optparse
.make_option("--bare", dest
="cloneBare",
4062 action
="store_true", default
=False),
4064 self
.cloneDestination
= None
4065 self
.needsGit
= False
4066 self
.cloneBare
= False
4068 def defaultDestination(self
, args
):
4069 ## TODO: use common prefix of args?
4071 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
4072 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
4073 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
4074 depotDir
= re
.sub(r
"/$", "", depotDir
)
4075 return os
.path
.split(depotDir
)[1]
4077 def run(self
, args
):
4081 if self
.keepRepoPath
and not self
.cloneDestination
:
4082 sys
.stderr
.write("Must specify destination for --keep-path\n")
4087 if not self
.cloneDestination
and len(depotPaths
) > 1:
4088 self
.cloneDestination
= depotPaths
[-1]
4089 depotPaths
= depotPaths
[:-1]
4091 for p
in depotPaths
:
4092 if not p
.startswith("//"):
4093 sys
.stderr
.write('Depot paths must start with "//": %s\n' % p
)
4096 if not self
.cloneDestination
:
4097 self
.cloneDestination
= self
.defaultDestination(args
)
4099 print("Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
))
4101 if not os
.path
.exists(self
.cloneDestination
):
4102 os
.makedirs(self
.cloneDestination
)
4103 chdir(self
.cloneDestination
)
4105 init_cmd
= [ "git", "init" ]
4107 init_cmd
.append("--bare")
4108 retcode
= subprocess
.call(init_cmd
)
4110 raise CalledProcessError(retcode
, init_cmd
)
4112 if not P4Sync
.run(self
, depotPaths
):
4115 # create a master branch and check out a work tree
4116 if gitBranchExists(self
.branch
):
4117 system([ "git", "branch", currentGitBranch(), self
.branch
])
4118 if not self
.cloneBare
:
4119 system([ "git", "checkout", "-f" ])
4121 print('Not checking out any branch, use ' \
4122 '"git checkout -q -b master <branch>"')
4124 # auto-set this variable if invoked with --use-client-spec
4125 if self
.useClientSpec_from_options
:
4126 system("git config --bool git-p4.useclientspec true")
4130 class P4Unshelve(Command
):
4132 Command
.__init
__(self
)
4134 self
.origin
= "HEAD"
4135 self
.description
= "Unshelve a P4 changelist into a git commit"
4136 self
.usage
= "usage: %prog [options] changelist"
4138 optparse
.make_option("--origin", dest
="origin",
4139 help="Use this base revision instead of the default (%s)" % self
.origin
),
4141 self
.verbose
= False
4142 self
.noCommit
= False
4143 self
.destbranch
= "refs/remotes/p4-unshelved"
4145 def renameBranch(self
, branch_name
):
4146 """ Rename the existing branch to branch_name.N
4150 for i
in range(0,1000):
4151 backup_branch_name
= "{0}.{1}".format(branch_name
, i
)
4152 if not gitBranchExists(backup_branch_name
):
4153 gitUpdateRef(backup_branch_name
, branch_name
) # copy ref to backup
4154 gitDeleteRef(branch_name
)
4156 print("renamed old unshelve branch to {0}".format(backup_branch_name
))
4160 sys
.exit("gave up trying to rename existing branch {0}".format(sync
.branch
))
4162 def findLastP4Revision(self
, starting_point
):
4163 """ Look back from starting_point for the first commit created by git-p4
4164 to find the P4 commit we are based on, and the depot-paths.
4167 for parent
in (range(65535)):
4168 log
= extractLogMessageFromGitCommit("{0}~{1}".format(starting_point
, parent
))
4169 settings
= extractSettingsGitLog(log
)
4170 if 'change' in settings
:
4173 sys
.exit("could not find git-p4 commits in {0}".format(self
.origin
))
4175 def createShelveParent(self
, change
, branch_name
, sync
, origin
):
4176 """ Create a commit matching the parent of the shelved changelist 'change'
4178 parent_description
= p4_describe(change
, shelved
=True)
4179 parent_description
['desc'] = 'parent for shelved changelist {}\n'.format(change
)
4180 files
= sync
.extractFilesFromCommit(parent_description
, shelved
=False, shelved_cl
=change
)
4184 # if it was added in the shelved changelist, it won't exist in the parent
4185 if f
['action'] in self
.add_actions
:
4188 # if it was deleted in the shelved changelist it must not be deleted
4189 # in the parent - we might even need to create it if the origin branch
4191 if f
['action'] in self
.delete_actions
:
4194 parent_files
.append(f
)
4196 sync
.commit(parent_description
, parent_files
, branch_name
,
4197 parent
=origin
, allow_empty
=True)
4198 print("created parent commit for {0} based on {1} in {2}".format(
4199 change
, self
.origin
, branch_name
))
4201 def run(self
, args
):
4205 if not gitBranchExists(self
.origin
):
4206 sys
.exit("origin branch {0} does not exist".format(self
.origin
))
4211 # only one change at a time
4214 # if the target branch already exists, rename it
4215 branch_name
= "{0}/{1}".format(self
.destbranch
, change
)
4216 if gitBranchExists(branch_name
):
4217 self
.renameBranch(branch_name
)
4218 sync
.branch
= branch_name
4220 sync
.verbose
= self
.verbose
4221 sync
.suppress_meta_comment
= True
4223 settings
= self
.findLastP4Revision(self
.origin
)
4224 sync
.depotPaths
= settings
['depot-paths']
4225 sync
.branchPrefixes
= sync
.depotPaths
4228 sync
.loadUserMapFromCache()
4231 # create a commit for the parent of the shelved changelist
4232 self
.createShelveParent(change
, branch_name
, sync
, self
.origin
)
4234 # create the commit for the shelved changelist itself
4235 description
= p4_describe(change
, True)
4236 files
= sync
.extractFilesFromCommit(description
, True, change
)
4238 sync
.commit(description
, files
, branch_name
, "")
4241 print("unshelved changelist {0} into {1}".format(change
, branch_name
))
4245 class P4Branches(Command
):
4247 Command
.__init
__(self
)
4249 self
.description
= ("Shows the git branches that hold imports and their "
4250 + "corresponding perforce depot paths")
4251 self
.verbose
= False
4253 def run(self
, args
):
4254 if originP4BranchesExist():
4255 createOrUpdateBranchesFromOrigin()
4257 cmdline
= "git rev-parse --symbolic "
4258 cmdline
+= " --remotes"
4260 for line
in read_pipe_lines(cmdline
):
4263 if not line
.startswith('p4/') or line
== "p4/HEAD":
4267 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
4268 settings
= extractSettingsGitLog(log
)
4270 print("%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"]))
4273 class HelpFormatter(optparse
.IndentedHelpFormatter
):
4275 optparse
.IndentedHelpFormatter
.__init
__(self
)
4277 def format_description(self
, description
):
4279 return description
+ "\n"
4283 def printUsage(commands
):
4284 print("usage: %s <command> [options]" % sys
.argv
[0])
4286 print("valid commands: %s" % ", ".join(commands
))
4288 print("Try %s <command> --help for command specific help." % sys
.argv
[0])
4292 "submit" : P4Submit
,
4293 "commit" : P4Submit
,
4295 "rebase" : P4Rebase
,
4297 "branches" : P4Branches
,
4298 "unshelve" : P4Unshelve
,
4302 if len(sys
.argv
[1:]) == 0:
4303 printUsage(commands
.keys())
4306 cmdName
= sys
.argv
[1]
4308 klass
= commands
[cmdName
]
4311 print("unknown command %s" % cmdName
)
4313 printUsage(commands
.keys())
4316 options
= cmd
.options
4317 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
4321 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
4323 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
4325 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
4327 description
= cmd
.description
,
4328 formatter
= HelpFormatter())
4331 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
4337 verbose
= cmd
.verbose
4339 if cmd
.gitdir
== None:
4340 cmd
.gitdir
= os
.path
.abspath(".git")
4341 if not isValidGitDir(cmd
.gitdir
):
4342 # "rev-parse --git-dir" without arguments will try $PWD/.git
4343 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
4344 if os
.path
.exists(cmd
.gitdir
):
4345 cdup
= read_pipe("git rev-parse --show-cdup").strip()
4349 if not isValidGitDir(cmd
.gitdir
):
4350 if isValidGitDir(cmd
.gitdir
+ "/.git"):
4351 cmd
.gitdir
+= "/.git"
4353 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
4355 # so git commands invoked from the P4 workspace will succeed
4356 os
.environ
["GIT_DIR"] = cmd
.gitdir
4358 if not cmd
.run(args
):
4363 if __name__
== '__main__':