3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 # pylint: disable=invalid-name,missing-docstring,too-many-arguments,broad-except
11 # pylint: disable=no-self-use,wrong-import-position,consider-iterating-dictionary
12 # pylint: disable=wrong-import-order,unused-import,too-few-public-methods
13 # pylint: disable=too-many-lines,ungrouped-imports,fixme,too-many-locals
14 # pylint: disable=line-too-long,bad-whitespace,superfluous-parens
15 # pylint: disable=too-many-statements,too-many-instance-attributes
16 # pylint: disable=too-many-branches,too-many-nested-blocks
19 if sys
.version_info
.major
< 3 and sys
.version_info
.minor
< 7:
20 sys
.stderr
.write("git-p4: requires Python 2.7 or later.\n")
39 # On python2.7 where raw_input() and input() are both availble,
40 # we want raw_input's semantics, but aliased to input for python3
42 # support basestring in python3
44 if raw_input and input:
51 # Only labels/tags matching this will be imported/exported
52 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
54 # The block size is reduced automatically if required
55 defaultBlockSize
= 1<<20
57 p4_access_checked
= False
59 re_ko_keywords
= re
.compile(br
'\$(Id|Header)(:[^$\n]+)?\$')
60 re_k_keywords
= re
.compile(br
'\$(Id|Header|Author|Date|DateTime|Change|File|Revision)(:[^$\n]+)?\$')
62 def format_size_human_readable(num
):
63 """ Returns a number of units (typically bytes) formatted as a human-readable
67 return '{:d} B'.format(num
)
68 for unit
in ["Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
71 return "{:3.1f} {}B".format(num
, unit
)
72 return "{:.1f} YiB".format(num
)
74 def p4_build_cmd(cmd
):
75 """Build a suitable p4 command line.
77 This consolidates building and returning a p4 command line into one
78 location. It means that hooking into the environment, or other configuration
79 can be done more easily.
83 user
= gitConfig("git-p4.user")
85 real_cmd
+= ["-u",user
]
87 password
= gitConfig("git-p4.password")
89 real_cmd
+= ["-P", password
]
91 port
= gitConfig("git-p4.port")
93 real_cmd
+= ["-p", port
]
95 host
= gitConfig("git-p4.host")
97 real_cmd
+= ["-H", host
]
99 client
= gitConfig("git-p4.client")
101 real_cmd
+= ["-c", client
]
103 retries
= gitConfigInt("git-p4.retries")
105 # Perform 3 retries by default
108 # Provide a way to not pass this option by setting git-p4.retries to 0
109 real_cmd
+= ["-r", str(retries
)]
113 # now check that we can actually talk to the server
114 global p4_access_checked
115 if not p4_access_checked
:
116 p4_access_checked
= True # suppress access checks in p4_check_access itself
122 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
123 This won't automatically add ".git" to a directory.
125 d
= read_pipe(["git", "--git-dir", path
, "rev-parse", "--git-dir"], True).strip()
126 if not d
or len(d
) == 0:
131 def chdir(path
, is_client_path
=False):
132 """Do chdir to the given path, and set the PWD environment
133 variable for use by P4. It does not look at getcwd() output.
134 Since we're not using the shell, it is necessary to set the
135 PWD environment variable explicitly.
137 Normally, expand the path to force it to be absolute. This
138 addresses the use of relative path names inside P4 settings,
139 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
140 as given; it looks for .p4config using PWD.
142 If is_client_path, the path was handed to us directly by p4,
143 and may be a symbolic link. Do not call os.getcwd() in this
144 case, because it will cause p4 to think that PWD is not inside
149 if not is_client_path
:
151 os
.environ
['PWD'] = path
154 """Return free space in bytes on the disk of the given dirname."""
155 if platform
.system() == 'Windows':
156 free_bytes
= ctypes
.c_ulonglong(0)
157 ctypes
.windll
.kernel32
.GetDiskFreeSpaceExW(ctypes
.c_wchar_p(os
.getcwd()), None, None, ctypes
.pointer(free_bytes
))
158 return free_bytes
.value
160 st
= os
.statvfs(os
.getcwd())
161 return st
.f_bavail
* st
.f_frsize
164 """ Terminate execution. Make sure that any running child processes have been wait()ed for before
170 sys
.stderr
.write(msg
+ "\n")
173 def prompt(prompt_text
):
174 """ Prompt the user to choose one of the choices
176 Choices are identified in the prompt_text by square brackets around
177 a single letter option.
179 choices
= set(m
.group(1) for m
in re
.finditer(r
"\[(.)\]", prompt_text
))
182 sys
.stdout
.write(prompt_text
)
184 response
=sys
.stdin
.readline().strip().lower()
187 response
= response
[0]
188 if response
in choices
:
191 # We need different encoding/decoding strategies for text data being passed
192 # around in pipes depending on python version
194 # For python3, always encode and decode as appropriate
195 def decode_text_stream(s
):
196 return s
.decode() if isinstance(s
, bytes
) else s
197 def encode_text_stream(s
):
198 return s
.encode() if isinstance(s
, str) else s
200 # For python2.7, pass read strings as-is, but also allow writing unicode
201 def decode_text_stream(s
):
203 def encode_text_stream(s
):
204 return s
.encode('utf_8') if isinstance(s
, unicode) else s
206 def decode_path(path
):
207 """Decode a given string (bytes or otherwise) using configured path encoding options
209 encoding
= gitConfig('git-p4.pathEncoding') or 'utf_8'
211 return path
.decode(encoding
, errors
='replace') if isinstance(path
, bytes
) else path
216 path
= path
.decode(encoding
, errors
='replace')
218 print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding
, path
))
221 def run_git_hook(cmd
, param
=[]):
222 """Execute a hook if the hook exists."""
224 sys
.stderr
.write("Looking for hook: %s\n" % cmd
)
227 hooks_path
= gitConfig("core.hooksPath")
228 if len(hooks_path
) <= 0:
229 hooks_path
= os
.path
.join(os
.environ
["GIT_DIR"], "hooks")
231 if not isinstance(param
, list):
234 # resolve hook file name, OS depdenent
235 hook_file
= os
.path
.join(hooks_path
, cmd
)
236 if platform
.system() == 'Windows':
237 if not os
.path
.isfile(hook_file
):
238 # look for the file with an extension
239 files
= glob
.glob(hook_file
+ ".*")
243 hook_file
= files
.pop()
244 while hook_file
.upper().endswith(".SAMPLE"):
245 # The file is a sample hook. We don't want it
247 hook_file
= files
.pop()
251 if not os
.path
.isfile(hook_file
) or not os
.access(hook_file
, os
.X_OK
):
254 return run_hook_command(hook_file
, param
) == 0
256 def run_hook_command(cmd
, param
):
257 """Executes a git hook command
258 cmd = the command line file to be executed. This can be
259 a file that is run by OS association.
261 param = a list of parameters to pass to the cmd command
263 On windows, the extension is checked to see if it should
264 be run with the Git for Windows Bash shell. If there
265 is no file extension, the file is deemed a bash shell
266 and will be handed off to sh.exe. Otherwise, Windows
267 will be called with the shell to handle the file assocation.
269 For non Windows operating systems, the file is called
274 if platform
.system() == 'Windows':
275 (root
,ext
) = os
.path
.splitext(cmd
)
277 exe_path
= os
.environ
.get("EXEPATH")
281 exe_path
= os
.path
.join(exe_path
, "bin")
282 cli
= [os
.path
.join(exe_path
, "SH.EXE")] + cli
285 return subprocess
.call(cli
, shell
=use_shell
)
288 def write_pipe(c
, stdin
, *k
, **kw
):
290 sys
.stderr
.write('Writing pipe: {}\n'.format(' '.join(c
)))
292 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, *k
, **kw
)
294 val
= pipe
.write(stdin
)
297 die('Command failed: {}'.format(' '.join(c
)))
301 def p4_write_pipe(c
, stdin
, *k
, **kw
):
302 real_cmd
= p4_build_cmd(c
)
303 if bytes
is not str and isinstance(stdin
, str):
304 stdin
= encode_text_stream(stdin
)
305 return write_pipe(real_cmd
, stdin
, *k
, **kw
)
307 def read_pipe_full(c
, *k
, **kw
):
308 """ Read output from command. Returns a tuple
309 of the return status, stdout text and stderr
313 sys
.stderr
.write('Reading pipe: {}\n'.format(' '.join(c
)))
315 p
= subprocess
.Popen(
316 c
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, *k
, **kw
)
317 (out
, err
) = p
.communicate()
318 return (p
.returncode
, out
, decode_text_stream(err
))
320 def read_pipe(c
, ignore_error
=False, raw
=False, *k
, **kw
):
321 """ Read output from command. Returns the output text on
322 success. On failure, terminates execution, unless
323 ignore_error is True, when it returns an empty string.
325 If raw is True, do not attempt to decode output text.
327 (retcode
, out
, err
) = read_pipe_full(c
, *k
, **kw
)
332 die('Command failed: {}\nError: {}'.format(' '.join(c
), err
))
334 out
= decode_text_stream(out
)
337 def read_pipe_text(c
, *k
, **kw
):
338 """ Read output from a command with trailing whitespace stripped.
339 On error, returns None.
341 (retcode
, out
, err
) = read_pipe_full(c
, *k
, **kw
)
345 return decode_text_stream(out
).rstrip()
347 def p4_read_pipe(c
, ignore_error
=False, raw
=False, *k
, **kw
):
348 real_cmd
= p4_build_cmd(c
)
349 return read_pipe(real_cmd
, ignore_error
, raw
=raw
, *k
, **kw
)
351 def read_pipe_lines(c
, raw
=False, *k
, **kw
):
353 sys
.stderr
.write('Reading pipe: {}\n'.format(' '.join(c
)))
355 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, *k
, **kw
)
357 lines
= pipe
.readlines()
359 lines
= [decode_text_stream(line
) for line
in lines
]
360 if pipe
.close() or p
.wait():
361 die('Command failed: {}'.format(' '.join(c
)))
364 def p4_read_pipe_lines(c
, *k
, **kw
):
365 """Specifically invoke p4 on the command supplied. """
366 real_cmd
= p4_build_cmd(c
)
367 return read_pipe_lines(real_cmd
, *k
, **kw
)
369 def p4_has_command(cmd
):
370 """Ask p4 for help on this command. If it returns an error, the
371 command does not exist in this version of p4."""
372 real_cmd
= p4_build_cmd(["help", cmd
])
373 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
374 stderr
=subprocess
.PIPE
)
376 return p
.returncode
== 0
378 def p4_has_move_command():
379 """See if the move command exists, that it supports -k, and that
380 it has not been administratively disabled. The arguments
381 must be correct, but the filenames do not have to exist. Use
382 ones with wildcards so even if they exist, it will fail."""
384 if not p4_has_command("move"):
386 cmd
= p4_build_cmd(["move", "-k", "@from", "@to"])
387 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
388 (out
, err
) = p
.communicate()
389 err
= decode_text_stream(err
)
390 # return code will be 1 in either case
391 if err
.find("Invalid option") >= 0:
393 if err
.find("disabled") >= 0:
395 # assume it failed because @... was invalid changelist
398 def system(cmd
, ignore_error
=False, *k
, **kw
):
400 sys
.stderr
.write("executing {}\n".format(
401 ' '.join(cmd
) if isinstance(cmd
, list) else cmd
))
402 retcode
= subprocess
.call(cmd
, *k
, **kw
)
403 if retcode
and not ignore_error
:
404 raise subprocess
.CalledProcessError(retcode
, cmd
)
408 def p4_system(cmd
, *k
, **kw
):
409 """Specifically invoke p4 as the system command. """
410 real_cmd
= p4_build_cmd(cmd
)
411 retcode
= subprocess
.call(real_cmd
, *k
, **kw
)
413 raise subprocess
.CalledProcessError(retcode
, real_cmd
)
415 def die_bad_access(s
):
416 die("failure accessing depot: {0}".format(s
.rstrip()))
418 def p4_check_access(min_expiration
=1):
419 """ Check if we can access Perforce - account still logged in
421 results
= p4CmdList(["login", "-s"])
423 if len(results
) == 0:
424 # should never get here: always get either some results, or a p4ExitCode
425 assert("could not parse response from perforce")
429 if 'p4ExitCode' in result
:
430 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
431 die_bad_access("could not run p4")
433 code
= result
.get("code")
435 # we get here if we couldn't connect and there was nothing to unmarshal
436 die_bad_access("could not connect")
439 expiry
= result
.get("TicketExpiration")
442 if expiry
> min_expiration
:
446 die_bad_access("perforce ticket expires in {0} seconds".format(expiry
))
449 # account without a timeout - all ok
452 elif code
== "error":
453 data
= result
.get("data")
455 die_bad_access("p4 error: {0}".format(data
))
457 die_bad_access("unknown error")
461 die_bad_access("unknown error code {0}".format(code
))
463 _p4_version_string
= None
464 def p4_version_string():
465 """Read the version string, showing just the last line, which
466 hopefully is the interesting version bit.
469 Perforce - The Fast Software Configuration Management System.
470 Copyright 1995-2011 Perforce Software. All rights reserved.
471 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
473 global _p4_version_string
474 if not _p4_version_string
:
475 a
= p4_read_pipe_lines(["-V"])
476 _p4_version_string
= a
[-1].rstrip()
477 return _p4_version_string
479 def p4_integrate(src
, dest
):
480 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
482 def p4_sync(f
, *options
):
483 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
486 # forcibly add file names with wildcards
487 if wildcard_present(f
):
488 p4_system(["add", "-f", f
])
490 p4_system(["add", f
])
493 p4_system(["delete", wildcard_encode(f
)])
495 def p4_edit(f
, *options
):
496 p4_system(["edit"] + list(options
) + [wildcard_encode(f
)])
499 p4_system(["revert", wildcard_encode(f
)])
501 def p4_reopen(type, f
):
502 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
504 def p4_reopen_in_change(changelist
, files
):
505 cmd
= ["reopen", "-c", str(changelist
)] + files
508 def p4_move(src
, dest
):
509 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
511 def p4_last_change():
512 results
= p4CmdList(["changes", "-m", "1"], skip_info
=True)
513 return int(results
[0]['change'])
515 def p4_describe(change
, shelved
=False):
516 """Make sure it returns a valid result by checking for
517 the presence of field "time". Return a dict of the
520 cmd
= ["describe", "-s"]
525 ds
= p4CmdList(cmd
, skip_info
=True)
527 die("p4 describe -s %d did not return 1 result: %s" % (change
, str(ds
)))
531 if "p4ExitCode" in d
:
532 die("p4 describe -s %d exited with %d: %s" % (change
, d
["p4ExitCode"],
535 if d
["code"] == "error":
536 die("p4 describe -s %d returned error code: %s" % (change
, str(d
)))
539 die("p4 describe -s %d returned no \"time\": %s" % (change
, str(d
)))
544 # Canonicalize the p4 type and return a tuple of the
545 # base type, plus any modifiers. See "p4 help filetypes"
546 # for a list and explanation.
548 def split_p4_type(p4type
):
550 p4_filetypes_historical
= {
551 "ctempobj": "binary+Sw",
557 "tempobj": "binary+FSw",
558 "ubinary": "binary+F",
559 "uresource": "resource+F",
560 "uxbinary": "binary+Fx",
561 "xbinary": "binary+x",
563 "xtempobj": "binary+Swx",
565 "xunicode": "unicode+x",
568 if p4type
in p4_filetypes_historical
:
569 p4type
= p4_filetypes_historical
[p4type
]
571 s
= p4type
.split("+")
579 # return the raw p4 type of a file (text, text+ko, etc)
582 results
= p4CmdList(["fstat", "-T", "headType", wildcard_encode(f
)])
583 return results
[0]['headType']
586 # Given a type base and modifier, return a regexp matching
587 # the keywords that can be expanded in the file
589 def p4_keywords_regexp_for_type(base
, type_mods
):
590 if base
in ("text", "unicode", "binary"):
591 if "ko" in type_mods
:
592 return re_ko_keywords
593 elif "k" in type_mods
:
601 # Given a file, return a regexp matching the possible
602 # RCS keywords that will be expanded, or None for files
603 # with kw expansion turned off.
605 def p4_keywords_regexp_for_file(file):
606 if not os
.path
.exists(file):
609 (type_base
, type_mods
) = split_p4_type(p4_type(file))
610 return p4_keywords_regexp_for_type(type_base
, type_mods
)
612 def setP4ExecBit(file, mode
):
613 # Reopens an already open file and changes the execute bit to match
614 # the execute bit setting in the passed in mode.
618 if not isModeExec(mode
):
619 p4Type
= getP4OpenedType(file)
620 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
621 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
622 if p4Type
[-1] == "+":
623 p4Type
= p4Type
[0:-1]
625 p4_reopen(p4Type
, file)
627 def getP4OpenedType(file):
628 # Returns the perforce file type for the given file.
630 result
= p4_read_pipe(["opened", wildcard_encode(file)])
631 match
= re
.match(".*\((.+)\)( \*exclusive\*)?\r?$", result
)
633 return match
.group(1)
635 die("Could not determine file type for %s (result: '%s')" % (file, result
))
637 # Return the set of all p4 labels
638 def getP4Labels(depotPaths
):
640 if not isinstance(depotPaths
, list):
641 depotPaths
= [depotPaths
]
643 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
649 # Return the set of all git tags
652 for line
in read_pipe_lines(["git", "tag"]):
657 _diff_tree_pattern
= None
659 def parseDiffTreeEntry(entry
):
660 """Parses a single diff tree entry into its component elements.
662 See git-diff-tree(1) manpage for details about the format of the diff
663 output. This method returns a dictionary with the following elements:
665 src_mode - The mode of the source file
666 dst_mode - The mode of the destination file
667 src_sha1 - The sha1 for the source file
668 dst_sha1 - The sha1 fr the destination file
669 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
670 status_score - The score for the status (applicable for 'C' and 'R'
671 statuses). This is None if there is no score.
672 src - The path for the source file.
673 dst - The path for the destination file. This is only present for
674 copy or renames. If it is not present, this is None.
676 If the pattern is not matched, None is returned."""
678 global _diff_tree_pattern
679 if not _diff_tree_pattern
:
680 _diff_tree_pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
682 match
= _diff_tree_pattern
.match(entry
)
685 'src_mode': match
.group(1),
686 'dst_mode': match
.group(2),
687 'src_sha1': match
.group(3),
688 'dst_sha1': match
.group(4),
689 'status': match
.group(5),
690 'status_score': match
.group(6),
691 'src': match
.group(7),
692 'dst': match
.group(10)
696 def isModeExec(mode
):
697 # Returns True if the given git mode represents an executable file,
699 return mode
[-3:] == "755"
701 class P4Exception(Exception):
702 """ Base class for exceptions from the p4 client """
703 def __init__(self
, exit_code
):
704 self
.p4ExitCode
= exit_code
706 class P4ServerException(P4Exception
):
707 """ Base class for exceptions where we get some kind of marshalled up result from the server """
708 def __init__(self
, exit_code
, p4_result
):
709 super(P4ServerException
, self
).__init
__(exit_code
)
710 self
.p4_result
= p4_result
711 self
.code
= p4_result
[0]['code']
712 self
.data
= p4_result
[0]['data']
714 class P4RequestSizeException(P4ServerException
):
715 """ One of the maxresults or maxscanrows errors """
716 def __init__(self
, exit_code
, p4_result
, limit
):
717 super(P4RequestSizeException
, self
).__init
__(exit_code
, p4_result
)
720 class P4CommandException(P4Exception
):
721 """ Something went wrong calling p4 which means we have to give up """
722 def __init__(self
, msg
):
728 def isModeExecChanged(src_mode
, dst_mode
):
729 return isModeExec(src_mode
) != isModeExec(dst_mode
)
731 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None, skip_info
=False,
732 errors_as_exceptions
=False, *k
, **kw
):
734 cmd
= p4_build_cmd(["-G"] + cmd
)
736 sys
.stderr
.write("Opening pipe: {}\n".format(' '.join(cmd
)))
738 # Use a temporary file to avoid deadlocks without
739 # subprocess.communicate(), which would put another copy
740 # of stdout into memory.
742 if stdin
is not None:
743 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
744 if not isinstance(stdin
, list):
745 stdin_file
.write(stdin
)
748 stdin_file
.write(encode_text_stream(i
))
749 stdin_file
.write(b
'\n')
753 p4
= subprocess
.Popen(
754 cmd
, stdin
=stdin_file
, stdout
=subprocess
.PIPE
, *k
, **kw
)
759 entry
= marshal
.load(p4
.stdout
)
761 # Decode unmarshalled dict to use str keys and values, except for:
762 # - `data` which may contain arbitrary binary data
763 # - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text
765 for key
, value
in entry
.items():
767 if isinstance(value
, bytes
) and not (key
in ('data', 'path', 'clientFile') or key
.startswith('depotFile')):
768 value
= value
.decode()
769 decoded_entry
[key
] = value
770 # Parse out data if it's an error response
771 if decoded_entry
.get('code') == 'error' and 'data' in decoded_entry
:
772 decoded_entry
['data'] = decoded_entry
['data'].decode()
773 entry
= decoded_entry
775 if 'code' in entry
and entry
['code'] == 'info':
785 if errors_as_exceptions
:
787 data
= result
[0].get('data')
789 m
= re
.search('Too many rows scanned \(over (\d+)\)', data
)
791 m
= re
.search('Request too large \(over (\d+)\)', data
)
794 limit
= int(m
.group(1))
795 raise P4RequestSizeException(exitCode
, result
, limit
)
797 raise P4ServerException(exitCode
, result
)
799 raise P4Exception(exitCode
)
802 entry
["p4ExitCode"] = exitCode
807 def p4Cmd(cmd
, *k
, **kw
):
808 list = p4CmdList(cmd
, *k
, **kw
)
814 def p4Where(depotPath
):
815 if not depotPath
.endswith("/"):
817 depotPathLong
= depotPath
+ "..."
818 outputList
= p4CmdList(["where", depotPathLong
])
820 for entry
in outputList
:
821 if "depotFile" in entry
:
822 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
823 # The base path always ends with "/...".
824 entry_path
= decode_path(entry
['depotFile'])
825 if entry_path
.find(depotPath
) == 0 and entry_path
[-4:] == "/...":
828 elif "data" in entry
:
829 data
= entry
.get("data")
830 space
= data
.find(" ")
831 if data
[:space
] == depotPath
:
836 if output
["code"] == "error":
840 clientPath
= decode_path(output
['path'])
841 elif "data" in output
:
842 data
= output
.get("data")
843 lastSpace
= data
.rfind(b
" ")
844 clientPath
= decode_path(data
[lastSpace
+ 1:])
846 if clientPath
.endswith("..."):
847 clientPath
= clientPath
[:-3]
850 def currentGitBranch():
851 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
853 def isValidGitDir(path
):
854 return git_dir(path
) != None
856 def parseRevision(ref
):
857 return read_pipe(["git", "rev-parse", ref
]).strip()
859 def branchExists(ref
):
860 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
864 def extractLogMessageFromGitCommit(commit
):
867 ## fixme: title is first line of commit, not 1st paragraph.
869 for log
in read_pipe_lines(["git", "cat-file", "commit", commit
]):
878 def extractSettingsGitLog(log
):
880 for line
in log
.split("\n"):
882 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
886 assignments
= m
.group(1).split (':')
887 for a
in assignments
:
889 key
= vals
[0].strip()
890 val
= ('='.join (vals
[1:])).strip()
891 if val
.endswith ('\"') and val
.startswith('"'):
896 paths
= values
.get("depot-paths")
898 paths
= values
.get("depot-path")
900 values
['depot-paths'] = paths
.split(',')
903 def gitBranchExists(branch
):
904 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
905 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
906 return proc
.wait() == 0;
908 def gitUpdateRef(ref
, newvalue
):
909 subprocess
.check_call(["git", "update-ref", ref
, newvalue
])
911 def gitDeleteRef(ref
):
912 subprocess
.check_call(["git", "update-ref", "-d", ref
])
916 def gitConfig(key
, typeSpecifier
=None):
917 if key
not in _gitConfig
:
918 cmd
= [ "git", "config" ]
920 cmd
+= [ typeSpecifier
]
922 s
= read_pipe(cmd
, ignore_error
=True)
923 _gitConfig
[key
] = s
.strip()
924 return _gitConfig
[key
]
926 def gitConfigBool(key
):
927 """Return a bool, using git config --bool. It is True only if the
928 variable is set to true, and False if set to false or not present
931 if key
not in _gitConfig
:
932 _gitConfig
[key
] = gitConfig(key
, '--bool') == "true"
933 return _gitConfig
[key
]
935 def gitConfigInt(key
):
936 if key
not in _gitConfig
:
937 cmd
= [ "git", "config", "--int", key
]
938 s
= read_pipe(cmd
, ignore_error
=True)
941 _gitConfig
[key
] = int(gitConfig(key
, '--int'))
943 _gitConfig
[key
] = None
944 return _gitConfig
[key
]
946 def gitConfigList(key
):
947 if key
not in _gitConfig
:
948 s
= read_pipe(["git", "config", "--get-all", key
], ignore_error
=True)
949 _gitConfig
[key
] = s
.strip().splitlines()
950 if _gitConfig
[key
] == ['']:
952 return _gitConfig
[key
]
954 def p4BranchesInGit(branchesAreInRemotes
=True):
955 """Find all the branches whose names start with "p4/", looking
956 in remotes or heads as specified by the argument. Return
957 a dictionary of { branch: revision } for each one found.
958 The branch names are the short names, without any
963 cmdline
= ["git", "rev-parse", "--symbolic"]
964 if branchesAreInRemotes
:
965 cmdline
.append("--remotes")
967 cmdline
.append("--branches")
969 for line
in read_pipe_lines(cmdline
):
973 if not line
.startswith('p4/'):
975 # special symbolic ref to p4/master
976 if line
== "p4/HEAD":
979 # strip off p4/ prefix
980 branch
= line
[len("p4/"):]
982 branches
[branch
] = parseRevision(line
)
986 def branch_exists(branch
):
987 """Make sure that the given ref name really exists."""
989 cmd
= [ "git", "rev-parse", "--symbolic", "--verify", branch
]
990 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
991 out
, _
= p
.communicate()
992 out
= decode_text_stream(out
)
995 # expect exactly one line of output: the branch name
996 return out
.rstrip() == branch
998 def findUpstreamBranchPoint(head
= "HEAD"):
999 branches
= p4BranchesInGit()
1000 # map from depot-path to branch name
1001 branchByDepotPath
= {}
1002 for branch
in branches
.keys():
1003 tip
= branches
[branch
]
1004 log
= extractLogMessageFromGitCommit(tip
)
1005 settings
= extractSettingsGitLog(log
)
1006 if "depot-paths" in settings
:
1007 paths
= ",".join(settings
["depot-paths"])
1008 branchByDepotPath
[paths
] = "remotes/p4/" + branch
1012 while parent
< 65535:
1013 commit
= head
+ "~%s" % parent
1014 log
= extractLogMessageFromGitCommit(commit
)
1015 settings
= extractSettingsGitLog(log
)
1016 if "depot-paths" in settings
:
1017 paths
= ",".join(settings
["depot-paths"])
1018 if paths
in branchByDepotPath
:
1019 return [branchByDepotPath
[paths
], settings
]
1023 return ["", settings
]
1025 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
1027 print("Creating/updating branch(es) in %s based on origin branch(es)"
1030 originPrefix
= "origin/p4/"
1032 for line
in read_pipe_lines(["git", "rev-parse", "--symbolic", "--remotes"]):
1034 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
1037 headName
= line
[len(originPrefix
):]
1038 remoteHead
= localRefPrefix
+ headName
1041 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
1042 if ('depot-paths' not in original
1043 or 'change' not in original
):
1047 if not gitBranchExists(remoteHead
):
1049 print("creating %s" % remoteHead
)
1052 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
1053 if 'change' in settings
:
1054 if settings
['depot-paths'] == original
['depot-paths']:
1055 originP4Change
= int(original
['change'])
1056 p4Change
= int(settings
['change'])
1057 if originP4Change
> p4Change
:
1058 print("%s (%s) is newer than %s (%s). "
1059 "Updating p4 branch from origin."
1060 % (originHead
, originP4Change
,
1061 remoteHead
, p4Change
))
1064 print("Ignoring: %s was imported from %s while "
1065 "%s was imported from %s"
1066 % (originHead
, ','.join(original
['depot-paths']),
1067 remoteHead
, ','.join(settings
['depot-paths'])))
1070 system(["git", "update-ref", remoteHead
, originHead
])
1072 def originP4BranchesExist():
1073 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
1076 def p4ParseNumericChangeRange(parts
):
1077 changeStart
= int(parts
[0][1:])
1078 if parts
[1] == '#head':
1079 changeEnd
= p4_last_change()
1081 changeEnd
= int(parts
[1])
1083 return (changeStart
, changeEnd
)
1085 def chooseBlockSize(blockSize
):
1089 return defaultBlockSize
1091 def p4ChangesForPaths(depotPaths
, changeRange
, requestedBlockSize
):
1094 # Parse the change range into start and end. Try to find integer
1095 # revision ranges as these can be broken up into blocks to avoid
1096 # hitting server-side limits (maxrows, maxscanresults). But if
1097 # that doesn't work, fall back to using the raw revision specifier
1098 # strings, without using block mode.
1100 if changeRange
is None or changeRange
== '':
1102 changeEnd
= p4_last_change()
1103 block_size
= chooseBlockSize(requestedBlockSize
)
1105 parts
= changeRange
.split(',')
1106 assert len(parts
) == 2
1108 (changeStart
, changeEnd
) = p4ParseNumericChangeRange(parts
)
1109 block_size
= chooseBlockSize(requestedBlockSize
)
1111 changeStart
= parts
[0][1:]
1112 changeEnd
= parts
[1]
1113 if requestedBlockSize
:
1114 die("cannot use --changes-block-size with non-numeric revisions")
1119 # Retrieve changes a block at a time, to prevent running
1120 # into a MaxResults/MaxScanRows error from the server. If
1121 # we _do_ hit one of those errors, turn down the block size
1127 end
= min(changeEnd
, changeStart
+ block_size
)
1128 revisionRange
= "%d,%d" % (changeStart
, end
)
1130 revisionRange
= "%s,%s" % (changeStart
, changeEnd
)
1132 for p
in depotPaths
:
1133 cmd
+= ["%s...@%s" % (p
, revisionRange
)]
1137 result
= p4CmdList(cmd
, errors_as_exceptions
=True)
1138 except P4RequestSizeException
as e
:
1140 block_size
= e
.limit
1141 elif block_size
> e
.limit
:
1142 block_size
= e
.limit
1144 block_size
= max(2, block_size
// 2)
1146 if verbose
: print("block size error, retrying with block size {0}".format(block_size
))
1148 except P4Exception
as e
:
1149 die('Error retrieving changes description ({0})'.format(e
.p4ExitCode
))
1151 # Insert changes in chronological order
1152 for entry
in reversed(result
):
1153 if 'change' not in entry
:
1155 changes
.add(int(entry
['change']))
1160 if end
>= changeEnd
:
1163 changeStart
= end
+ 1
1165 changes
= sorted(changes
)
1168 def p4PathStartsWith(path
, prefix
):
1169 # This method tries to remedy a potential mixed-case issue:
1171 # If UserA adds //depot/DirA/file1
1172 # and UserB adds //depot/dira/file2
1174 # we may or may not have a problem. If you have core.ignorecase=true,
1175 # we treat DirA and dira as the same directory
1176 if gitConfigBool("core.ignorecase"):
1177 return path
.lower().startswith(prefix
.lower())
1178 return path
.startswith(prefix
)
1180 def getClientSpec():
1181 """Look at the p4 client spec, create a View() object that contains
1182 all the mappings, and return it."""
1184 specList
= p4CmdList(["client", "-o"])
1185 if len(specList
) != 1:
1186 die('Output from "client -o" is %d lines, expecting 1' %
1189 # dictionary of all client parameters
1192 # the //client/ name
1193 client_name
= entry
["Client"]
1195 # just the keys that start with "View"
1196 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
1198 # hold this new View
1199 view
= View(client_name
)
1201 # append the lines, in order, to the view
1202 for view_num
in range(len(view_keys
)):
1203 k
= "View%d" % view_num
1204 if k
not in view_keys
:
1205 die("Expected view key %s missing" % k
)
1206 view
.append(entry
[k
])
1210 def getClientRoot():
1211 """Grab the client directory."""
1213 output
= p4CmdList(["client", "-o"])
1214 if len(output
) != 1:
1215 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
1218 if "Root" not in entry
:
1219 die('Client has no "Root"')
1221 return entry
["Root"]
1224 # P4 wildcards are not allowed in filenames. P4 complains
1225 # if you simply add them, but you can force it with "-f", in
1226 # which case it translates them into %xx encoding internally.
1228 def wildcard_decode(path
):
1229 # Search for and fix just these four characters. Do % last so
1230 # that fixing it does not inadvertently create new %-escapes.
1231 # Cannot have * in a filename in windows; untested as to
1232 # what p4 would do in such a case.
1233 if not platform
.system() == "Windows":
1234 path
= path
.replace("%2A", "*")
1235 path
= path
.replace("%23", "#") \
1236 .replace("%40", "@") \
1237 .replace("%25", "%")
1240 def wildcard_encode(path
):
1241 # do % first to avoid double-encoding the %s introduced here
1242 path
= path
.replace("%", "%25") \
1243 .replace("*", "%2A") \
1244 .replace("#", "%23") \
1245 .replace("@", "%40")
1248 def wildcard_present(path
):
1249 m
= re
.search("[*#@%]", path
)
1250 return m
is not None
1252 class LargeFileSystem(object):
1253 """Base class for large file system support."""
1255 def __init__(self
, writeToGitStream
):
1256 self
.largeFiles
= set()
1257 self
.writeToGitStream
= writeToGitStream
1259 def generatePointer(self
, cloneDestination
, contentFile
):
1260 """Return the content of a pointer file that is stored in Git instead of
1261 the actual content."""
1262 assert False, "Method 'generatePointer' required in " + self
.__class
__.__name
__
1264 def pushFile(self
, localLargeFile
):
1265 """Push the actual content which is not stored in the Git repository to
1267 assert False, "Method 'pushFile' required in " + self
.__class
__.__name
__
1269 def hasLargeFileExtension(self
, relPath
):
1270 return functools
.reduce(
1271 lambda a
, b
: a
or b
,
1272 [relPath
.endswith('.' + e
) for e
in gitConfigList('git-p4.largeFileExtensions')],
1276 def generateTempFile(self
, contents
):
1277 contentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
1279 contentFile
.write(d
)
1281 return contentFile
.name
1283 def exceedsLargeFileThreshold(self
, relPath
, contents
):
1284 if gitConfigInt('git-p4.largeFileThreshold'):
1285 contentsSize
= sum(len(d
) for d
in contents
)
1286 if contentsSize
> gitConfigInt('git-p4.largeFileThreshold'):
1288 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1289 contentsSize
= sum(len(d
) for d
in contents
)
1290 if contentsSize
<= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1292 contentTempFile
= self
.generateTempFile(contents
)
1293 compressedContentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=True)
1294 with zipfile
.ZipFile(compressedContentFile
, mode
='w') as zf
:
1295 zf
.write(contentTempFile
, compress_type
=zipfile
.ZIP_DEFLATED
)
1296 compressedContentsSize
= zf
.infolist()[0].compress_size
1297 os
.remove(contentTempFile
)
1298 if compressedContentsSize
> gitConfigInt('git-p4.largeFileCompressedThreshold'):
1302 def addLargeFile(self
, relPath
):
1303 self
.largeFiles
.add(relPath
)
1305 def removeLargeFile(self
, relPath
):
1306 self
.largeFiles
.remove(relPath
)
1308 def isLargeFile(self
, relPath
):
1309 return relPath
in self
.largeFiles
1311 def processContent(self
, git_mode
, relPath
, contents
):
1312 """Processes the content of git fast import. This method decides if a
1313 file is stored in the large file system and handles all necessary
1315 if self
.exceedsLargeFileThreshold(relPath
, contents
) or self
.hasLargeFileExtension(relPath
):
1316 contentTempFile
= self
.generateTempFile(contents
)
1317 (pointer_git_mode
, contents
, localLargeFile
) = self
.generatePointer(contentTempFile
)
1318 if pointer_git_mode
:
1319 git_mode
= pointer_git_mode
1321 # Move temp file to final location in large file system
1322 largeFileDir
= os
.path
.dirname(localLargeFile
)
1323 if not os
.path
.isdir(largeFileDir
):
1324 os
.makedirs(largeFileDir
)
1325 shutil
.move(contentTempFile
, localLargeFile
)
1326 self
.addLargeFile(relPath
)
1327 if gitConfigBool('git-p4.largeFilePush'):
1328 self
.pushFile(localLargeFile
)
1330 sys
.stderr
.write("%s moved to large file system (%s)\n" % (relPath
, localLargeFile
))
1331 return (git_mode
, contents
)
1333 class MockLFS(LargeFileSystem
):
1334 """Mock large file system for testing."""
1336 def generatePointer(self
, contentFile
):
1337 """The pointer content is the original content prefixed with "pointer-".
1338 The local filename of the large file storage is derived from the file content.
1340 with
open(contentFile
, 'r') as f
:
1343 pointerContents
= 'pointer-' + content
1344 localLargeFile
= os
.path
.join(os
.getcwd(), '.git', 'mock-storage', 'local', content
[:-1])
1345 return (gitMode
, pointerContents
, localLargeFile
)
1347 def pushFile(self
, localLargeFile
):
1348 """The remote filename of the large file storage is the same as the local
1349 one but in a different directory.
1351 remotePath
= os
.path
.join(os
.path
.dirname(localLargeFile
), '..', 'remote')
1352 if not os
.path
.exists(remotePath
):
1353 os
.makedirs(remotePath
)
1354 shutil
.copyfile(localLargeFile
, os
.path
.join(remotePath
, os
.path
.basename(localLargeFile
)))
1356 class GitLFS(LargeFileSystem
):
1357 """Git LFS as backend for the git-p4 large file system.
1358 See https://git-lfs.github.com/ for details."""
1360 def __init__(self
, *args
):
1361 LargeFileSystem
.__init
__(self
, *args
)
1362 self
.baseGitAttributes
= []
1364 def generatePointer(self
, contentFile
):
1365 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1366 mode and content which is stored in the Git repository instead of
1367 the actual content. Return also the new location of the actual
1370 if os
.path
.getsize(contentFile
) == 0:
1371 return (None, '', None)
1373 pointerProcess
= subprocess
.Popen(
1374 ['git', 'lfs', 'pointer', '--file=' + contentFile
],
1375 stdout
=subprocess
.PIPE
1377 pointerFile
= decode_text_stream(pointerProcess
.stdout
.read())
1378 if pointerProcess
.wait():
1379 os
.remove(contentFile
)
1380 die('git-lfs pointer command failed. Did you install the extension?')
1382 # Git LFS removed the preamble in the output of the 'pointer' command
1383 # starting from version 1.2.0. Check for the preamble here to support
1385 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1386 if pointerFile
.startswith('Git LFS pointer for'):
1387 pointerFile
= re
.sub(r
'Git LFS pointer for.*\n\n', '', pointerFile
)
1389 oid
= re
.search(r
'^oid \w+:(\w+)', pointerFile
, re
.MULTILINE
).group(1)
1390 # if someone use external lfs.storage ( not in local repo git )
1391 lfs_path
= gitConfig('lfs.storage')
1394 if not os
.path
.isabs(lfs_path
):
1395 lfs_path
= os
.path
.join(os
.getcwd(), '.git', lfs_path
)
1396 localLargeFile
= os
.path
.join(
1398 'objects', oid
[:2], oid
[2:4],
1401 # LFS Spec states that pointer files should not have the executable bit set.
1403 return (gitMode
, pointerFile
, localLargeFile
)
1405 def pushFile(self
, localLargeFile
):
1406 uploadProcess
= subprocess
.Popen(
1407 ['git', 'lfs', 'push', '--object-id', 'origin', os
.path
.basename(localLargeFile
)]
1409 if uploadProcess
.wait():
1410 die('git-lfs push command failed. Did you define a remote?')
1412 def generateGitAttributes(self
):
1414 self
.baseGitAttributes
+
1418 '# Git LFS (see https://git-lfs.github.com/)\n',
1421 ['*.' + f
.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1422 for f
in sorted(gitConfigList('git-p4.largeFileExtensions'))
1424 ['/' + f
.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1425 for f
in sorted(self
.largeFiles
) if not self
.hasLargeFileExtension(f
)
1429 def addLargeFile(self
, relPath
):
1430 LargeFileSystem
.addLargeFile(self
, relPath
)
1431 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1433 def removeLargeFile(self
, relPath
):
1434 LargeFileSystem
.removeLargeFile(self
, relPath
)
1435 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1437 def processContent(self
, git_mode
, relPath
, contents
):
1438 if relPath
== '.gitattributes':
1439 self
.baseGitAttributes
= contents
1440 return (git_mode
, self
.generateGitAttributes())
1442 return LargeFileSystem
.processContent(self
, git_mode
, relPath
, contents
)
1445 delete_actions
= ( "delete", "move/delete", "purge" )
1446 add_actions
= ( "add", "branch", "move/add" )
1449 self
.usage
= "usage: %prog [options]"
1450 self
.needsGit
= True
1451 self
.verbose
= False
1453 # This is required for the "append" update_shelve action
1454 def ensure_value(self
, attr
, value
):
1455 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
1456 setattr(self
, attr
, value
)
1457 return getattr(self
, attr
)
1461 self
.userMapFromPerforceServer
= False
1462 self
.myP4UserId
= None
1466 return self
.myP4UserId
1468 results
= p4CmdList(["user", "-o"])
1471 self
.myP4UserId
= r
['User']
1473 die("Could not find your p4 user id")
1475 def p4UserIsMe(self
, p4User
):
1476 # return True if the given p4 user is actually me
1477 me
= self
.p4UserId()
1478 if not p4User
or p4User
!= me
:
1483 def getUserCacheFilename(self
):
1484 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
1485 return home
+ "/.gitp4-usercache.txt"
1487 def getUserMapFromPerforceServer(self
):
1488 if self
.userMapFromPerforceServer
:
1493 for output
in p4CmdList(["users"]):
1494 if "User" not in output
:
1496 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
1497 self
.emails
[output
["Email"]] = output
["User"]
1499 mapUserConfigRegex
= re
.compile(r
"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re
.VERBOSE
)
1500 for mapUserConfig
in gitConfigList("git-p4.mapUser"):
1501 mapUser
= mapUserConfigRegex
.findall(mapUserConfig
)
1502 if mapUser
and len(mapUser
[0]) == 3:
1503 user
= mapUser
[0][0]
1504 fullname
= mapUser
[0][1]
1505 email
= mapUser
[0][2]
1506 self
.users
[user
] = fullname
+ " <" + email
+ ">"
1507 self
.emails
[email
] = user
1510 for (key
, val
) in self
.users
.items():
1511 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
1513 open(self
.getUserCacheFilename(), 'w').write(s
)
1514 self
.userMapFromPerforceServer
= True
1516 def loadUserMapFromCache(self
):
1518 self
.userMapFromPerforceServer
= False
1520 cache
= open(self
.getUserCacheFilename(), 'r')
1521 lines
= cache
.readlines()
1524 entry
= line
.strip().split("\t")
1525 self
.users
[entry
[0]] = entry
[1]
1527 self
.getUserMapFromPerforceServer()
1529 class P4Submit(Command
, P4UserMap
):
1531 conflict_behavior_choices
= ("ask", "skip", "quit")
1534 Command
.__init
__(self
)
1535 P4UserMap
.__init
__(self
)
1537 optparse
.make_option("--origin", dest
="origin"),
1538 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
1539 # preserve the user, requires relevant p4 permissions
1540 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
1541 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
1542 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
1543 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
1544 optparse
.make_option("--conflict", dest
="conflict_behavior",
1545 choices
=self
.conflict_behavior_choices
),
1546 optparse
.make_option("--branch", dest
="branch"),
1547 optparse
.make_option("--shelve", dest
="shelve", action
="store_true",
1548 help="Shelve instead of submit. Shelved files are reverted, "
1549 "restoring the workspace to the state before the shelve"),
1550 optparse
.make_option("--update-shelve", dest
="update_shelve", action
="append", type="int",
1551 metavar
="CHANGELIST",
1552 help="update an existing shelved changelist, implies --shelve, "
1553 "repeat in-order for multiple shelved changelists"),
1554 optparse
.make_option("--commit", dest
="commit", metavar
="COMMIT",
1555 help="submit only the specified commit(s), one commit or xxx..xxx"),
1556 optparse
.make_option("--disable-rebase", dest
="disable_rebase", action
="store_true",
1557 help="Disable rebase after submit is completed. Can be useful if you "
1558 "work from a local git branch that is not master"),
1559 optparse
.make_option("--disable-p4sync", dest
="disable_p4sync", action
="store_true",
1560 help="Skip Perforce sync of p4/master after submit or shelve"),
1561 optparse
.make_option("--no-verify", dest
="no_verify", action
="store_true",
1562 help="Bypass p4-pre-submit and p4-changelist hooks"),
1564 self
.description
= """Submit changes from git to the perforce depot.\n
1565 The `p4-pre-submit` hook is executed if it exists and is executable. It
1566 can be bypassed with the `--no-verify` command line option. The hook takes
1567 no parameters and nothing from standard input. Exiting with a non-zero status
1568 from this script prevents `git-p4 submit` from launching.
1570 One usage scenario is to run unit tests in the hook.
1572 The `p4-prepare-changelist` hook is executed right after preparing the default
1573 changelist message and before the editor is started. It takes one parameter,
1574 the name of the file that contains the changelist text. Exiting with a non-zero
1575 status from the script will abort the process.
1577 The purpose of the hook is to edit the message file in place, and it is not
1578 supressed by the `--no-verify` option. This hook is called even if
1579 `--prepare-p4-only` is set.
1581 The `p4-changelist` hook is executed after the changelist message has been
1582 edited by the user. It can be bypassed with the `--no-verify` option. It
1583 takes a single parameter, the name of the file that holds the proposed
1584 changelist text. Exiting with a non-zero status causes the command to abort.
1586 The hook is allowed to edit the changelist file and can be used to normalize
1587 the text into some project standard format. It can also be used to refuse the
1588 Submit after inspect the message file.
1590 The `p4-post-changelist` hook is invoked after the submit has successfully
1591 occurred in P4. It takes no parameters and is meant primarily for notification
1592 and cannot affect the outcome of the git p4 submit action.
1595 self
.usage
+= " [name of git branch to submit into perforce depot]"
1597 self
.detectRenames
= False
1598 self
.preserveUser
= gitConfigBool("git-p4.preserveUser")
1599 self
.dry_run
= False
1601 self
.update_shelve
= list()
1603 self
.disable_rebase
= gitConfigBool("git-p4.disableRebase")
1604 self
.disable_p4sync
= gitConfigBool("git-p4.disableP4Sync")
1605 self
.prepare_p4_only
= False
1606 self
.conflict_behavior
= None
1607 self
.isWindows
= (platform
.system() == "Windows")
1608 self
.exportLabels
= False
1609 self
.p4HasMoveCommand
= p4_has_move_command()
1611 self
.no_verify
= False
1613 if gitConfig('git-p4.largeFileSystem'):
1614 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1617 if len(p4CmdList(["opened", "..."])) > 0:
1618 die("You have files opened with perforce! Close them before starting the sync.")
1620 def separate_jobs_from_description(self
, message
):
1621 """Extract and return a possible Jobs field in the commit
1622 message. It goes into a separate section in the p4 change
1625 A jobs line starts with "Jobs:" and looks like a new field
1626 in a form. Values are white-space separated on the same
1627 line or on following lines that start with a tab.
1629 This does not parse and extract the full git commit message
1630 like a p4 form. It just sees the Jobs: line as a marker
1631 to pass everything from then on directly into the p4 form,
1632 but outside the description section.
1634 Return a tuple (stripped log message, jobs string)."""
1636 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
1638 return (message
, None)
1640 jobtext
= message
[m
.start():]
1641 stripped_message
= message
[:m
.start()].rstrip()
1642 return (stripped_message
, jobtext
)
1644 def prepareLogMessage(self
, template
, message
, jobs
):
1645 """Edits the template returned from "p4 change -o" to insert
1646 the message in the Description field, and the jobs text in
1650 inDescriptionSection
= False
1652 for line
in template
.split("\n"):
1653 if line
.startswith("#"):
1654 result
+= line
+ "\n"
1657 if inDescriptionSection
:
1658 if line
.startswith("Files:") or line
.startswith("Jobs:"):
1659 inDescriptionSection
= False
1660 # insert Jobs section
1662 result
+= jobs
+ "\n"
1666 if line
.startswith("Description:"):
1667 inDescriptionSection
= True
1669 for messageLine
in message
.split("\n"):
1670 line
+= "\t" + messageLine
+ "\n"
1672 result
+= line
+ "\n"
1676 def patchRCSKeywords(self
, file, regexp
):
1677 # Attempt to zap the RCS keywords in a p4 controlled file matching the given regex
1678 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
1680 with os
.fdopen(handle
, "wb") as outFile
, open(file, "rb") as inFile
:
1681 for line
in inFile
.readlines():
1682 outFile
.write(regexp
.sub(br
'$\1$', line
))
1683 # Forcibly overwrite the original file
1685 shutil
.move(outFileName
, file)
1687 # cleanup our temporary file
1688 os
.unlink(outFileName
)
1689 print("Failed to strip RCS keywords in %s" % file)
1692 print("Patched up RCS keywords in %s" % file)
1694 def p4UserForCommit(self
,id):
1695 # Return the tuple (perforce user,git email) for a given git commit id
1696 self
.getUserMapFromPerforceServer()
1697 gitEmail
= read_pipe(["git", "log", "--max-count=1",
1698 "--format=%ae", id])
1699 gitEmail
= gitEmail
.strip()
1700 if gitEmail
not in self
.emails
:
1701 return (None,gitEmail
)
1703 return (self
.emails
[gitEmail
],gitEmail
)
1705 def checkValidP4Users(self
,commits
):
1706 # check if any git authors cannot be mapped to p4 users
1708 (user
,email
) = self
.p4UserForCommit(id)
1710 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
1711 if gitConfigBool("git-p4.allowMissingP4Users"):
1714 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
1716 def lastP4Changelist(self
):
1717 # Get back the last changelist number submitted in this client spec. This
1718 # then gets used to patch up the username in the change. If the same
1719 # client spec is being used by multiple processes then this might go
1721 results
= p4CmdList(["client", "-o"]) # find the current client
1725 client
= r
['Client']
1728 die("could not get client spec")
1729 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
1733 die("Could not get changelist number for last submit - cannot patch up user details")
1735 def modifyChangelistUser(self
, changelist
, newUser
):
1736 # fixup the user field of a changelist after it has been submitted.
1737 changes
= p4CmdList(["change", "-o", changelist
])
1738 if len(changes
) != 1:
1739 die("Bad output from p4 change modifying %s to user %s" %
1740 (changelist
, newUser
))
1743 if c
['User'] == newUser
: return # nothing to do
1745 # p4 does not understand format version 3 and above
1746 input = marshal
.dumps(c
, 2)
1748 result
= p4CmdList(["change", "-f", "-i"], stdin
=input)
1751 if r
['code'] == 'error':
1752 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1754 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1756 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1758 def canChangeChangelists(self
):
1759 # check to see if we have p4 admin or super-user permissions, either of
1760 # which are required to modify changelists.
1761 results
= p4CmdList(["protects", self
.depotPath
])
1764 if r
['perm'] == 'admin':
1766 if r
['perm'] == 'super':
1770 def prepareSubmitTemplate(self
, changelist
=None):
1771 """Run "p4 change -o" to grab a change specification template.
1772 This does not use "p4 -G", as it is nice to keep the submission
1773 template in original order, since a human might edit it.
1775 Remove lines in the Files section that show changes to files
1776 outside the depot path we're committing into."""
1778 [upstream
, settings
] = findUpstreamBranchPoint()
1781 # A Perforce Change Specification.
1783 # Change: The change number. 'new' on a new changelist.
1784 # Date: The date this specification was last modified.
1785 # Client: The client on which the changelist was created. Read-only.
1786 # User: The user who created the changelist.
1787 # Status: Either 'pending' or 'submitted'. Read-only.
1788 # Type: Either 'public' or 'restricted'. Default is 'public'.
1789 # Description: Comments about the changelist. Required.
1790 # Jobs: What opened jobs are to be closed by this changelist.
1791 # You may delete jobs from this list. (New changelists only.)
1792 # Files: What opened files from the default changelist are to be added
1793 # to this changelist. You may delete files from this list.
1794 # (New changelists only.)
1797 inFilesSection
= False
1799 args
= ['change', '-o']
1801 args
.append(str(changelist
))
1802 for entry
in p4CmdList(args
):
1803 if 'code' not in entry
:
1805 if entry
['code'] == 'stat':
1806 change_entry
= entry
1808 if not change_entry
:
1809 die('Failed to decode output of p4 change -o')
1810 for key
, value
in change_entry
.items():
1811 if key
.startswith('File'):
1812 if 'depot-paths' in settings
:
1813 if not [p
for p
in settings
['depot-paths']
1814 if p4PathStartsWith(value
, p
)]:
1817 if not p4PathStartsWith(value
, self
.depotPath
):
1819 files_list
.append(value
)
1821 # Output in the order expected by prepareLogMessage
1822 for key
in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1823 if key
not in change_entry
:
1826 template
+= key
+ ':'
1827 if key
== 'Description':
1829 for field_line
in change_entry
[key
].splitlines():
1830 template
+= '\t'+field_line
+'\n'
1831 if len(files_list
) > 0:
1833 template
+= 'Files:\n'
1834 for path
in files_list
:
1835 template
+= '\t'+path
+'\n'
1838 def edit_template(self
, template_file
):
1839 """Invoke the editor to let the user change the submission
1840 message. Return true if okay to continue with the submit."""
1842 # if configured to skip the editing part, just submit
1843 if gitConfigBool("git-p4.skipSubmitEdit"):
1846 # look at the modification time, to check later if the user saved
1848 mtime
= os
.stat(template_file
).st_mtime
1851 if "P4EDITOR" in os
.environ
and (os
.environ
.get("P4EDITOR") != ""):
1852 editor
= os
.environ
.get("P4EDITOR")
1854 editor
= read_pipe(["git", "var", "GIT_EDITOR"]).strip()
1855 system(["sh", "-c", ('%s "$@"' % editor
), editor
, template_file
])
1857 # If the file was not saved, prompt to see if this patch should
1858 # be skipped. But skip this verification step if configured so.
1859 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1862 # modification time updated means user saved the file
1863 if os
.stat(template_file
).st_mtime
> mtime
:
1866 response
= prompt("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1872 def get_diff_description(self
, editedFiles
, filesToAdd
, symlinks
):
1874 if "P4DIFF" in os
.environ
:
1875 del(os
.environ
["P4DIFF"])
1877 for editedFile
in editedFiles
:
1878 diff
+= p4_read_pipe(['diff', '-du',
1879 wildcard_encode(editedFile
)])
1883 for newFile
in filesToAdd
:
1884 newdiff
+= "==== new file ====\n"
1885 newdiff
+= "--- /dev/null\n"
1886 newdiff
+= "+++ %s\n" % newFile
1888 is_link
= os
.path
.islink(newFile
)
1889 expect_link
= newFile
in symlinks
1891 if is_link
and expect_link
:
1892 newdiff
+= "+%s\n" % os
.readlink(newFile
)
1894 f
= open(newFile
, "r")
1896 for line
in f
.readlines():
1897 newdiff
+= "+" + line
1898 except UnicodeDecodeError:
1899 pass # Found non-text data and skip, since diff description should only include text
1902 return (diff
+ newdiff
).replace('\r\n', '\n')
1904 def applyCommit(self
, id):
1905 """Apply one commit, return True if it succeeded."""
1907 print("Applying", read_pipe(["git", "show", "-s",
1908 "--format=format:%h %s", id]))
1910 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1912 diff
= read_pipe_lines(
1913 ["git", "diff-tree", "-r"] + self
.diffOpts
+ ["{}^".format(id), id])
1915 filesToChangeType
= set()
1916 filesToDelete
= set()
1918 pureRenameCopy
= set()
1920 filesToChangeExecBit
= {}
1924 diff
= parseDiffTreeEntry(line
)
1925 modifier
= diff
['status']
1927 all_files
.append(path
)
1931 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1932 filesToChangeExecBit
[path
] = diff
['dst_mode']
1933 editedFiles
.add(path
)
1934 elif modifier
== "A":
1935 filesToAdd
.add(path
)
1936 filesToChangeExecBit
[path
] = diff
['dst_mode']
1937 if path
in filesToDelete
:
1938 filesToDelete
.remove(path
)
1940 dst_mode
= int(diff
['dst_mode'], 8)
1941 if dst_mode
== 0o120000:
1944 elif modifier
== "D":
1945 filesToDelete
.add(path
)
1946 if path
in filesToAdd
:
1947 filesToAdd
.remove(path
)
1948 elif modifier
== "C":
1949 src
, dest
= diff
['src'], diff
['dst']
1950 all_files
.append(dest
)
1951 p4_integrate(src
, dest
)
1952 pureRenameCopy
.add(dest
)
1953 if diff
['src_sha1'] != diff
['dst_sha1']:
1955 pureRenameCopy
.discard(dest
)
1956 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1958 pureRenameCopy
.discard(dest
)
1959 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1961 # turn off read-only attribute
1962 os
.chmod(dest
, stat
.S_IWRITE
)
1964 editedFiles
.add(dest
)
1965 elif modifier
== "R":
1966 src
, dest
= diff
['src'], diff
['dst']
1967 all_files
.append(dest
)
1968 if self
.p4HasMoveCommand
:
1969 p4_edit(src
) # src must be open before move
1970 p4_move(src
, dest
) # opens for (move/delete, move/add)
1972 p4_integrate(src
, dest
)
1973 if diff
['src_sha1'] != diff
['dst_sha1']:
1976 pureRenameCopy
.add(dest
)
1977 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1978 if not self
.p4HasMoveCommand
:
1979 p4_edit(dest
) # with move: already open, writable
1980 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1981 if not self
.p4HasMoveCommand
:
1983 os
.chmod(dest
, stat
.S_IWRITE
)
1985 filesToDelete
.add(src
)
1986 editedFiles
.add(dest
)
1987 elif modifier
== "T":
1988 filesToChangeType
.add(path
)
1990 die("unknown modifier %s for %s" % (modifier
, path
))
1992 diffcmd
= "git diff-tree --full-index -p \"%s\"" % (id)
1993 patchcmd
= diffcmd
+ " | git apply "
1994 tryPatchCmd
= patchcmd
+ "--check -"
1995 applyPatchCmd
= patchcmd
+ "--check --apply -"
1996 patch_succeeded
= True
1999 print("TryPatch: %s" % tryPatchCmd
)
2001 if os
.system(tryPatchCmd
) != 0:
2002 fixed_rcs_keywords
= False
2003 patch_succeeded
= False
2004 print("Unfortunately applying the change failed!")
2006 # Patch failed, maybe it's just RCS keyword woes. Look through
2007 # the patch to see if that's possible.
2008 if gitConfigBool("git-p4.attemptRCSCleanup"):
2011 for file in editedFiles | filesToDelete
:
2012 # did this file's delta contain RCS keywords?
2013 regexp
= p4_keywords_regexp_for_file(file)
2015 # this file is a possibility...look for RCS keywords.
2016 for line
in read_pipe_lines(
2017 ["git", "diff", "%s^..%s" % (id, id), file],
2019 if regexp
.search(line
):
2021 print("got keyword match on %s in %s in %s" % (regex
.pattern
, line
, file))
2022 kwfiles
[file] = regexp
2025 for file, regexp
in kwfiles
.items():
2027 print("zapping %s with %s" % (line
, regexp
.pattern
))
2028 # File is being deleted, so not open in p4. Must
2029 # disable the read-only bit on windows.
2030 if self
.isWindows
and file not in editedFiles
:
2031 os
.chmod(file, stat
.S_IWRITE
)
2032 self
.patchRCSKeywords(file, kwfiles
[file])
2033 fixed_rcs_keywords
= True
2035 if fixed_rcs_keywords
:
2036 print("Retrying the patch with RCS keywords cleaned up")
2037 if os
.system(tryPatchCmd
) == 0:
2038 patch_succeeded
= True
2039 print("Patch succeesed this time with RCS keywords cleaned")
2041 if not patch_succeeded
:
2042 for f
in editedFiles
:
2047 # Apply the patch for real, and do add/delete/+x handling.
2049 system(applyPatchCmd
, shell
=True)
2051 for f
in filesToChangeType
:
2052 p4_edit(f
, "-t", "auto")
2053 for f
in filesToAdd
:
2055 for f
in filesToDelete
:
2059 # Set/clear executable bits
2060 for f
in filesToChangeExecBit
.keys():
2061 mode
= filesToChangeExecBit
[f
]
2062 setP4ExecBit(f
, mode
)
2065 if len(self
.update_shelve
) > 0:
2066 update_shelve
= self
.update_shelve
.pop(0)
2067 p4_reopen_in_change(update_shelve
, all_files
)
2070 # Build p4 change description, starting with the contents
2071 # of the git commit message.
2073 logMessage
= extractLogMessageFromGitCommit(id)
2074 logMessage
= logMessage
.strip()
2075 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
2077 template
= self
.prepareSubmitTemplate(update_shelve
)
2078 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
2080 if self
.preserveUser
:
2081 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
2083 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
2084 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
2085 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
2086 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
2088 separatorLine
= "######## everything below this line is just the diff #######\n"
2089 if not self
.prepare_p4_only
:
2090 submitTemplate
+= separatorLine
2091 submitTemplate
+= self
.get_diff_description(editedFiles
, filesToAdd
, symlinks
)
2093 (handle
, fileName
) = tempfile
.mkstemp()
2094 tmpFile
= os
.fdopen(handle
, "w+b")
2096 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
2097 tmpFile
.write(encode_text_stream(submitTemplate
))
2103 # Allow the hook to edit the changelist text before presenting it
2105 if not run_git_hook("p4-prepare-changelist", [fileName
]):
2108 if self
.prepare_p4_only
:
2110 # Leave the p4 tree prepared, and the submit template around
2111 # and let the user decide what to do next
2115 print("P4 workspace prepared for submission.")
2116 print("To submit or revert, go to client workspace")
2117 print(" " + self
.clientPath
)
2119 print("To submit, use \"p4 submit\" to write a new description,")
2120 print("or \"p4 submit -i <%s\" to use the one prepared by" \
2121 " \"git p4\"." % fileName
)
2122 print("You can delete the file \"%s\" when finished." % fileName
)
2124 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
2125 print("To preserve change ownership by user %s, you must\n" \
2126 "do \"p4 change -f <change>\" after submitting and\n" \
2127 "edit the User field.")
2129 print("After submitting, renamed files must be re-synced.")
2130 print("Invoke \"p4 sync -f\" on each of these files:")
2131 for f
in pureRenameCopy
:
2135 print("To revert the changes, use \"p4 revert ...\", and delete")
2136 print("the submit template file \"%s\"" % fileName
)
2138 print("Since the commit adds new files, they must be deleted:")
2139 for f
in filesToAdd
:
2145 if self
.edit_template(fileName
):
2146 if not self
.no_verify
:
2147 if not run_git_hook("p4-changelist", [fileName
]):
2148 print("The p4-changelist hook failed.")
2152 # read the edited message and submit
2153 tmpFile
= open(fileName
, "rb")
2154 message
= decode_text_stream(tmpFile
.read())
2157 message
= message
.replace("\r\n", "\n")
2158 if message
.find(separatorLine
) != -1:
2159 submitTemplate
= message
[:message
.index(separatorLine
)]
2161 submitTemplate
= message
2163 if len(submitTemplate
.strip()) == 0:
2164 print("Changelist is empty, aborting this changelist.")
2169 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate
)
2171 p4_write_pipe(['shelve', '-i'], submitTemplate
)
2173 p4_write_pipe(['submit', '-i'], submitTemplate
)
2174 # The rename/copy happened by applying a patch that created a
2175 # new file. This leaves it writable, which confuses p4.
2176 for f
in pureRenameCopy
:
2179 if self
.preserveUser
:
2181 # Get last changelist number. Cannot easily get it from
2182 # the submit command output as the output is
2184 changelist
= self
.lastP4Changelist()
2185 self
.modifyChangelistUser(changelist
, p4User
)
2189 run_git_hook("p4-post-changelist")
2191 # Revert changes if we skip this patch
2192 if not submitted
or self
.shelve
:
2194 print ("Reverting shelved files.")
2196 print ("Submission cancelled, undoing p4 changes.")
2198 for f
in editedFiles | filesToDelete
:
2200 for f
in filesToAdd
:
2204 if not self
.prepare_p4_only
:
2208 # Export git tags as p4 labels. Create a p4 label and then tag
2210 def exportGitTags(self
, gitTags
):
2211 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
2212 if len(validLabelRegexp
) == 0:
2213 validLabelRegexp
= defaultLabelRegexp
2214 m
= re
.compile(validLabelRegexp
)
2216 for name
in gitTags
:
2218 if not m
.match(name
):
2220 print("tag %s does not match regexp %s" % (name
, validLabelRegexp
))
2223 # Get the p4 commit this corresponds to
2224 logMessage
= extractLogMessageFromGitCommit(name
)
2225 values
= extractSettingsGitLog(logMessage
)
2227 if 'change' not in values
:
2228 # a tag pointing to something not sent to p4; ignore
2230 print("git tag %s does not give a p4 commit" % name
)
2233 changelist
= values
['change']
2235 # Get the tag details.
2239 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
2242 if re
.match(r
'tag\s+', l
):
2244 elif re
.match(r
'\s*$', l
):
2251 body
= ["lightweight tag imported by git p4\n"]
2253 # Create the label - use the same view as the client spec we are using
2254 clientSpec
= getClientSpec()
2256 labelTemplate
= "Label: %s\n" % name
2257 labelTemplate
+= "Description:\n"
2259 labelTemplate
+= "\t" + b
+ "\n"
2260 labelTemplate
+= "View:\n"
2261 for depot_side
in clientSpec
.mappings
:
2262 labelTemplate
+= "\t%s\n" % depot_side
2265 print("Would create p4 label %s for tag" % name
)
2266 elif self
.prepare_p4_only
:
2267 print("Not creating p4 label %s for tag due to option" \
2268 " --prepare-p4-only" % name
)
2270 p4_write_pipe(["label", "-i"], labelTemplate
)
2273 p4_system(["tag", "-l", name
] +
2274 ["%s@%s" % (depot_side
, changelist
) for depot_side
in clientSpec
.mappings
])
2277 print("created p4 label for tag %s" % name
)
2279 def run(self
, args
):
2281 self
.master
= currentGitBranch()
2282 elif len(args
) == 1:
2283 self
.master
= args
[0]
2284 if not branchExists(self
.master
):
2285 die("Branch %s does not exist" % self
.master
)
2289 for i
in self
.update_shelve
:
2291 sys
.exit("invalid changelist %d" % i
)
2294 allowSubmit
= gitConfig("git-p4.allowSubmit")
2295 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
2296 die("%s is not in git-p4.allowSubmit" % self
.master
)
2298 [upstream
, settings
] = findUpstreamBranchPoint()
2299 self
.depotPath
= settings
['depot-paths'][0]
2300 if len(self
.origin
) == 0:
2301 self
.origin
= upstream
2303 if len(self
.update_shelve
) > 0:
2306 if self
.preserveUser
:
2307 if not self
.canChangeChangelists():
2308 die("Cannot preserve user names without p4 super-user or admin permissions")
2310 # if not set from the command line, try the config file
2311 if self
.conflict_behavior
is None:
2312 val
= gitConfig("git-p4.conflict")
2314 if val
not in self
.conflict_behavior_choices
:
2315 die("Invalid value '%s' for config git-p4.conflict" % val
)
2318 self
.conflict_behavior
= val
2321 print("Origin branch is " + self
.origin
)
2323 if len(self
.depotPath
) == 0:
2324 print("Internal error: cannot locate perforce depot path from existing branches")
2327 self
.useClientSpec
= False
2328 if gitConfigBool("git-p4.useclientspec"):
2329 self
.useClientSpec
= True
2330 if self
.useClientSpec
:
2331 self
.clientSpecDirs
= getClientSpec()
2333 # Check for the existence of P4 branches
2334 branchesDetected
= (len(p4BranchesInGit().keys()) > 1)
2336 if self
.useClientSpec
and not branchesDetected
:
2337 # all files are relative to the client spec
2338 self
.clientPath
= getClientRoot()
2340 self
.clientPath
= p4Where(self
.depotPath
)
2342 if self
.clientPath
== "":
2343 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
2345 print("Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
))
2346 self
.oldWorkingDirectory
= os
.getcwd()
2348 # ensure the clientPath exists
2349 new_client_dir
= False
2350 if not os
.path
.exists(self
.clientPath
):
2351 new_client_dir
= True
2352 os
.makedirs(self
.clientPath
)
2354 chdir(self
.clientPath
, is_client_path
=True)
2356 print("Would synchronize p4 checkout in %s" % self
.clientPath
)
2358 print("Synchronizing p4 checkout...")
2360 # old one was destroyed, and maybe nobody told p4
2361 p4_sync("...", "-f")
2368 committish
= self
.master
2372 if self
.commit
!= "":
2373 if self
.commit
.find("..") != -1:
2374 limits_ish
= self
.commit
.split("..")
2375 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish
[0], limits_ish
[1])]):
2376 commits
.append(line
.strip())
2379 commits
.append(self
.commit
)
2381 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self
.origin
, committish
)]):
2382 commits
.append(line
.strip())
2385 if self
.preserveUser
or gitConfigBool("git-p4.skipUserNameCheck"):
2386 self
.checkAuthorship
= False
2388 self
.checkAuthorship
= True
2390 if self
.preserveUser
:
2391 self
.checkValidP4Users(commits
)
2394 # Build up a set of options to be passed to diff when
2395 # submitting each commit to p4.
2397 if self
.detectRenames
:
2398 # command-line -M arg
2399 self
.diffOpts
= ["-M"]
2401 # If not explicitly set check the config variable
2402 detectRenames
= gitConfig("git-p4.detectRenames")
2404 if detectRenames
.lower() == "false" or detectRenames
== "":
2406 elif detectRenames
.lower() == "true":
2407 self
.diffOpts
= ["-M"]
2409 self
.diffOpts
= ["-M{}".format(detectRenames
)]
2411 # no command-line arg for -C or --find-copies-harder, just
2413 detectCopies
= gitConfig("git-p4.detectCopies")
2414 if detectCopies
.lower() == "false" or detectCopies
== "":
2416 elif detectCopies
.lower() == "true":
2417 self
.diffOpts
.append("-C")
2419 self
.diffOpts
.append("-C{}".format(detectCopies
))
2421 if gitConfigBool("git-p4.detectCopiesHarder"):
2422 self
.diffOpts
.append("--find-copies-harder")
2424 num_shelves
= len(self
.update_shelve
)
2425 if num_shelves
> 0 and num_shelves
!= len(commits
):
2426 sys
.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2427 (len(commits
), num_shelves
))
2429 if not self
.no_verify
:
2431 if not run_git_hook("p4-pre-submit"):
2432 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nYou can skip " \
2433 "this pre-submission check by adding\nthe command line option '--no-verify', " \
2434 "however,\nthis will also skip the p4-changelist hook as well.")
2436 except Exception as e
:
2437 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nThe hook failed "\
2438 "with the error '{0}'".format(e
.message
) )
2442 # Apply the commits, one at a time. On failure, ask if should
2443 # continue to try the rest of the patches, or quit.
2446 print("Would apply")
2448 last
= len(commits
) - 1
2449 for i
, commit
in enumerate(commits
):
2451 print(" ", read_pipe(["git", "show", "-s",
2452 "--format=format:%h %s", commit
]))
2455 ok
= self
.applyCommit(commit
)
2457 applied
.append(commit
)
2458 if self
.prepare_p4_only
:
2460 print("Processing only the first commit due to option" \
2461 " --prepare-p4-only")
2465 # prompt for what to do, or use the option/variable
2466 if self
.conflict_behavior
== "ask":
2467 print("What do you want to do?")
2468 response
= prompt("[s]kip this commit but apply the rest, or [q]uit? ")
2469 elif self
.conflict_behavior
== "skip":
2471 elif self
.conflict_behavior
== "quit":
2474 die("Unknown conflict_behavior '%s'" %
2475 self
.conflict_behavior
)
2478 print("Skipping this commit, but applying the rest")
2483 chdir(self
.oldWorkingDirectory
)
2484 shelved_applied
= "shelved" if self
.shelve
else "applied"
2487 elif self
.prepare_p4_only
:
2489 elif len(commits
) == len(applied
):
2490 print("All commits {0}!".format(shelved_applied
))
2494 sync
.branch
= self
.branch
2495 if self
.disable_p4sync
:
2496 sync
.sync_origin_only()
2500 if not self
.disable_rebase
:
2505 if len(applied
) == 0:
2506 print("No commits {0}.".format(shelved_applied
))
2508 print("{0} only the commits marked with '*':".format(shelved_applied
.capitalize()))
2514 print(star
, read_pipe(["git", "show", "-s",
2515 "--format=format:%h %s", c
]))
2516 print("You will have to do 'git p4 sync' and rebase.")
2518 if gitConfigBool("git-p4.exportLabels"):
2519 self
.exportLabels
= True
2521 if self
.exportLabels
:
2522 p4Labels
= getP4Labels(self
.depotPath
)
2523 gitTags
= getGitTags()
2525 missingGitTags
= gitTags
- p4Labels
2526 self
.exportGitTags(missingGitTags
)
2528 # exit with error unless everything applied perfectly
2529 if len(commits
) != len(applied
):
2535 """Represent a p4 view ("p4 help views"), and map files in a
2536 repo according to the view."""
2538 def __init__(self
, client_name
):
2540 self
.client_prefix
= "//%s/" % client_name
2541 # cache results of "p4 where" to lookup client file locations
2542 self
.client_spec_path_cache
= {}
2544 def append(self
, view_line
):
2545 """Parse a view line, splitting it into depot and client
2546 sides. Append to self.mappings, preserving order. This
2547 is only needed for tag creation."""
2549 # Split the view line into exactly two words. P4 enforces
2550 # structure on these lines that simplifies this quite a bit.
2552 # Either or both words may be double-quoted.
2553 # Single quotes do not matter.
2554 # Double-quote marks cannot occur inside the words.
2555 # A + or - prefix is also inside the quotes.
2556 # There are no quotes unless they contain a space.
2557 # The line is already white-space stripped.
2558 # The two words are separated by a single space.
2560 if view_line
[0] == '"':
2561 # First word is double quoted. Find its end.
2562 close_quote_index
= view_line
.find('"', 1)
2563 if close_quote_index
<= 0:
2564 die("No first-word closing quote found: %s" % view_line
)
2565 depot_side
= view_line
[1:close_quote_index
]
2566 # skip closing quote and space
2567 rhs_index
= close_quote_index
+ 1 + 1
2569 space_index
= view_line
.find(" ")
2570 if space_index
<= 0:
2571 die("No word-splitting space found: %s" % view_line
)
2572 depot_side
= view_line
[0:space_index
]
2573 rhs_index
= space_index
+ 1
2575 # prefix + means overlay on previous mapping
2576 if depot_side
.startswith("+"):
2577 depot_side
= depot_side
[1:]
2579 # prefix - means exclude this path, leave out of mappings
2581 if depot_side
.startswith("-"):
2583 depot_side
= depot_side
[1:]
2586 self
.mappings
.append(depot_side
)
2588 def convert_client_path(self
, clientFile
):
2589 # chop off //client/ part to make it relative
2590 if not decode_path(clientFile
).startswith(self
.client_prefix
):
2591 die("No prefix '%s' on clientFile '%s'" %
2592 (self
.client_prefix
, clientFile
))
2593 return clientFile
[len(self
.client_prefix
):]
2595 def update_client_spec_path_cache(self
, files
):
2596 """ Caching file paths by "p4 where" batch query """
2598 # List depot file paths exclude that already cached
2599 fileArgs
= [f
['path'] for f
in files
if decode_path(f
['path']) not in self
.client_spec_path_cache
]
2601 if len(fileArgs
) == 0:
2602 return # All files in cache
2604 where_result
= p4CmdList(["-x", "-", "where"], stdin
=fileArgs
)
2605 for res
in where_result
:
2606 if "code" in res
and res
["code"] == "error":
2607 # assume error is "... file(s) not in client view"
2609 if "clientFile" not in res
:
2610 die("No clientFile in 'p4 where' output")
2612 # it will list all of them, but only one not unmap-ped
2614 depot_path
= decode_path(res
['depotFile'])
2615 if gitConfigBool("core.ignorecase"):
2616 depot_path
= depot_path
.lower()
2617 self
.client_spec_path_cache
[depot_path
] = self
.convert_client_path(res
["clientFile"])
2619 # not found files or unmap files set to ""
2620 for depotFile
in fileArgs
:
2621 depotFile
= decode_path(depotFile
)
2622 if gitConfigBool("core.ignorecase"):
2623 depotFile
= depotFile
.lower()
2624 if depotFile
not in self
.client_spec_path_cache
:
2625 self
.client_spec_path_cache
[depotFile
] = b
''
2627 def map_in_client(self
, depot_path
):
2628 """Return the relative location in the client where this
2629 depot file should live. Returns "" if the file should
2630 not be mapped in the client."""
2632 if gitConfigBool("core.ignorecase"):
2633 depot_path
= depot_path
.lower()
2635 if depot_path
in self
.client_spec_path_cache
:
2636 return self
.client_spec_path_cache
[depot_path
]
2638 die( "Error: %s is not found in client spec path" % depot_path
)
2641 def cloneExcludeCallback(option
, opt_str
, value
, parser
):
2642 # prepend "/" because the first "/" was consumed as part of the option itself.
2643 # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2644 parser
.values
.cloneExclude
+= ["/" + re
.sub(r
"\.\.\.$", "", value
)]
2646 class P4Sync(Command
, P4UserMap
):
2649 Command
.__init
__(self
)
2650 P4UserMap
.__init
__(self
)
2652 optparse
.make_option("--branch", dest
="branch"),
2653 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
2654 optparse
.make_option("--changesfile", dest
="changesFile"),
2655 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
2656 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
2657 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2658 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
2659 help="Import into refs/heads/ , not refs/remotes"),
2660 optparse
.make_option("--max-changes", dest
="maxChanges",
2661 help="Maximum number of changes to import"),
2662 optparse
.make_option("--changes-block-size", dest
="changes_block_size", type="int",
2663 help="Internal block size to use when iteratively calling p4 changes"),
2664 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
2665 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2666 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
2667 help="Only sync files that are included in the Perforce Client Spec"),
2668 optparse
.make_option("-/", dest
="cloneExclude",
2669 action
="callback", callback
=cloneExcludeCallback
, type="string",
2670 help="exclude depot path"),
2672 self
.description
= """Imports from Perforce into a git repository.\n
2674 //depot/my/project/ -- to import the current head
2675 //depot/my/project/@all -- to import everything
2676 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2678 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2680 self
.usage
+= " //depot/path[@revRange]"
2682 self
.createdBranches
= set()
2683 self
.committedChanges
= set()
2685 self
.detectBranches
= False
2686 self
.detectLabels
= False
2687 self
.importLabels
= False
2688 self
.changesFile
= ""
2689 self
.syncWithOrigin
= True
2690 self
.importIntoRemotes
= True
2691 self
.maxChanges
= ""
2692 self
.changes_block_size
= None
2693 self
.keepRepoPath
= False
2694 self
.depotPaths
= None
2695 self
.p4BranchesInGit
= []
2696 self
.cloneExclude
= []
2697 self
.useClientSpec
= False
2698 self
.useClientSpec_from_options
= False
2699 self
.clientSpecDirs
= None
2700 self
.tempBranches
= []
2701 self
.tempBranchLocation
= "refs/git-p4-tmp"
2702 self
.largeFileSystem
= None
2703 self
.suppress_meta_comment
= False
2705 if gitConfig('git-p4.largeFileSystem'):
2706 largeFileSystemConstructor
= globals()[gitConfig('git-p4.largeFileSystem')]
2707 self
.largeFileSystem
= largeFileSystemConstructor(
2708 lambda git_mode
, relPath
, contents
: self
.writeToGitStream(git_mode
, relPath
, contents
)
2711 if gitConfig("git-p4.syncFromOrigin") == "false":
2712 self
.syncWithOrigin
= False
2714 self
.depotPaths
= []
2715 self
.changeRange
= ""
2716 self
.previousDepotPaths
= []
2717 self
.hasOrigin
= False
2719 # map from branch depot path to parent branch
2720 self
.knownBranches
= {}
2721 self
.initialParents
= {}
2723 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
2726 # Force a checkpoint in fast-import and wait for it to finish
2727 def checkpoint(self
):
2728 self
.gitStream
.write("checkpoint\n\n")
2729 self
.gitStream
.write("progress checkpoint\n\n")
2730 self
.gitStream
.flush()
2731 out
= self
.gitOutput
.readline()
2733 print("checkpoint finished: " + out
)
2735 def isPathWanted(self
, path
):
2736 for p
in self
.cloneExclude
:
2738 if p4PathStartsWith(path
, p
):
2740 # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2741 elif path
.lower() == p
.lower():
2743 for p
in self
.depotPaths
:
2744 if p4PathStartsWith(path
, decode_path(p
)):
2748 def extractFilesFromCommit(self
, commit
, shelved
=False, shelved_cl
= 0):
2751 while "depotFile%s" % fnum
in commit
:
2752 path
= commit
["depotFile%s" % fnum
]
2753 found
= self
.isPathWanted(decode_path(path
))
2760 file["rev"] = commit
["rev%s" % fnum
]
2761 file["action"] = commit
["action%s" % fnum
]
2762 file["type"] = commit
["type%s" % fnum
]
2764 file["shelved_cl"] = int(shelved_cl
)
2769 def extractJobsFromCommit(self
, commit
):
2772 while "job%s" % jnum
in commit
:
2773 job
= commit
["job%s" % jnum
]
2778 def stripRepoPath(self
, path
, prefixes
):
2779 """When streaming files, this is called to map a p4 depot path
2780 to where it should go in git. The prefixes are either
2781 self.depotPaths, or self.branchPrefixes in the case of
2782 branch detection."""
2784 if self
.useClientSpec
:
2785 # branch detection moves files up a level (the branch name)
2786 # from what client spec interpretation gives
2787 path
= decode_path(self
.clientSpecDirs
.map_in_client(path
))
2788 if self
.detectBranches
:
2789 for b
in self
.knownBranches
:
2790 if p4PathStartsWith(path
, b
+ "/"):
2791 path
= path
[len(b
)+1:]
2793 elif self
.keepRepoPath
:
2794 # Preserve everything in relative path name except leading
2795 # //depot/; just look at first prefix as they all should
2796 # be in the same depot.
2797 depot
= re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])
2798 if p4PathStartsWith(path
, depot
):
2799 path
= path
[len(depot
):]
2803 if p4PathStartsWith(path
, p
):
2804 path
= path
[len(p
):]
2807 path
= wildcard_decode(path
)
2810 def splitFilesIntoBranches(self
, commit
):
2811 """Look at each depotFile in the commit to figure out to what
2812 branch it belongs."""
2814 if self
.clientSpecDirs
:
2815 files
= self
.extractFilesFromCommit(commit
)
2816 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2820 while "depotFile%s" % fnum
in commit
:
2821 raw_path
= commit
["depotFile%s" % fnum
]
2822 path
= decode_path(raw_path
)
2823 found
= self
.isPathWanted(path
)
2829 file["path"] = raw_path
2830 file["rev"] = commit
["rev%s" % fnum
]
2831 file["action"] = commit
["action%s" % fnum
]
2832 file["type"] = commit
["type%s" % fnum
]
2835 # start with the full relative path where this file would
2837 if self
.useClientSpec
:
2838 relPath
= decode_path(self
.clientSpecDirs
.map_in_client(path
))
2840 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
2842 for branch
in self
.knownBranches
.keys():
2843 # add a trailing slash so that a commit into qt/4.2foo
2844 # doesn't end up in qt/4.2, e.g.
2845 if p4PathStartsWith(relPath
, branch
+ "/"):
2846 if branch
not in branches
:
2847 branches
[branch
] = []
2848 branches
[branch
].append(file)
2853 def writeToGitStream(self
, gitMode
, relPath
, contents
):
2854 self
.gitStream
.write(encode_text_stream(u
'M {} inline {}\n'.format(gitMode
, relPath
)))
2855 self
.gitStream
.write('data %d\n' % sum(len(d
) for d
in contents
))
2857 self
.gitStream
.write(d
)
2858 self
.gitStream
.write('\n')
2860 def encodeWithUTF8(self
, path
):
2862 path
.decode('ascii')
2865 if gitConfig('git-p4.pathEncoding'):
2866 encoding
= gitConfig('git-p4.pathEncoding')
2867 path
= path
.decode(encoding
, 'replace').encode('utf8', 'replace')
2869 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding
, path
))
2872 # output one file from the P4 stream
2873 # - helper for streamP4Files
2875 def streamOneP4File(self
, file, contents
):
2876 file_path
= file['depotFile']
2877 relPath
= self
.stripRepoPath(decode_path(file_path
), self
.branchPrefixes
)
2880 if 'fileSize' in self
.stream_file
:
2881 size
= int(self
.stream_file
['fileSize'])
2883 size
= 0 # deleted files don't get a fileSize apparently
2884 sys
.stdout
.write('\r%s --> %s (%s)\n' % (
2885 file_path
, relPath
, format_size_human_readable(size
)))
2888 (type_base
, type_mods
) = split_p4_type(file["type"])
2891 if "x" in type_mods
:
2893 if type_base
== "symlink":
2895 # p4 print on a symlink sometimes contains "target\n";
2896 # if it does, remove the newline
2897 data
= ''.join(decode_text_stream(c
) for c
in contents
)
2899 # Some version of p4 allowed creating a symlink that pointed
2900 # to nothing. This causes p4 errors when checking out such
2901 # a change, and errors here too. Work around it by ignoring
2902 # the bad symlink; hopefully a future change fixes it.
2903 print("\nIgnoring empty symlink in %s" % file_path
)
2905 elif data
[-1] == '\n':
2906 contents
= [data
[:-1]]
2910 if type_base
== "utf16":
2911 # p4 delivers different text in the python output to -G
2912 # than it does when using "print -o", or normal p4 client
2913 # operations. utf16 is converted to ascii or utf8, perhaps.
2914 # But ascii text saved as -t utf16 is completely mangled.
2915 # Invoke print -o to get the real contents.
2917 # On windows, the newlines will always be mangled by print, so put
2918 # them back too. This is not needed to the cygwin windows version,
2919 # just the native "NT" type.
2922 text
= p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (decode_path(file['depotFile']), file['change'])], raw
=True)
2923 except Exception as e
:
2924 if 'Translation of file content failed' in str(e
):
2925 type_base
= 'binary'
2929 if p4_version_string().find('/NT') >= 0:
2930 text
= text
.replace(b
'\r\n', b
'\n')
2933 if type_base
== "apple":
2934 # Apple filetype files will be streamed as a concatenation of
2935 # its appledouble header and the contents. This is useless
2936 # on both macs and non-macs. If using "print -q -o xx", it
2937 # will create "xx" with the data, and "%xx" with the header.
2938 # This is also not very useful.
2940 # Ideally, someday, this script can learn how to generate
2941 # appledouble files directly and import those to git, but
2942 # non-mac machines can never find a use for apple filetype.
2943 print("\nIgnoring apple filetype file %s" % file['depotFile'])
2946 # Note that we do not try to de-mangle keywords on utf16 files,
2947 # even though in theory somebody may want that.
2948 regexp
= p4_keywords_regexp_for_type(type_base
, type_mods
)
2950 contents
= [regexp
.sub(br
'$\1$', c
) for c
in contents
]
2952 if self
.largeFileSystem
:
2953 (git_mode
, contents
) = self
.largeFileSystem
.processContent(git_mode
, relPath
, contents
)
2955 self
.writeToGitStream(git_mode
, relPath
, contents
)
2957 def streamOneP4Deletion(self
, file):
2958 relPath
= self
.stripRepoPath(decode_path(file['path']), self
.branchPrefixes
)
2960 sys
.stdout
.write("delete %s\n" % relPath
)
2962 self
.gitStream
.write(encode_text_stream(u
'D {}\n'.format(relPath
)))
2964 if self
.largeFileSystem
and self
.largeFileSystem
.isLargeFile(relPath
):
2965 self
.largeFileSystem
.removeLargeFile(relPath
)
2967 # handle another chunk of streaming data
2968 def streamP4FilesCb(self
, marshalled
):
2970 # catch p4 errors and complain
2972 if "code" in marshalled
:
2973 if marshalled
["code"] == "error":
2974 if "data" in marshalled
:
2975 err
= marshalled
["data"].rstrip()
2977 if not err
and 'fileSize' in self
.stream_file
:
2978 required_bytes
= int((4 * int(self
.stream_file
["fileSize"])) - calcDiskFree())
2979 if required_bytes
> 0:
2980 err
= 'Not enough space left on %s! Free at least %s.' % (
2981 os
.getcwd(), format_size_human_readable(required_bytes
))
2985 if self
.stream_have_file_info
:
2986 if "depotFile" in self
.stream_file
:
2987 f
= self
.stream_file
["depotFile"]
2988 # force a failure in fast-import, else an empty
2989 # commit will be made
2990 self
.gitStream
.write("\n")
2991 self
.gitStream
.write("die-now\n")
2992 self
.gitStream
.close()
2993 # ignore errors, but make sure it exits first
2994 self
.importProcess
.wait()
2996 die("Error from p4 print for %s: %s" % (f
, err
))
2998 die("Error from p4 print: %s" % err
)
3000 if 'depotFile' in marshalled
and self
.stream_have_file_info
:
3001 # start of a new file - output the old one first
3002 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
3003 self
.stream_file
= {}
3004 self
.stream_contents
= []
3005 self
.stream_have_file_info
= False
3007 # pick up the new file information... for the
3008 # 'data' field we need to append to our array
3009 for k
in marshalled
.keys():
3011 if 'streamContentSize' not in self
.stream_file
:
3012 self
.stream_file
['streamContentSize'] = 0
3013 self
.stream_file
['streamContentSize'] += len(marshalled
['data'])
3014 self
.stream_contents
.append(marshalled
['data'])
3016 self
.stream_file
[k
] = marshalled
[k
]
3019 'streamContentSize' in self
.stream_file
and
3020 'fileSize' in self
.stream_file
and
3021 'depotFile' in self
.stream_file
):
3022 size
= int(self
.stream_file
["fileSize"])
3024 progress
= 100*self
.stream_file
['streamContentSize']/size
3025 sys
.stdout
.write('\r%s %d%% (%s)' % (
3026 self
.stream_file
['depotFile'], progress
,
3027 format_size_human_readable(size
)))
3030 self
.stream_have_file_info
= True
3032 # Stream directly from "p4 files" into "git fast-import"
3033 def streamP4Files(self
, files
):
3039 filesForCommit
.append(f
)
3040 if f
['action'] in self
.delete_actions
:
3041 filesToDelete
.append(f
)
3043 filesToRead
.append(f
)
3046 for f
in filesToDelete
:
3047 self
.streamOneP4Deletion(f
)
3049 if len(filesToRead
) > 0:
3050 self
.stream_file
= {}
3051 self
.stream_contents
= []
3052 self
.stream_have_file_info
= False
3054 # curry self argument
3055 def streamP4FilesCbSelf(entry
):
3056 self
.streamP4FilesCb(entry
)
3059 for f
in filesToRead
:
3060 if 'shelved_cl' in f
:
3061 # Handle shelved CLs using the "p4 print file@=N" syntax to print
3063 fileArg
= f
['path'] + encode_text_stream('@={}'.format(f
['shelved_cl']))
3065 fileArg
= f
['path'] + encode_text_stream('#{}'.format(f
['rev']))
3067 fileArgs
.append(fileArg
)
3069 p4CmdList(["-x", "-", "print"],
3071 cb
=streamP4FilesCbSelf
)
3074 if 'depotFile' in self
.stream_file
:
3075 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
3077 def make_email(self
, userid
):
3078 if userid
in self
.users
:
3079 return self
.users
[userid
]
3081 return "%s <a@b>" % userid
3083 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
3084 """ Stream a p4 tag.
3085 commit is either a git commit, or a fast-import mark, ":<p4commit>"
3089 print("writing tag %s for commit %s" % (labelName
, commit
))
3090 gitStream
.write("tag %s\n" % labelName
)
3091 gitStream
.write("from %s\n" % commit
)
3093 if 'Owner' in labelDetails
:
3094 owner
= labelDetails
["Owner"]
3098 # Try to use the owner of the p4 label, or failing that,
3099 # the current p4 user id.
3101 email
= self
.make_email(owner
)
3103 email
= self
.make_email(self
.p4UserId())
3104 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
3106 gitStream
.write("tagger %s\n" % tagger
)
3108 print("labelDetails=",labelDetails
)
3109 if 'Description' in labelDetails
:
3110 description
= labelDetails
['Description']
3112 description
= 'Label from git p4'
3114 gitStream
.write("data %d\n" % len(description
))
3115 gitStream
.write(description
)
3116 gitStream
.write("\n")
3118 def inClientSpec(self
, path
):
3119 if not self
.clientSpecDirs
:
3121 inClientSpec
= self
.clientSpecDirs
.map_in_client(path
)
3122 if not inClientSpec
and self
.verbose
:
3123 print('Ignoring file outside of client spec: {0}'.format(path
))
3126 def hasBranchPrefix(self
, path
):
3127 if not self
.branchPrefixes
:
3129 hasPrefix
= [p
for p
in self
.branchPrefixes
3130 if p4PathStartsWith(path
, p
)]
3131 if not hasPrefix
and self
.verbose
:
3132 print('Ignoring file outside of prefix: {0}'.format(path
))
3135 def findShadowedFiles(self
, files
, change
):
3136 # Perforce allows you commit files and directories with the same name,
3137 # so you could have files //depot/foo and //depot/foo/bar both checked
3138 # in. A p4 sync of a repository in this state fails. Deleting one of
3139 # the files recovers the repository.
3141 # Git will not allow the broken state to exist and only the most recent
3142 # of the conflicting names is left in the repository. When one of the
3143 # conflicting files is deleted we need to re-add the other one to make
3144 # sure the git repository recovers in the same way as perforce.
3145 deleted
= [f
for f
in files
if f
['action'] in self
.delete_actions
]
3148 path
= decode_path(f
['path'])
3149 to_check
.add(path
+ '/...')
3151 path
= path
.rsplit("/", 1)[0]
3152 if path
== "/" or path
in to_check
:
3155 to_check
= ['%s@%s' % (wildcard_encode(p
), change
) for p
in to_check
3156 if self
.hasBranchPrefix(p
)]
3158 stat_result
= p4CmdList(["-x", "-", "fstat", "-T",
3159 "depotFile,headAction,headRev,headType"], stdin
=to_check
)
3160 for record
in stat_result
:
3161 if record
['code'] != 'stat':
3163 if record
['headAction'] in self
.delete_actions
:
3167 'path': record
['depotFile'],
3168 'rev': record
['headRev'],
3169 'type': record
['headType']})
3171 def commit(self
, details
, files
, branch
, parent
= "", allow_empty
=False):
3172 epoch
= details
["time"]
3173 author
= details
["user"]
3174 jobs
= self
.extractJobsFromCommit(details
)
3177 print('commit into {0}'.format(branch
))
3179 files
= [f
for f
in files
3180 if self
.hasBranchPrefix(decode_path(f
['path']))]
3181 self
.findShadowedFiles(files
, details
['change'])
3183 if self
.clientSpecDirs
:
3184 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
3186 files
= [f
for f
in files
if self
.inClientSpec(decode_path(f
['path']))]
3188 if gitConfigBool('git-p4.keepEmptyCommits'):
3191 if not files
and not allow_empty
:
3192 print('Ignoring revision {0} as it would produce an empty commit.'
3193 .format(details
['change']))
3196 self
.gitStream
.write("commit %s\n" % branch
)
3197 self
.gitStream
.write("mark :%s\n" % details
["change"])
3198 self
.committedChanges
.add(int(details
["change"]))
3200 if author
not in self
.users
:
3201 self
.getUserMapFromPerforceServer()
3202 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
3204 self
.gitStream
.write("committer %s\n" % committer
)
3206 self
.gitStream
.write("data <<EOT\n")
3207 self
.gitStream
.write(details
["desc"])
3209 self
.gitStream
.write("\nJobs: %s" % (' '.join(jobs
)))
3211 if not self
.suppress_meta_comment
:
3212 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3213 (','.join(self
.branchPrefixes
), details
["change"]))
3214 if len(details
['options']) > 0:
3215 self
.gitStream
.write(": options = %s" % details
['options'])
3216 self
.gitStream
.write("]\n")
3218 self
.gitStream
.write("EOT\n\n")
3222 print("parent %s" % parent
)
3223 self
.gitStream
.write("from %s\n" % parent
)
3225 self
.streamP4Files(files
)
3226 self
.gitStream
.write("\n")
3228 change
= int(details
["change"])
3230 if change
in self
.labels
:
3231 label
= self
.labels
[change
]
3232 labelDetails
= label
[0]
3233 labelRevisions
= label
[1]
3235 print("Change %s is labelled %s" % (change
, labelDetails
))
3237 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
3238 for p
in self
.branchPrefixes
])
3240 if len(files
) == len(labelRevisions
):
3244 if info
["action"] in self
.delete_actions
:
3246 cleanedFiles
[info
["depotFile"]] = info
["rev"]
3248 if cleanedFiles
== labelRevisions
:
3249 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
3253 print("Tag %s does not match with change %s: files do not match."
3254 % (labelDetails
["label"], change
))
3258 print("Tag %s does not match with change %s: file count is different."
3259 % (labelDetails
["label"], change
))
3261 # Build a dictionary of changelists and labels, for "detect-labels" option.
3262 def getLabels(self
):
3265 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
3266 if len(l
) > 0 and not self
.silent
:
3267 print("Finding files belonging to labels in %s" % self
.depotPaths
)
3270 label
= output
["label"]
3274 print("Querying files for label %s" % label
)
3275 for file in p4CmdList(["files"] +
3276 ["%s...@%s" % (p
, label
)
3277 for p
in self
.depotPaths
]):
3278 revisions
[file["depotFile"]] = file["rev"]
3279 change
= int(file["change"])
3280 if change
> newestChange
:
3281 newestChange
= change
3283 self
.labels
[newestChange
] = [output
, revisions
]
3286 print("Label changes: %s" % self
.labels
.keys())
3288 # Import p4 labels as git tags. A direct mapping does not
3289 # exist, so assume that if all the files are at the same revision
3290 # then we can use that, or it's something more complicated we should
3292 def importP4Labels(self
, stream
, p4Labels
):
3294 print("import p4 labels: " + ' '.join(p4Labels
))
3296 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
3297 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
3298 if len(validLabelRegexp
) == 0:
3299 validLabelRegexp
= defaultLabelRegexp
3300 m
= re
.compile(validLabelRegexp
)
3302 for name
in p4Labels
:
3305 if not m
.match(name
):
3307 print("label %s does not match regexp %s" % (name
,validLabelRegexp
))
3310 if name
in ignoredP4Labels
:
3313 labelDetails
= p4CmdList(['label', "-o", name
])[0]
3315 # get the most recent changelist for each file in this label
3316 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
3317 for p
in self
.depotPaths
])
3319 if 'change' in change
:
3320 # find the corresponding git commit; take the oldest commit
3321 changelist
= int(change
['change'])
3322 if changelist
in self
.committedChanges
:
3323 gitCommit
= ":%d" % changelist
# use a fast-import mark
3326 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
3327 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
], ignore_error
=True)
3328 if len(gitCommit
) == 0:
3329 print("importing label %s: could not find git commit for changelist %d" % (name
, changelist
))
3332 gitCommit
= gitCommit
.strip()
3335 # Convert from p4 time format
3337 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
3339 print("Could not convert label time %s" % labelDetails
['Update'])
3342 when
= int(time
.mktime(tmwhen
))
3343 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
3345 print("p4 label %s mapped to git commit %s" % (name
, gitCommit
))
3348 print("Label %s has no changelists - possibly deleted?" % name
)
3351 # We can't import this label; don't try again as it will get very
3352 # expensive repeatedly fetching all the files for labels that will
3353 # never be imported. If the label is moved in the future, the
3354 # ignore will need to be removed manually.
3355 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
3357 def guessProjectName(self
):
3358 for p
in self
.depotPaths
:
3361 p
= p
[p
.strip().rfind("/") + 1:]
3362 if not p
.endswith("/"):
3366 def getBranchMapping(self
):
3367 lostAndFoundBranches
= set()
3369 user
= gitConfig("git-p4.branchUser")
3371 for info
in p4CmdList(
3372 ["branches"] + (["-u", user
] if len(user
) > 0 else [])):
3373 details
= p4Cmd(["branch", "-o", info
["branch"]])
3375 while "View%s" % viewIdx
in details
:
3376 paths
= details
["View%s" % viewIdx
].split(" ")
3377 viewIdx
= viewIdx
+ 1
3378 # require standard //depot/foo/... //depot/bar/... mapping
3379 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
3382 destination
= paths
[1]
3384 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
3385 source
= source
[len(self
.depotPaths
[0]):-4]
3386 destination
= destination
[len(self
.depotPaths
[0]):-4]
3388 if destination
in self
.knownBranches
:
3390 print("p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
))
3391 print("but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
))
3394 self
.knownBranches
[destination
] = source
3396 lostAndFoundBranches
.discard(destination
)
3398 if source
not in self
.knownBranches
:
3399 lostAndFoundBranches
.add(source
)
3401 # Perforce does not strictly require branches to be defined, so we also
3402 # check git config for a branch list.
3404 # Example of branch definition in git config file:
3406 # branchList=main:branchA
3407 # branchList=main:branchB
3408 # branchList=branchA:branchC
3409 configBranches
= gitConfigList("git-p4.branchList")
3410 for branch
in configBranches
:
3412 (source
, destination
) = branch
.split(":")
3413 self
.knownBranches
[destination
] = source
3415 lostAndFoundBranches
.discard(destination
)
3417 if source
not in self
.knownBranches
:
3418 lostAndFoundBranches
.add(source
)
3421 for branch
in lostAndFoundBranches
:
3422 self
.knownBranches
[branch
] = branch
3424 def getBranchMappingFromGitBranches(self
):
3425 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3426 for branch
in branches
.keys():
3427 if branch
== "master":
3430 branch
= branch
[len(self
.projectName
):]
3431 self
.knownBranches
[branch
] = branch
3433 def updateOptionDict(self
, d
):
3435 if self
.keepRepoPath
:
3436 option_keys
['keepRepoPath'] = 1
3438 d
["options"] = ' '.join(sorted(option_keys
.keys()))
3440 def readOptions(self
, d
):
3441 self
.keepRepoPath
= ('options' in d
3442 and ('keepRepoPath' in d
['options']))
3444 def gitRefForBranch(self
, branch
):
3445 if branch
== "main":
3446 return self
.refPrefix
+ "master"
3448 if len(branch
) <= 0:
3451 return self
.refPrefix
+ self
.projectName
+ branch
3453 def gitCommitByP4Change(self
, ref
, change
):
3455 print("looking in ref " + ref
+ " for change %s using bisect..." % change
)
3458 latestCommit
= parseRevision(ref
)
3462 print("trying: earliest %s latest %s" % (earliestCommit
, latestCommit
))
3463 next
= read_pipe(["git", "rev-list", "--bisect",
3464 latestCommit
, earliestCommit
]).strip()
3469 log
= extractLogMessageFromGitCommit(next
)
3470 settings
= extractSettingsGitLog(log
)
3471 currentChange
= int(settings
['change'])
3473 print("current change %s" % currentChange
)
3475 if currentChange
== change
:
3477 print("found %s" % next
)
3480 if currentChange
< change
:
3481 earliestCommit
= "^%s" % next
3483 if next
== latestCommit
:
3484 die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref
, change
))
3485 latestCommit
= "%s^@" % next
3489 def importNewBranch(self
, branch
, maxChange
):
3490 # make fast-import flush all changes to disk and update the refs using the checkpoint
3491 # command so that we can try to find the branch parent in the git history
3492 self
.gitStream
.write("checkpoint\n\n");
3493 self
.gitStream
.flush();
3494 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3495 range = "@1,%s" % maxChange
3496 #print "prefix" + branchPrefix
3497 changes
= p4ChangesForPaths([branchPrefix
], range, self
.changes_block_size
)
3498 if len(changes
) <= 0:
3500 firstChange
= changes
[0]
3501 #print "first change in branch: %s" % firstChange
3502 sourceBranch
= self
.knownBranches
[branch
]
3503 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
3504 sourceRef
= self
.gitRefForBranch(sourceBranch
)
3505 #print "source " + sourceBranch
3507 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
3508 #print "branch parent: %s" % branchParentChange
3509 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
3510 if len(gitParent
) > 0:
3511 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
3512 #print "parent git commit: %s" % gitParent
3514 self
.importChanges(changes
)
3517 def searchParent(self
, parent
, branch
, target
):
3518 targetTree
= read_pipe(["git", "rev-parse",
3519 "{}^{{tree}}".format(target
)]).strip()
3520 for line
in read_pipe_lines(["git", "rev-list", "--format=%H %T",
3521 "--no-merges", parent
]):
3522 if line
.startswith("commit "):
3524 commit
, tree
= line
.strip().split(" ")
3525 if tree
== targetTree
:
3527 print("Found parent of %s in commit %s" % (branch
, commit
))
3531 def importChanges(self
, changes
, origin_revision
=0):
3533 for change
in changes
:
3534 description
= p4_describe(change
)
3535 self
.updateOptionDict(description
)
3538 sys
.stdout
.write("\rImporting revision %s (%d%%)" % (
3539 change
, (cnt
* 100) // len(changes
)))
3544 if self
.detectBranches
:
3545 branches
= self
.splitFilesIntoBranches(description
)
3546 for branch
in branches
.keys():
3548 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3549 self
.branchPrefixes
= [ branchPrefix
]
3553 filesForCommit
= branches
[branch
]
3556 print("branch is %s" % branch
)
3558 self
.updatedBranches
.add(branch
)
3560 if branch
not in self
.createdBranches
:
3561 self
.createdBranches
.add(branch
)
3562 parent
= self
.knownBranches
[branch
]
3563 if parent
== branch
:
3566 fullBranch
= self
.projectName
+ branch
3567 if fullBranch
not in self
.p4BranchesInGit
:
3569 print("\n Importing new branch %s" % fullBranch
);
3570 if self
.importNewBranch(branch
, change
- 1):
3572 self
.p4BranchesInGit
.append(fullBranch
)
3574 print("\n Resuming with change %s" % change
);
3577 print("parent determined through known branches: %s" % parent
)
3579 branch
= self
.gitRefForBranch(branch
)
3580 parent
= self
.gitRefForBranch(parent
)
3583 print("looking for initial parent for %s; current parent is %s" % (branch
, parent
))
3585 if len(parent
) == 0 and branch
in self
.initialParents
:
3586 parent
= self
.initialParents
[branch
]
3587 del self
.initialParents
[branch
]
3591 tempBranch
= "%s/%d" % (self
.tempBranchLocation
, change
)
3593 print("Creating temporary branch: " + tempBranch
)
3594 self
.commit(description
, filesForCommit
, tempBranch
)
3595 self
.tempBranches
.append(tempBranch
)
3597 blob
= self
.searchParent(parent
, branch
, tempBranch
)
3599 self
.commit(description
, filesForCommit
, branch
, blob
)
3602 print("Parent of %s not found. Committing into head of %s" % (branch
, parent
))
3603 self
.commit(description
, filesForCommit
, branch
, parent
)
3605 files
= self
.extractFilesFromCommit(description
)
3606 self
.commit(description
, files
, self
.branch
,
3608 # only needed once, to connect to the previous commit
3609 self
.initialParent
= ""
3611 print(self
.gitError
.read())
3614 def sync_origin_only(self
):
3615 if self
.syncWithOrigin
:
3616 self
.hasOrigin
= originP4BranchesExist()
3619 print('Syncing with origin first, using "git fetch origin"')
3620 system(["git", "fetch", "origin"])
3622 def importHeadRevision(self
, revision
):
3623 print("Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
))
3626 details
["user"] = "git perforce import user"
3627 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
3628 % (' '.join(self
.depotPaths
), revision
))
3629 details
["change"] = revision
3633 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
3635 for info
in p4CmdList(["files"] + fileArgs
):
3637 if 'code' in info
and info
['code'] == 'error':
3638 sys
.stderr
.write("p4 returned an error: %s\n"
3640 if info
['data'].find("must refer to client") >= 0:
3641 sys
.stderr
.write("This particular p4 error is misleading.\n")
3642 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
3643 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
3645 if 'p4ExitCode' in info
:
3646 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
3650 change
= int(info
["change"])
3651 if change
> newestRevision
:
3652 newestRevision
= change
3654 if info
["action"] in self
.delete_actions
:
3655 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3656 #fileCnt = fileCnt + 1
3659 for prop
in ["depotFile", "rev", "action", "type" ]:
3660 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
3662 fileCnt
= fileCnt
+ 1
3664 details
["change"] = newestRevision
3666 # Use time from top-most change so that all git p4 clones of
3667 # the same p4 repo have the same commit SHA1s.
3668 res
= p4_describe(newestRevision
)
3669 details
["time"] = res
["time"]
3671 self
.updateOptionDict(details
)
3673 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
3674 except IOError as err
:
3675 print("IO error with git fast-import. Is your git version recent enough?")
3676 print("IO error details: {}".format(err
))
3677 print(self
.gitError
.read())
3680 def importRevisions(self
, args
, branch_arg_given
):
3683 if len(self
.changesFile
) > 0:
3684 with
open(self
.changesFile
) as f
:
3685 output
= f
.readlines()
3688 changeSet
.add(int(line
))
3690 for change
in changeSet
:
3691 changes
.append(change
)
3695 # catch "git p4 sync" with no new branches, in a repo that
3696 # does not have any existing p4 branches
3698 if not self
.p4BranchesInGit
:
3699 raise P4CommandException("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3701 # The default branch is master, unless --branch is used to
3702 # specify something else. Make sure it exists, or complain
3703 # nicely about how to use --branch.
3704 if not self
.detectBranches
:
3705 if not branch_exists(self
.branch
):
3706 if branch_arg_given
:
3707 raise P4CommandException("Error: branch %s does not exist." % self
.branch
)
3709 raise P4CommandException("Error: no branch %s; perhaps specify one with --branch." %
3713 print("Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
3715 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
, self
.changes_block_size
)
3717 if len(self
.maxChanges
) > 0:
3718 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
3720 if len(changes
) == 0:
3722 print("No changes to import!")
3724 if not self
.silent
and not self
.detectBranches
:
3725 print("Import destination: %s" % self
.branch
)
3727 self
.updatedBranches
= set()
3729 if not self
.detectBranches
:
3731 # start a new branch
3732 self
.initialParent
= ""
3734 # build on a previous revision
3735 self
.initialParent
= parseRevision(self
.branch
)
3737 self
.importChanges(changes
)
3741 if len(self
.updatedBranches
) > 0:
3742 sys
.stdout
.write("Updated branches: ")
3743 for b
in self
.updatedBranches
:
3744 sys
.stdout
.write("%s " % b
)
3745 sys
.stdout
.write("\n")
3747 def openStreams(self
):
3748 self
.importProcess
= subprocess
.Popen(["git", "fast-import"],
3749 stdin
=subprocess
.PIPE
,
3750 stdout
=subprocess
.PIPE
,
3751 stderr
=subprocess
.PIPE
);
3752 self
.gitOutput
= self
.importProcess
.stdout
3753 self
.gitStream
= self
.importProcess
.stdin
3754 self
.gitError
= self
.importProcess
.stderr
3756 if bytes
is not str:
3757 # Wrap gitStream.write() so that it can be called using `str` arguments
3758 def make_encoded_write(write
):
3759 def encoded_write(s
):
3760 return write(s
.encode() if isinstance(s
, str) else s
)
3761 return encoded_write
3763 self
.gitStream
.write
= make_encoded_write(self
.gitStream
.write
)
3765 def closeStreams(self
):
3766 if self
.gitStream
is None:
3768 self
.gitStream
.close()
3769 if self
.importProcess
.wait() != 0:
3770 die("fast-import failed: %s" % self
.gitError
.read())
3771 self
.gitOutput
.close()
3772 self
.gitError
.close()
3773 self
.gitStream
= None
3775 def run(self
, args
):
3776 if self
.importIntoRemotes
:
3777 self
.refPrefix
= "refs/remotes/p4/"
3779 self
.refPrefix
= "refs/heads/p4/"
3781 self
.sync_origin_only()
3783 branch_arg_given
= bool(self
.branch
)
3784 if len(self
.branch
) == 0:
3785 self
.branch
= self
.refPrefix
+ "master"
3786 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
3787 system(["git", "update-ref", self
.branch
, "refs/heads/p4"])
3788 system(["git", "branch", "-D", "p4"])
3790 # accept either the command-line option, or the configuration variable
3791 if self
.useClientSpec
:
3792 # will use this after clone to set the variable
3793 self
.useClientSpec_from_options
= True
3795 if gitConfigBool("git-p4.useclientspec"):
3796 self
.useClientSpec
= True
3797 if self
.useClientSpec
:
3798 self
.clientSpecDirs
= getClientSpec()
3800 # TODO: should always look at previous commits,
3801 # merge with previous imports, if possible.
3804 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
3806 # branches holds mapping from branch name to sha1
3807 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3809 # restrict to just this one, disabling detect-branches
3810 if branch_arg_given
:
3811 short
= self
.branch
.split("/")[-1]
3812 if short
in branches
:
3813 self
.p4BranchesInGit
= [ short
]
3815 self
.p4BranchesInGit
= branches
.keys()
3817 if len(self
.p4BranchesInGit
) > 1:
3819 print("Importing from/into multiple branches")
3820 self
.detectBranches
= True
3821 for branch
in branches
.keys():
3822 self
.initialParents
[self
.refPrefix
+ branch
] = \
3826 print("branches: %s" % self
.p4BranchesInGit
)
3829 for branch
in self
.p4BranchesInGit
:
3830 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
3832 settings
= extractSettingsGitLog(logMsg
)
3834 self
.readOptions(settings
)
3835 if ('depot-paths' in settings
3836 and 'change' in settings
):
3837 change
= int(settings
['change']) + 1
3838 p4Change
= max(p4Change
, change
)
3840 depotPaths
= sorted(settings
['depot-paths'])
3841 if self
.previousDepotPaths
== []:
3842 self
.previousDepotPaths
= depotPaths
3845 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
3846 prev_list
= prev
.split("/")
3847 cur_list
= cur
.split("/")
3848 for i
in range(0, min(len(cur_list
), len(prev_list
))):
3849 if cur_list
[i
] != prev_list
[i
]:
3853 paths
.append ("/".join(cur_list
[:i
+ 1]))
3855 self
.previousDepotPaths
= paths
3858 self
.depotPaths
= sorted(self
.previousDepotPaths
)
3859 self
.changeRange
= "@%s,#head" % p4Change
3860 if not self
.silent
and not self
.detectBranches
:
3861 print("Performing incremental import into %s git branch" % self
.branch
)
3863 # accept multiple ref name abbreviations:
3864 # refs/foo/bar/branch -> use it exactly
3865 # p4/branch -> prepend refs/remotes/ or refs/heads/
3866 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3867 if not self
.branch
.startswith("refs/"):
3868 if self
.importIntoRemotes
:
3869 prepend
= "refs/remotes/"
3871 prepend
= "refs/heads/"
3872 if not self
.branch
.startswith("p4/"):
3874 self
.branch
= prepend
+ self
.branch
3876 if len(args
) == 0 and self
.depotPaths
:
3878 print("Depot paths: %s" % ' '.join(self
.depotPaths
))
3880 if self
.depotPaths
and self
.depotPaths
!= args
:
3881 print("previous import used depot path %s and now %s was specified. "
3882 "This doesn't work!" % (' '.join (self
.depotPaths
),
3886 self
.depotPaths
= sorted(args
)
3891 # Make sure no revision specifiers are used when --changesfile
3893 bad_changesfile
= False
3894 if len(self
.changesFile
) > 0:
3895 for p
in self
.depotPaths
:
3896 if p
.find("@") >= 0 or p
.find("#") >= 0:
3897 bad_changesfile
= True
3900 die("Option --changesfile is incompatible with revision specifiers")
3903 for p
in self
.depotPaths
:
3904 if p
.find("@") != -1:
3905 atIdx
= p
.index("@")
3906 self
.changeRange
= p
[atIdx
:]
3907 if self
.changeRange
== "@all":
3908 self
.changeRange
= ""
3909 elif ',' not in self
.changeRange
:
3910 revision
= self
.changeRange
3911 self
.changeRange
= ""
3913 elif p
.find("#") != -1:
3914 hashIdx
= p
.index("#")
3915 revision
= p
[hashIdx
:]
3917 elif self
.previousDepotPaths
== []:
3918 # pay attention to changesfile, if given, else import
3919 # the entire p4 tree at the head revision
3920 if len(self
.changesFile
) == 0:
3923 p
= re
.sub ("\.\.\.$", "", p
)
3924 if not p
.endswith("/"):
3929 self
.depotPaths
= newPaths
3931 # --detect-branches may change this for each branch
3932 self
.branchPrefixes
= self
.depotPaths
3934 self
.loadUserMapFromCache()
3936 if self
.detectLabels
:
3939 if self
.detectBranches
:
3940 ## FIXME - what's a P4 projectName ?
3941 self
.projectName
= self
.guessProjectName()
3944 self
.getBranchMappingFromGitBranches()
3946 self
.getBranchMapping()
3948 print("p4-git branches: %s" % self
.p4BranchesInGit
)
3949 print("initial parents: %s" % self
.initialParents
)
3950 for b
in self
.p4BranchesInGit
:
3954 b
= b
[len(self
.projectName
):]
3955 self
.createdBranches
.add(b
)
3965 self
.importHeadRevision(revision
)
3967 self
.importRevisions(args
, branch_arg_given
)
3969 if gitConfigBool("git-p4.importLabels"):
3970 self
.importLabels
= True
3972 if self
.importLabels
:
3973 p4Labels
= getP4Labels(self
.depotPaths
)
3974 gitTags
= getGitTags()
3976 missingP4Labels
= p4Labels
- gitTags
3977 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
3979 except P4CommandException
as e
:
3988 # Cleanup temporary branches created during import
3989 if self
.tempBranches
!= []:
3990 for branch
in self
.tempBranches
:
3991 read_pipe(["git", "update-ref", "-d", branch
])
3992 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
3994 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3995 # a convenient shortcut refname "p4".
3996 if self
.importIntoRemotes
:
3997 head_ref
= self
.refPrefix
+ "HEAD"
3998 if not gitBranchExists(head_ref
) and gitBranchExists(self
.branch
):
3999 system(["git", "symbolic-ref", head_ref
, self
.branch
])
4003 class P4Rebase(Command
):
4005 Command
.__init
__(self
)
4007 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
4009 self
.importLabels
= False
4010 self
.description
= ("Fetches the latest revision from perforce and "
4011 + "rebases the current work (branch) against it")
4013 def run(self
, args
):
4015 sync
.importLabels
= self
.importLabels
4018 return self
.rebase()
4021 if os
.system("git update-index --refresh") != 0:
4022 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
4023 if len(read_pipe(["git", "diff-index", "HEAD", "--"])) > 0:
4024 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
4026 [upstream
, settings
] = findUpstreamBranchPoint()
4027 if len(upstream
) == 0:
4028 die("Cannot find upstream branchpoint for rebase")
4030 # the branchpoint may be p4/foo~3, so strip off the parent
4031 upstream
= re
.sub("~[0-9]+$", "", upstream
)
4033 print("Rebasing the current branch onto %s" % upstream
)
4034 oldHead
= read_pipe(["git", "rev-parse", "HEAD"]).strip()
4035 system(["git", "rebase", upstream
])
4036 system(["git", "diff-tree", "--stat", "--summary", "-M", oldHead
,
4040 class P4Clone(P4Sync
):
4042 P4Sync
.__init
__(self
)
4043 self
.description
= "Creates a new git repository and imports from Perforce into it"
4044 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
4046 optparse
.make_option("--destination", dest
="cloneDestination",
4047 action
='store', default
=None,
4048 help="where to leave result of the clone"),
4049 optparse
.make_option("--bare", dest
="cloneBare",
4050 action
="store_true", default
=False),
4052 self
.cloneDestination
= None
4053 self
.needsGit
= False
4054 self
.cloneBare
= False
4056 def defaultDestination(self
, args
):
4057 ## TODO: use common prefix of args?
4059 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
4060 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
4061 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
4062 depotDir
= re
.sub(r
"/$", "", depotDir
)
4063 return os
.path
.split(depotDir
)[1]
4065 def run(self
, args
):
4069 if self
.keepRepoPath
and not self
.cloneDestination
:
4070 sys
.stderr
.write("Must specify destination for --keep-path\n")
4075 if not self
.cloneDestination
and len(depotPaths
) > 1:
4076 self
.cloneDestination
= depotPaths
[-1]
4077 depotPaths
= depotPaths
[:-1]
4079 for p
in depotPaths
:
4080 if not p
.startswith("//"):
4081 sys
.stderr
.write('Depot paths must start with "//": %s\n' % p
)
4084 if not self
.cloneDestination
:
4085 self
.cloneDestination
= self
.defaultDestination(args
)
4087 print("Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
))
4089 if not os
.path
.exists(self
.cloneDestination
):
4090 os
.makedirs(self
.cloneDestination
)
4091 chdir(self
.cloneDestination
)
4093 init_cmd
= [ "git", "init" ]
4095 init_cmd
.append("--bare")
4096 retcode
= subprocess
.call(init_cmd
)
4098 raise subprocess
.CalledProcessError(retcode
, init_cmd
)
4100 if not P4Sync
.run(self
, depotPaths
):
4103 # create a master branch and check out a work tree
4104 if gitBranchExists(self
.branch
):
4105 system([ "git", "branch", currentGitBranch(), self
.branch
])
4106 if not self
.cloneBare
:
4107 system([ "git", "checkout", "-f" ])
4109 print('Not checking out any branch, use ' \
4110 '"git checkout -q -b master <branch>"')
4112 # auto-set this variable if invoked with --use-client-spec
4113 if self
.useClientSpec_from_options
:
4114 system(["git", "config", "--bool", "git-p4.useclientspec", "true"])
4118 class P4Unshelve(Command
):
4120 Command
.__init
__(self
)
4122 self
.origin
= "HEAD"
4123 self
.description
= "Unshelve a P4 changelist into a git commit"
4124 self
.usage
= "usage: %prog [options] changelist"
4126 optparse
.make_option("--origin", dest
="origin",
4127 help="Use this base revision instead of the default (%s)" % self
.origin
),
4129 self
.verbose
= False
4130 self
.noCommit
= False
4131 self
.destbranch
= "refs/remotes/p4-unshelved"
4133 def renameBranch(self
, branch_name
):
4134 """ Rename the existing branch to branch_name.N
4138 for i
in range(0,1000):
4139 backup_branch_name
= "{0}.{1}".format(branch_name
, i
)
4140 if not gitBranchExists(backup_branch_name
):
4141 gitUpdateRef(backup_branch_name
, branch_name
) # copy ref to backup
4142 gitDeleteRef(branch_name
)
4144 print("renamed old unshelve branch to {0}".format(backup_branch_name
))
4148 sys
.exit("gave up trying to rename existing branch {0}".format(sync
.branch
))
4150 def findLastP4Revision(self
, starting_point
):
4151 """ Look back from starting_point for the first commit created by git-p4
4152 to find the P4 commit we are based on, and the depot-paths.
4155 for parent
in (range(65535)):
4156 log
= extractLogMessageFromGitCommit("{0}~{1}".format(starting_point
, parent
))
4157 settings
= extractSettingsGitLog(log
)
4158 if 'change' in settings
:
4161 sys
.exit("could not find git-p4 commits in {0}".format(self
.origin
))
4163 def createShelveParent(self
, change
, branch_name
, sync
, origin
):
4164 """ Create a commit matching the parent of the shelved changelist 'change'
4166 parent_description
= p4_describe(change
, shelved
=True)
4167 parent_description
['desc'] = 'parent for shelved changelist {}\n'.format(change
)
4168 files
= sync
.extractFilesFromCommit(parent_description
, shelved
=False, shelved_cl
=change
)
4172 # if it was added in the shelved changelist, it won't exist in the parent
4173 if f
['action'] in self
.add_actions
:
4176 # if it was deleted in the shelved changelist it must not be deleted
4177 # in the parent - we might even need to create it if the origin branch
4179 if f
['action'] in self
.delete_actions
:
4182 parent_files
.append(f
)
4184 sync
.commit(parent_description
, parent_files
, branch_name
,
4185 parent
=origin
, allow_empty
=True)
4186 print("created parent commit for {0} based on {1} in {2}".format(
4187 change
, self
.origin
, branch_name
))
4189 def run(self
, args
):
4193 if not gitBranchExists(self
.origin
):
4194 sys
.exit("origin branch {0} does not exist".format(self
.origin
))
4199 # only one change at a time
4202 # if the target branch already exists, rename it
4203 branch_name
= "{0}/{1}".format(self
.destbranch
, change
)
4204 if gitBranchExists(branch_name
):
4205 self
.renameBranch(branch_name
)
4206 sync
.branch
= branch_name
4208 sync
.verbose
= self
.verbose
4209 sync
.suppress_meta_comment
= True
4211 settings
= self
.findLastP4Revision(self
.origin
)
4212 sync
.depotPaths
= settings
['depot-paths']
4213 sync
.branchPrefixes
= sync
.depotPaths
4216 sync
.loadUserMapFromCache()
4219 # create a commit for the parent of the shelved changelist
4220 self
.createShelveParent(change
, branch_name
, sync
, self
.origin
)
4222 # create the commit for the shelved changelist itself
4223 description
= p4_describe(change
, True)
4224 files
= sync
.extractFilesFromCommit(description
, True, change
)
4226 sync
.commit(description
, files
, branch_name
, "")
4229 print("unshelved changelist {0} into {1}".format(change
, branch_name
))
4233 class P4Branches(Command
):
4235 Command
.__init
__(self
)
4237 self
.description
= ("Shows the git branches that hold imports and their "
4238 + "corresponding perforce depot paths")
4239 self
.verbose
= False
4241 def run(self
, args
):
4242 if originP4BranchesExist():
4243 createOrUpdateBranchesFromOrigin()
4245 for line
in read_pipe_lines(["git", "rev-parse", "--symbolic", "--remotes"]):
4248 if not line
.startswith('p4/') or line
== "p4/HEAD":
4252 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
4253 settings
= extractSettingsGitLog(log
)
4255 print("%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"]))
4258 class HelpFormatter(optparse
.IndentedHelpFormatter
):
4260 optparse
.IndentedHelpFormatter
.__init
__(self
)
4262 def format_description(self
, description
):
4264 return description
+ "\n"
4268 def printUsage(commands
):
4269 print("usage: %s <command> [options]" % sys
.argv
[0])
4271 print("valid commands: %s" % ", ".join(commands
))
4273 print("Try %s <command> --help for command specific help." % sys
.argv
[0])
4277 "submit" : P4Submit
,
4278 "commit" : P4Submit
,
4280 "rebase" : P4Rebase
,
4282 "branches" : P4Branches
,
4283 "unshelve" : P4Unshelve
,
4287 if len(sys
.argv
[1:]) == 0:
4288 printUsage(commands
.keys())
4291 cmdName
= sys
.argv
[1]
4293 klass
= commands
[cmdName
]
4296 print("unknown command %s" % cmdName
)
4298 printUsage(commands
.keys())
4301 options
= cmd
.options
4302 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
4306 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
4308 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
4310 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
4312 description
= cmd
.description
,
4313 formatter
= HelpFormatter())
4316 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
4322 verbose
= cmd
.verbose
4324 if cmd
.gitdir
== None:
4325 cmd
.gitdir
= os
.path
.abspath(".git")
4326 if not isValidGitDir(cmd
.gitdir
):
4327 # "rev-parse --git-dir" without arguments will try $PWD/.git
4328 cmd
.gitdir
= read_pipe(["git", "rev-parse", "--git-dir"]).strip()
4329 if os
.path
.exists(cmd
.gitdir
):
4330 cdup
= read_pipe(["git", "rev-parse", "--show-cdup"]).strip()
4334 if not isValidGitDir(cmd
.gitdir
):
4335 if isValidGitDir(cmd
.gitdir
+ "/.git"):
4336 cmd
.gitdir
+= "/.git"
4338 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
4340 # so git commands invoked from the P4 workspace will succeed
4341 os
.environ
["GIT_DIR"] = cmd
.gitdir
4343 if not cmd
.run(args
):
4348 if __name__
== '__main__':