3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 # pylint: disable=invalid-name,missing-docstring,too-many-arguments,broad-except
11 # pylint: disable=no-self-use,wrong-import-position,consider-iterating-dictionary
12 # pylint: disable=wrong-import-order,unused-import,too-few-public-methods
13 # pylint: disable=too-many-lines,ungrouped-imports,fixme,too-many-locals
14 # pylint: disable=line-too-long,bad-whitespace,superfluous-parens
15 # pylint: disable=too-many-statements,too-many-instance-attributes
16 # pylint: disable=too-many-branches,too-many-nested-blocks
19 if sys
.version_info
.major
< 3 and sys
.version_info
.minor
< 7:
20 sys
.stderr
.write("git-p4: requires Python 2.7 or later.\n")
39 # On python2.7 where raw_input() and input() are both availble,
40 # we want raw_input's semantics, but aliased to input for python3
42 # support basestring in python3
44 if raw_input and input:
51 # Only labels/tags matching this will be imported/exported
52 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
54 # The block size is reduced automatically if required
55 defaultBlockSize
= 1<<20
57 p4_access_checked
= False
59 re_ko_keywords
= re
.compile(br
'\$(Id|Header)(:[^$\n]+)?\$')
60 re_k_keywords
= re
.compile(br
'\$(Id|Header|Author|Date|DateTime|Change|File|Revision)(:[^$\n]+)?\$')
62 def format_size_human_readable(num
):
63 """ Returns a number of units (typically bytes) formatted as a human-readable
67 return '{:d} B'.format(num
)
68 for unit
in ["Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
71 return "{:3.1f} {}B".format(num
, unit
)
72 return "{:.1f} YiB".format(num
)
74 def p4_build_cmd(cmd
):
75 """Build a suitable p4 command line.
77 This consolidates building and returning a p4 command line into one
78 location. It means that hooking into the environment, or other configuration
79 can be done more easily.
83 user
= gitConfig("git-p4.user")
85 real_cmd
+= ["-u",user
]
87 password
= gitConfig("git-p4.password")
89 real_cmd
+= ["-P", password
]
91 port
= gitConfig("git-p4.port")
93 real_cmd
+= ["-p", port
]
95 host
= gitConfig("git-p4.host")
97 real_cmd
+= ["-H", host
]
99 client
= gitConfig("git-p4.client")
101 real_cmd
+= ["-c", client
]
103 retries
= gitConfigInt("git-p4.retries")
105 # Perform 3 retries by default
108 # Provide a way to not pass this option by setting git-p4.retries to 0
109 real_cmd
+= ["-r", str(retries
)]
111 if not isinstance(cmd
, list):
112 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
116 # now check that we can actually talk to the server
117 global p4_access_checked
118 if not p4_access_checked
:
119 p4_access_checked
= True # suppress access checks in p4_check_access itself
125 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
126 This won't automatically add ".git" to a directory.
128 d
= read_pipe(["git", "--git-dir", path
, "rev-parse", "--git-dir"], True).strip()
129 if not d
or len(d
) == 0:
134 def chdir(path
, is_client_path
=False):
135 """Do chdir to the given path, and set the PWD environment
136 variable for use by P4. It does not look at getcwd() output.
137 Since we're not using the shell, it is necessary to set the
138 PWD environment variable explicitly.
140 Normally, expand the path to force it to be absolute. This
141 addresses the use of relative path names inside P4 settings,
142 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
143 as given; it looks for .p4config using PWD.
145 If is_client_path, the path was handed to us directly by p4,
146 and may be a symbolic link. Do not call os.getcwd() in this
147 case, because it will cause p4 to think that PWD is not inside
152 if not is_client_path
:
154 os
.environ
['PWD'] = path
157 """Return free space in bytes on the disk of the given dirname."""
158 if platform
.system() == 'Windows':
159 free_bytes
= ctypes
.c_ulonglong(0)
160 ctypes
.windll
.kernel32
.GetDiskFreeSpaceExW(ctypes
.c_wchar_p(os
.getcwd()), None, None, ctypes
.pointer(free_bytes
))
161 return free_bytes
.value
163 st
= os
.statvfs(os
.getcwd())
164 return st
.f_bavail
* st
.f_frsize
167 """ Terminate execution. Make sure that any running child processes have been wait()ed for before
173 sys
.stderr
.write(msg
+ "\n")
176 def prompt(prompt_text
):
177 """ Prompt the user to choose one of the choices
179 Choices are identified in the prompt_text by square brackets around
180 a single letter option.
182 choices
= set(m
.group(1) for m
in re
.finditer(r
"\[(.)\]", prompt_text
))
185 sys
.stdout
.write(prompt_text
)
187 response
=sys
.stdin
.readline().strip().lower()
190 response
= response
[0]
191 if response
in choices
:
194 # We need different encoding/decoding strategies for text data being passed
195 # around in pipes depending on python version
197 # For python3, always encode and decode as appropriate
198 def decode_text_stream(s
):
199 return s
.decode() if isinstance(s
, bytes
) else s
200 def encode_text_stream(s
):
201 return s
.encode() if isinstance(s
, str) else s
203 # For python2.7, pass read strings as-is, but also allow writing unicode
204 def decode_text_stream(s
):
206 def encode_text_stream(s
):
207 return s
.encode('utf_8') if isinstance(s
, unicode) else s
209 def decode_path(path
):
210 """Decode a given string (bytes or otherwise) using configured path encoding options
212 encoding
= gitConfig('git-p4.pathEncoding') or 'utf_8'
214 return path
.decode(encoding
, errors
='replace') if isinstance(path
, bytes
) else path
219 path
= path
.decode(encoding
, errors
='replace')
221 print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding
, path
))
224 def run_git_hook(cmd
, param
=[]):
225 """Execute a hook if the hook exists."""
227 sys
.stderr
.write("Looking for hook: %s\n" % cmd
)
230 hooks_path
= gitConfig("core.hooksPath")
231 if len(hooks_path
) <= 0:
232 hooks_path
= os
.path
.join(os
.environ
["GIT_DIR"], "hooks")
234 if not isinstance(param
, list):
237 # resolve hook file name, OS depdenent
238 hook_file
= os
.path
.join(hooks_path
, cmd
)
239 if platform
.system() == 'Windows':
240 if not os
.path
.isfile(hook_file
):
241 # look for the file with an extension
242 files
= glob
.glob(hook_file
+ ".*")
246 hook_file
= files
.pop()
247 while hook_file
.upper().endswith(".SAMPLE"):
248 # The file is a sample hook. We don't want it
250 hook_file
= files
.pop()
254 if not os
.path
.isfile(hook_file
) or not os
.access(hook_file
, os
.X_OK
):
257 return run_hook_command(hook_file
, param
) == 0
259 def run_hook_command(cmd
, param
):
260 """Executes a git hook command
261 cmd = the command line file to be executed. This can be
262 a file that is run by OS association.
264 param = a list of parameters to pass to the cmd command
266 On windows, the extension is checked to see if it should
267 be run with the Git for Windows Bash shell. If there
268 is no file extension, the file is deemed a bash shell
269 and will be handed off to sh.exe. Otherwise, Windows
270 will be called with the shell to handle the file assocation.
272 For non Windows operating systems, the file is called
277 if platform
.system() == 'Windows':
278 (root
,ext
) = os
.path
.splitext(cmd
)
280 exe_path
= os
.environ
.get("EXEPATH")
284 exe_path
= os
.path
.join(exe_path
, "bin")
285 cli
= [os
.path
.join(exe_path
, "SH.EXE")] + cli
288 return subprocess
.call(cli
, shell
=use_shell
)
291 def write_pipe(c
, stdin
):
293 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
295 expand
= not isinstance(c
, list)
296 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
298 val
= pipe
.write(stdin
)
301 die('Command failed: %s' % str(c
))
305 def p4_write_pipe(c
, stdin
):
306 real_cmd
= p4_build_cmd(c
)
307 if bytes
is not str and isinstance(stdin
, str):
308 stdin
= encode_text_stream(stdin
)
309 return write_pipe(real_cmd
, stdin
)
311 def read_pipe_full(c
):
312 """ Read output from command. Returns a tuple
313 of the return status, stdout text and stderr
317 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
319 expand
= not isinstance(c
, list)
320 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, shell
=expand
)
321 (out
, err
) = p
.communicate()
322 return (p
.returncode
, out
, decode_text_stream(err
))
324 def read_pipe(c
, ignore_error
=False, raw
=False):
325 """ Read output from command. Returns the output text on
326 success. On failure, terminates execution, unless
327 ignore_error is True, when it returns an empty string.
329 If raw is True, do not attempt to decode output text.
331 (retcode
, out
, err
) = read_pipe_full(c
)
336 die('Command failed: %s\nError: %s' % (str(c
), err
))
338 out
= decode_text_stream(out
)
341 def read_pipe_text(c
):
342 """ Read output from a command with trailing whitespace stripped.
343 On error, returns None.
345 (retcode
, out
, err
) = read_pipe_full(c
)
349 return decode_text_stream(out
).rstrip()
351 def p4_read_pipe(c
, ignore_error
=False, raw
=False):
352 real_cmd
= p4_build_cmd(c
)
353 return read_pipe(real_cmd
, ignore_error
, raw
=raw
)
355 def read_pipe_lines(c
, raw
=False):
357 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
359 expand
= not isinstance(c
, list)
360 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
362 lines
= pipe
.readlines()
364 lines
= [decode_text_stream(line
) for line
in lines
]
365 if pipe
.close() or p
.wait():
366 die('Command failed: %s' % str(c
))
369 def p4_read_pipe_lines(c
):
370 """Specifically invoke p4 on the command supplied. """
371 real_cmd
= p4_build_cmd(c
)
372 return read_pipe_lines(real_cmd
)
374 def p4_has_command(cmd
):
375 """Ask p4 for help on this command. If it returns an error, the
376 command does not exist in this version of p4."""
377 real_cmd
= p4_build_cmd(["help", cmd
])
378 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
379 stderr
=subprocess
.PIPE
)
381 return p
.returncode
== 0
383 def p4_has_move_command():
384 """See if the move command exists, that it supports -k, and that
385 it has not been administratively disabled. The arguments
386 must be correct, but the filenames do not have to exist. Use
387 ones with wildcards so even if they exist, it will fail."""
389 if not p4_has_command("move"):
391 cmd
= p4_build_cmd(["move", "-k", "@from", "@to"])
392 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
393 (out
, err
) = p
.communicate()
394 err
= decode_text_stream(err
)
395 # return code will be 1 in either case
396 if err
.find("Invalid option") >= 0:
398 if err
.find("disabled") >= 0:
400 # assume it failed because @... was invalid changelist
403 def system(cmd
, ignore_error
=False):
404 expand
= not isinstance(cmd
, list)
406 sys
.stderr
.write("executing %s\n" % str(cmd
))
407 retcode
= subprocess
.call(cmd
, shell
=expand
)
408 if retcode
and not ignore_error
:
409 raise CalledProcessError(retcode
, cmd
)
414 """Specifically invoke p4 as the system command. """
415 real_cmd
= p4_build_cmd(cmd
)
416 expand
= not isinstance(real_cmd
, list)
417 retcode
= subprocess
.call(real_cmd
, shell
=expand
)
419 raise CalledProcessError(retcode
, real_cmd
)
421 def die_bad_access(s
):
422 die("failure accessing depot: {0}".format(s
.rstrip()))
424 def p4_check_access(min_expiration
=1):
425 """ Check if we can access Perforce - account still logged in
427 results
= p4CmdList(["login", "-s"])
429 if len(results
) == 0:
430 # should never get here: always get either some results, or a p4ExitCode
431 assert("could not parse response from perforce")
435 if 'p4ExitCode' in result
:
436 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
437 die_bad_access("could not run p4")
439 code
= result
.get("code")
441 # we get here if we couldn't connect and there was nothing to unmarshal
442 die_bad_access("could not connect")
445 expiry
= result
.get("TicketExpiration")
448 if expiry
> min_expiration
:
452 die_bad_access("perforce ticket expires in {0} seconds".format(expiry
))
455 # account without a timeout - all ok
458 elif code
== "error":
459 data
= result
.get("data")
461 die_bad_access("p4 error: {0}".format(data
))
463 die_bad_access("unknown error")
467 die_bad_access("unknown error code {0}".format(code
))
469 _p4_version_string
= None
470 def p4_version_string():
471 """Read the version string, showing just the last line, which
472 hopefully is the interesting version bit.
475 Perforce - The Fast Software Configuration Management System.
476 Copyright 1995-2011 Perforce Software. All rights reserved.
477 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
479 global _p4_version_string
480 if not _p4_version_string
:
481 a
= p4_read_pipe_lines(["-V"])
482 _p4_version_string
= a
[-1].rstrip()
483 return _p4_version_string
485 def p4_integrate(src
, dest
):
486 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
488 def p4_sync(f
, *options
):
489 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
492 # forcibly add file names with wildcards
493 if wildcard_present(f
):
494 p4_system(["add", "-f", f
])
496 p4_system(["add", f
])
499 p4_system(["delete", wildcard_encode(f
)])
501 def p4_edit(f
, *options
):
502 p4_system(["edit"] + list(options
) + [wildcard_encode(f
)])
505 p4_system(["revert", wildcard_encode(f
)])
507 def p4_reopen(type, f
):
508 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
510 def p4_reopen_in_change(changelist
, files
):
511 cmd
= ["reopen", "-c", str(changelist
)] + files
514 def p4_move(src
, dest
):
515 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
517 def p4_last_change():
518 results
= p4CmdList(["changes", "-m", "1"], skip_info
=True)
519 return int(results
[0]['change'])
521 def p4_describe(change
, shelved
=False):
522 """Make sure it returns a valid result by checking for
523 the presence of field "time". Return a dict of the
526 cmd
= ["describe", "-s"]
531 ds
= p4CmdList(cmd
, skip_info
=True)
533 die("p4 describe -s %d did not return 1 result: %s" % (change
, str(ds
)))
537 if "p4ExitCode" in d
:
538 die("p4 describe -s %d exited with %d: %s" % (change
, d
["p4ExitCode"],
541 if d
["code"] == "error":
542 die("p4 describe -s %d returned error code: %s" % (change
, str(d
)))
545 die("p4 describe -s %d returned no \"time\": %s" % (change
, str(d
)))
550 # Canonicalize the p4 type and return a tuple of the
551 # base type, plus any modifiers. See "p4 help filetypes"
552 # for a list and explanation.
554 def split_p4_type(p4type
):
556 p4_filetypes_historical
= {
557 "ctempobj": "binary+Sw",
563 "tempobj": "binary+FSw",
564 "ubinary": "binary+F",
565 "uresource": "resource+F",
566 "uxbinary": "binary+Fx",
567 "xbinary": "binary+x",
569 "xtempobj": "binary+Swx",
571 "xunicode": "unicode+x",
574 if p4type
in p4_filetypes_historical
:
575 p4type
= p4_filetypes_historical
[p4type
]
577 s
= p4type
.split("+")
585 # return the raw p4 type of a file (text, text+ko, etc)
588 results
= p4CmdList(["fstat", "-T", "headType", wildcard_encode(f
)])
589 return results
[0]['headType']
592 # Given a type base and modifier, return a regexp matching
593 # the keywords that can be expanded in the file
595 def p4_keywords_regexp_for_type(base
, type_mods
):
596 if base
in ("text", "unicode", "binary"):
597 if "ko" in type_mods
:
598 return re_ko_keywords
599 elif "k" in type_mods
:
607 # Given a file, return a regexp matching the possible
608 # RCS keywords that will be expanded, or None for files
609 # with kw expansion turned off.
611 def p4_keywords_regexp_for_file(file):
612 if not os
.path
.exists(file):
615 (type_base
, type_mods
) = split_p4_type(p4_type(file))
616 return p4_keywords_regexp_for_type(type_base
, type_mods
)
618 def setP4ExecBit(file, mode
):
619 # Reopens an already open file and changes the execute bit to match
620 # the execute bit setting in the passed in mode.
624 if not isModeExec(mode
):
625 p4Type
= getP4OpenedType(file)
626 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
627 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
628 if p4Type
[-1] == "+":
629 p4Type
= p4Type
[0:-1]
631 p4_reopen(p4Type
, file)
633 def getP4OpenedType(file):
634 # Returns the perforce file type for the given file.
636 result
= p4_read_pipe(["opened", wildcard_encode(file)])
637 match
= re
.match(".*\((.+)\)( \*exclusive\*)?\r?$", result
)
639 return match
.group(1)
641 die("Could not determine file type for %s (result: '%s')" % (file, result
))
643 # Return the set of all p4 labels
644 def getP4Labels(depotPaths
):
646 if not isinstance(depotPaths
, list):
647 depotPaths
= [depotPaths
]
649 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
655 # Return the set of all git tags
658 for line
in read_pipe_lines(["git", "tag"]):
663 _diff_tree_pattern
= None
665 def parseDiffTreeEntry(entry
):
666 """Parses a single diff tree entry into its component elements.
668 See git-diff-tree(1) manpage for details about the format of the diff
669 output. This method returns a dictionary with the following elements:
671 src_mode - The mode of the source file
672 dst_mode - The mode of the destination file
673 src_sha1 - The sha1 for the source file
674 dst_sha1 - The sha1 fr the destination file
675 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
676 status_score - The score for the status (applicable for 'C' and 'R'
677 statuses). This is None if there is no score.
678 src - The path for the source file.
679 dst - The path for the destination file. This is only present for
680 copy or renames. If it is not present, this is None.
682 If the pattern is not matched, None is returned."""
684 global _diff_tree_pattern
685 if not _diff_tree_pattern
:
686 _diff_tree_pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
688 match
= _diff_tree_pattern
.match(entry
)
691 'src_mode': match
.group(1),
692 'dst_mode': match
.group(2),
693 'src_sha1': match
.group(3),
694 'dst_sha1': match
.group(4),
695 'status': match
.group(5),
696 'status_score': match
.group(6),
697 'src': match
.group(7),
698 'dst': match
.group(10)
702 def isModeExec(mode
):
703 # Returns True if the given git mode represents an executable file,
705 return mode
[-3:] == "755"
707 class P4Exception(Exception):
708 """ Base class for exceptions from the p4 client """
709 def __init__(self
, exit_code
):
710 self
.p4ExitCode
= exit_code
712 class P4ServerException(P4Exception
):
713 """ Base class for exceptions where we get some kind of marshalled up result from the server """
714 def __init__(self
, exit_code
, p4_result
):
715 super(P4ServerException
, self
).__init
__(exit_code
)
716 self
.p4_result
= p4_result
717 self
.code
= p4_result
[0]['code']
718 self
.data
= p4_result
[0]['data']
720 class P4RequestSizeException(P4ServerException
):
721 """ One of the maxresults or maxscanrows errors """
722 def __init__(self
, exit_code
, p4_result
, limit
):
723 super(P4RequestSizeException
, self
).__init
__(exit_code
, p4_result
)
726 class P4CommandException(P4Exception
):
727 """ Something went wrong calling p4 which means we have to give up """
728 def __init__(self
, msg
):
734 def isModeExecChanged(src_mode
, dst_mode
):
735 return isModeExec(src_mode
) != isModeExec(dst_mode
)
737 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None, skip_info
=False,
738 errors_as_exceptions
=False):
740 if not isinstance(cmd
, list):
747 cmd
= p4_build_cmd(cmd
)
749 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
751 # Use a temporary file to avoid deadlocks without
752 # subprocess.communicate(), which would put another copy
753 # of stdout into memory.
755 if stdin
is not None:
756 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
757 if not isinstance(stdin
, list):
758 stdin_file
.write(stdin
)
761 stdin_file
.write(encode_text_stream(i
))
762 stdin_file
.write(b
'\n')
766 p4
= subprocess
.Popen(cmd
,
769 stdout
=subprocess
.PIPE
)
774 entry
= marshal
.load(p4
.stdout
)
776 # Decode unmarshalled dict to use str keys and values, except for:
777 # - `data` which may contain arbitrary binary data
778 # - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text
780 for key
, value
in entry
.items():
782 if isinstance(value
, bytes
) and not (key
in ('data', 'path', 'clientFile') or key
.startswith('depotFile')):
783 value
= value
.decode()
784 decoded_entry
[key
] = value
785 # Parse out data if it's an error response
786 if decoded_entry
.get('code') == 'error' and 'data' in decoded_entry
:
787 decoded_entry
['data'] = decoded_entry
['data'].decode()
788 entry
= decoded_entry
790 if 'code' in entry
and entry
['code'] == 'info':
800 if errors_as_exceptions
:
802 data
= result
[0].get('data')
804 m
= re
.search('Too many rows scanned \(over (\d+)\)', data
)
806 m
= re
.search('Request too large \(over (\d+)\)', data
)
809 limit
= int(m
.group(1))
810 raise P4RequestSizeException(exitCode
, result
, limit
)
812 raise P4ServerException(exitCode
, result
)
814 raise P4Exception(exitCode
)
817 entry
["p4ExitCode"] = exitCode
823 list = p4CmdList(cmd
)
829 def p4Where(depotPath
):
830 if not depotPath
.endswith("/"):
832 depotPathLong
= depotPath
+ "..."
833 outputList
= p4CmdList(["where", depotPathLong
])
835 for entry
in outputList
:
836 if "depotFile" in entry
:
837 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
838 # The base path always ends with "/...".
839 entry_path
= decode_path(entry
['depotFile'])
840 if entry_path
.find(depotPath
) == 0 and entry_path
[-4:] == "/...":
843 elif "data" in entry
:
844 data
= entry
.get("data")
845 space
= data
.find(" ")
846 if data
[:space
] == depotPath
:
851 if output
["code"] == "error":
855 clientPath
= decode_path(output
['path'])
856 elif "data" in output
:
857 data
= output
.get("data")
858 lastSpace
= data
.rfind(b
" ")
859 clientPath
= decode_path(data
[lastSpace
+ 1:])
861 if clientPath
.endswith("..."):
862 clientPath
= clientPath
[:-3]
865 def currentGitBranch():
866 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
868 def isValidGitDir(path
):
869 return git_dir(path
) != None
871 def parseRevision(ref
):
872 return read_pipe("git rev-parse %s" % ref
).strip()
874 def branchExists(ref
):
875 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
879 def extractLogMessageFromGitCommit(commit
):
882 ## fixme: title is first line of commit, not 1st paragraph.
884 for log
in read_pipe_lines(["git", "cat-file", "commit", commit
]):
893 def extractSettingsGitLog(log
):
895 for line
in log
.split("\n"):
897 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
901 assignments
= m
.group(1).split (':')
902 for a
in assignments
:
904 key
= vals
[0].strip()
905 val
= ('='.join (vals
[1:])).strip()
906 if val
.endswith ('\"') and val
.startswith('"'):
911 paths
= values
.get("depot-paths")
913 paths
= values
.get("depot-path")
915 values
['depot-paths'] = paths
.split(',')
918 def gitBranchExists(branch
):
919 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
920 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
921 return proc
.wait() == 0;
923 def gitUpdateRef(ref
, newvalue
):
924 subprocess
.check_call(["git", "update-ref", ref
, newvalue
])
926 def gitDeleteRef(ref
):
927 subprocess
.check_call(["git", "update-ref", "-d", ref
])
931 def gitConfig(key
, typeSpecifier
=None):
932 if key
not in _gitConfig
:
933 cmd
= [ "git", "config" ]
935 cmd
+= [ typeSpecifier
]
937 s
= read_pipe(cmd
, ignore_error
=True)
938 _gitConfig
[key
] = s
.strip()
939 return _gitConfig
[key
]
941 def gitConfigBool(key
):
942 """Return a bool, using git config --bool. It is True only if the
943 variable is set to true, and False if set to false or not present
946 if key
not in _gitConfig
:
947 _gitConfig
[key
] = gitConfig(key
, '--bool') == "true"
948 return _gitConfig
[key
]
950 def gitConfigInt(key
):
951 if key
not in _gitConfig
:
952 cmd
= [ "git", "config", "--int", key
]
953 s
= read_pipe(cmd
, ignore_error
=True)
956 _gitConfig
[key
] = int(gitConfig(key
, '--int'))
958 _gitConfig
[key
] = None
959 return _gitConfig
[key
]
961 def gitConfigList(key
):
962 if key
not in _gitConfig
:
963 s
= read_pipe(["git", "config", "--get-all", key
], ignore_error
=True)
964 _gitConfig
[key
] = s
.strip().splitlines()
965 if _gitConfig
[key
] == ['']:
967 return _gitConfig
[key
]
969 def p4BranchesInGit(branchesAreInRemotes
=True):
970 """Find all the branches whose names start with "p4/", looking
971 in remotes or heads as specified by the argument. Return
972 a dictionary of { branch: revision } for each one found.
973 The branch names are the short names, without any
978 cmdline
= "git rev-parse --symbolic "
979 if branchesAreInRemotes
:
980 cmdline
+= "--remotes"
982 cmdline
+= "--branches"
984 for line
in read_pipe_lines(cmdline
):
988 if not line
.startswith('p4/'):
990 # special symbolic ref to p4/master
991 if line
== "p4/HEAD":
994 # strip off p4/ prefix
995 branch
= line
[len("p4/"):]
997 branches
[branch
] = parseRevision(line
)
1001 def branch_exists(branch
):
1002 """Make sure that the given ref name really exists."""
1004 cmd
= [ "git", "rev-parse", "--symbolic", "--verify", branch
]
1005 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
1006 out
, _
= p
.communicate()
1007 out
= decode_text_stream(out
)
1010 # expect exactly one line of output: the branch name
1011 return out
.rstrip() == branch
1013 def findUpstreamBranchPoint(head
= "HEAD"):
1014 branches
= p4BranchesInGit()
1015 # map from depot-path to branch name
1016 branchByDepotPath
= {}
1017 for branch
in branches
.keys():
1018 tip
= branches
[branch
]
1019 log
= extractLogMessageFromGitCommit(tip
)
1020 settings
= extractSettingsGitLog(log
)
1021 if "depot-paths" in settings
:
1022 paths
= ",".join(settings
["depot-paths"])
1023 branchByDepotPath
[paths
] = "remotes/p4/" + branch
1027 while parent
< 65535:
1028 commit
= head
+ "~%s" % parent
1029 log
= extractLogMessageFromGitCommit(commit
)
1030 settings
= extractSettingsGitLog(log
)
1031 if "depot-paths" in settings
:
1032 paths
= ",".join(settings
["depot-paths"])
1033 if paths
in branchByDepotPath
:
1034 return [branchByDepotPath
[paths
], settings
]
1038 return ["", settings
]
1040 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
1042 print("Creating/updating branch(es) in %s based on origin branch(es)"
1045 originPrefix
= "origin/p4/"
1047 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
1049 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
1052 headName
= line
[len(originPrefix
):]
1053 remoteHead
= localRefPrefix
+ headName
1056 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
1057 if ('depot-paths' not in original
1058 or 'change' not in original
):
1062 if not gitBranchExists(remoteHead
):
1064 print("creating %s" % remoteHead
)
1067 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
1068 if 'change' in settings
:
1069 if settings
['depot-paths'] == original
['depot-paths']:
1070 originP4Change
= int(original
['change'])
1071 p4Change
= int(settings
['change'])
1072 if originP4Change
> p4Change
:
1073 print("%s (%s) is newer than %s (%s). "
1074 "Updating p4 branch from origin."
1075 % (originHead
, originP4Change
,
1076 remoteHead
, p4Change
))
1079 print("Ignoring: %s was imported from %s while "
1080 "%s was imported from %s"
1081 % (originHead
, ','.join(original
['depot-paths']),
1082 remoteHead
, ','.join(settings
['depot-paths'])))
1085 system("git update-ref %s %s" % (remoteHead
, originHead
))
1087 def originP4BranchesExist():
1088 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
1091 def p4ParseNumericChangeRange(parts
):
1092 changeStart
= int(parts
[0][1:])
1093 if parts
[1] == '#head':
1094 changeEnd
= p4_last_change()
1096 changeEnd
= int(parts
[1])
1098 return (changeStart
, changeEnd
)
1100 def chooseBlockSize(blockSize
):
1104 return defaultBlockSize
1106 def p4ChangesForPaths(depotPaths
, changeRange
, requestedBlockSize
):
1109 # Parse the change range into start and end. Try to find integer
1110 # revision ranges as these can be broken up into blocks to avoid
1111 # hitting server-side limits (maxrows, maxscanresults). But if
1112 # that doesn't work, fall back to using the raw revision specifier
1113 # strings, without using block mode.
1115 if changeRange
is None or changeRange
== '':
1117 changeEnd
= p4_last_change()
1118 block_size
= chooseBlockSize(requestedBlockSize
)
1120 parts
= changeRange
.split(',')
1121 assert len(parts
) == 2
1123 (changeStart
, changeEnd
) = p4ParseNumericChangeRange(parts
)
1124 block_size
= chooseBlockSize(requestedBlockSize
)
1126 changeStart
= parts
[0][1:]
1127 changeEnd
= parts
[1]
1128 if requestedBlockSize
:
1129 die("cannot use --changes-block-size with non-numeric revisions")
1134 # Retrieve changes a block at a time, to prevent running
1135 # into a MaxResults/MaxScanRows error from the server. If
1136 # we _do_ hit one of those errors, turn down the block size
1142 end
= min(changeEnd
, changeStart
+ block_size
)
1143 revisionRange
= "%d,%d" % (changeStart
, end
)
1145 revisionRange
= "%s,%s" % (changeStart
, changeEnd
)
1147 for p
in depotPaths
:
1148 cmd
+= ["%s...@%s" % (p
, revisionRange
)]
1152 result
= p4CmdList(cmd
, errors_as_exceptions
=True)
1153 except P4RequestSizeException
as e
:
1155 block_size
= e
.limit
1156 elif block_size
> e
.limit
:
1157 block_size
= e
.limit
1159 block_size
= max(2, block_size
// 2)
1161 if verbose
: print("block size error, retrying with block size {0}".format(block_size
))
1163 except P4Exception
as e
:
1164 die('Error retrieving changes description ({0})'.format(e
.p4ExitCode
))
1166 # Insert changes in chronological order
1167 for entry
in reversed(result
):
1168 if 'change' not in entry
:
1170 changes
.add(int(entry
['change']))
1175 if end
>= changeEnd
:
1178 changeStart
= end
+ 1
1180 changes
= sorted(changes
)
1183 def p4PathStartsWith(path
, prefix
):
1184 # This method tries to remedy a potential mixed-case issue:
1186 # If UserA adds //depot/DirA/file1
1187 # and UserB adds //depot/dira/file2
1189 # we may or may not have a problem. If you have core.ignorecase=true,
1190 # we treat DirA and dira as the same directory
1191 if gitConfigBool("core.ignorecase"):
1192 return path
.lower().startswith(prefix
.lower())
1193 return path
.startswith(prefix
)
1195 def getClientSpec():
1196 """Look at the p4 client spec, create a View() object that contains
1197 all the mappings, and return it."""
1199 specList
= p4CmdList("client -o")
1200 if len(specList
) != 1:
1201 die('Output from "client -o" is %d lines, expecting 1' %
1204 # dictionary of all client parameters
1207 # the //client/ name
1208 client_name
= entry
["Client"]
1210 # just the keys that start with "View"
1211 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
1213 # hold this new View
1214 view
= View(client_name
)
1216 # append the lines, in order, to the view
1217 for view_num
in range(len(view_keys
)):
1218 k
= "View%d" % view_num
1219 if k
not in view_keys
:
1220 die("Expected view key %s missing" % k
)
1221 view
.append(entry
[k
])
1225 def getClientRoot():
1226 """Grab the client directory."""
1228 output
= p4CmdList("client -o")
1229 if len(output
) != 1:
1230 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
1233 if "Root" not in entry
:
1234 die('Client has no "Root"')
1236 return entry
["Root"]
1239 # P4 wildcards are not allowed in filenames. P4 complains
1240 # if you simply add them, but you can force it with "-f", in
1241 # which case it translates them into %xx encoding internally.
1243 def wildcard_decode(path
):
1244 # Search for and fix just these four characters. Do % last so
1245 # that fixing it does not inadvertently create new %-escapes.
1246 # Cannot have * in a filename in windows; untested as to
1247 # what p4 would do in such a case.
1248 if not platform
.system() == "Windows":
1249 path
= path
.replace("%2A", "*")
1250 path
= path
.replace("%23", "#") \
1251 .replace("%40", "@") \
1252 .replace("%25", "%")
1255 def wildcard_encode(path
):
1256 # do % first to avoid double-encoding the %s introduced here
1257 path
= path
.replace("%", "%25") \
1258 .replace("*", "%2A") \
1259 .replace("#", "%23") \
1260 .replace("@", "%40")
1263 def wildcard_present(path
):
1264 m
= re
.search("[*#@%]", path
)
1265 return m
is not None
1267 class LargeFileSystem(object):
1268 """Base class for large file system support."""
1270 def __init__(self
, writeToGitStream
):
1271 self
.largeFiles
= set()
1272 self
.writeToGitStream
= writeToGitStream
1274 def generatePointer(self
, cloneDestination
, contentFile
):
1275 """Return the content of a pointer file that is stored in Git instead of
1276 the actual content."""
1277 assert False, "Method 'generatePointer' required in " + self
.__class
__.__name
__
1279 def pushFile(self
, localLargeFile
):
1280 """Push the actual content which is not stored in the Git repository to
1282 assert False, "Method 'pushFile' required in " + self
.__class
__.__name
__
1284 def hasLargeFileExtension(self
, relPath
):
1285 return functools
.reduce(
1286 lambda a
, b
: a
or b
,
1287 [relPath
.endswith('.' + e
) for e
in gitConfigList('git-p4.largeFileExtensions')],
1291 def generateTempFile(self
, contents
):
1292 contentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
1294 contentFile
.write(d
)
1296 return contentFile
.name
1298 def exceedsLargeFileThreshold(self
, relPath
, contents
):
1299 if gitConfigInt('git-p4.largeFileThreshold'):
1300 contentsSize
= sum(len(d
) for d
in contents
)
1301 if contentsSize
> gitConfigInt('git-p4.largeFileThreshold'):
1303 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1304 contentsSize
= sum(len(d
) for d
in contents
)
1305 if contentsSize
<= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1307 contentTempFile
= self
.generateTempFile(contents
)
1308 compressedContentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=True)
1309 with zipfile
.ZipFile(compressedContentFile
, mode
='w') as zf
:
1310 zf
.write(contentTempFile
, compress_type
=zipfile
.ZIP_DEFLATED
)
1311 compressedContentsSize
= zf
.infolist()[0].compress_size
1312 os
.remove(contentTempFile
)
1313 if compressedContentsSize
> gitConfigInt('git-p4.largeFileCompressedThreshold'):
1317 def addLargeFile(self
, relPath
):
1318 self
.largeFiles
.add(relPath
)
1320 def removeLargeFile(self
, relPath
):
1321 self
.largeFiles
.remove(relPath
)
1323 def isLargeFile(self
, relPath
):
1324 return relPath
in self
.largeFiles
1326 def processContent(self
, git_mode
, relPath
, contents
):
1327 """Processes the content of git fast import. This method decides if a
1328 file is stored in the large file system and handles all necessary
1330 if self
.exceedsLargeFileThreshold(relPath
, contents
) or self
.hasLargeFileExtension(relPath
):
1331 contentTempFile
= self
.generateTempFile(contents
)
1332 (pointer_git_mode
, contents
, localLargeFile
) = self
.generatePointer(contentTempFile
)
1333 if pointer_git_mode
:
1334 git_mode
= pointer_git_mode
1336 # Move temp file to final location in large file system
1337 largeFileDir
= os
.path
.dirname(localLargeFile
)
1338 if not os
.path
.isdir(largeFileDir
):
1339 os
.makedirs(largeFileDir
)
1340 shutil
.move(contentTempFile
, localLargeFile
)
1341 self
.addLargeFile(relPath
)
1342 if gitConfigBool('git-p4.largeFilePush'):
1343 self
.pushFile(localLargeFile
)
1345 sys
.stderr
.write("%s moved to large file system (%s)\n" % (relPath
, localLargeFile
))
1346 return (git_mode
, contents
)
1348 class MockLFS(LargeFileSystem
):
1349 """Mock large file system for testing."""
1351 def generatePointer(self
, contentFile
):
1352 """The pointer content is the original content prefixed with "pointer-".
1353 The local filename of the large file storage is derived from the file content.
1355 with
open(contentFile
, 'r') as f
:
1358 pointerContents
= 'pointer-' + content
1359 localLargeFile
= os
.path
.join(os
.getcwd(), '.git', 'mock-storage', 'local', content
[:-1])
1360 return (gitMode
, pointerContents
, localLargeFile
)
1362 def pushFile(self
, localLargeFile
):
1363 """The remote filename of the large file storage is the same as the local
1364 one but in a different directory.
1366 remotePath
= os
.path
.join(os
.path
.dirname(localLargeFile
), '..', 'remote')
1367 if not os
.path
.exists(remotePath
):
1368 os
.makedirs(remotePath
)
1369 shutil
.copyfile(localLargeFile
, os
.path
.join(remotePath
, os
.path
.basename(localLargeFile
)))
1371 class GitLFS(LargeFileSystem
):
1372 """Git LFS as backend for the git-p4 large file system.
1373 See https://git-lfs.github.com/ for details."""
1375 def __init__(self
, *args
):
1376 LargeFileSystem
.__init
__(self
, *args
)
1377 self
.baseGitAttributes
= []
1379 def generatePointer(self
, contentFile
):
1380 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1381 mode and content which is stored in the Git repository instead of
1382 the actual content. Return also the new location of the actual
1385 if os
.path
.getsize(contentFile
) == 0:
1386 return (None, '', None)
1388 pointerProcess
= subprocess
.Popen(
1389 ['git', 'lfs', 'pointer', '--file=' + contentFile
],
1390 stdout
=subprocess
.PIPE
1392 pointerFile
= decode_text_stream(pointerProcess
.stdout
.read())
1393 if pointerProcess
.wait():
1394 os
.remove(contentFile
)
1395 die('git-lfs pointer command failed. Did you install the extension?')
1397 # Git LFS removed the preamble in the output of the 'pointer' command
1398 # starting from version 1.2.0. Check for the preamble here to support
1400 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1401 if pointerFile
.startswith('Git LFS pointer for'):
1402 pointerFile
= re
.sub(r
'Git LFS pointer for.*\n\n', '', pointerFile
)
1404 oid
= re
.search(r
'^oid \w+:(\w+)', pointerFile
, re
.MULTILINE
).group(1)
1405 # if someone use external lfs.storage ( not in local repo git )
1406 lfs_path
= gitConfig('lfs.storage')
1409 if not os
.path
.isabs(lfs_path
):
1410 lfs_path
= os
.path
.join(os
.getcwd(), '.git', lfs_path
)
1411 localLargeFile
= os
.path
.join(
1413 'objects', oid
[:2], oid
[2:4],
1416 # LFS Spec states that pointer files should not have the executable bit set.
1418 return (gitMode
, pointerFile
, localLargeFile
)
1420 def pushFile(self
, localLargeFile
):
1421 uploadProcess
= subprocess
.Popen(
1422 ['git', 'lfs', 'push', '--object-id', 'origin', os
.path
.basename(localLargeFile
)]
1424 if uploadProcess
.wait():
1425 die('git-lfs push command failed. Did you define a remote?')
1427 def generateGitAttributes(self
):
1429 self
.baseGitAttributes
+
1433 '# Git LFS (see https://git-lfs.github.com/)\n',
1436 ['*.' + f
.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1437 for f
in sorted(gitConfigList('git-p4.largeFileExtensions'))
1439 ['/' + f
.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1440 for f
in sorted(self
.largeFiles
) if not self
.hasLargeFileExtension(f
)
1444 def addLargeFile(self
, relPath
):
1445 LargeFileSystem
.addLargeFile(self
, relPath
)
1446 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1448 def removeLargeFile(self
, relPath
):
1449 LargeFileSystem
.removeLargeFile(self
, relPath
)
1450 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1452 def processContent(self
, git_mode
, relPath
, contents
):
1453 if relPath
== '.gitattributes':
1454 self
.baseGitAttributes
= contents
1455 return (git_mode
, self
.generateGitAttributes())
1457 return LargeFileSystem
.processContent(self
, git_mode
, relPath
, contents
)
1460 delete_actions
= ( "delete", "move/delete", "purge" )
1461 add_actions
= ( "add", "branch", "move/add" )
1464 self
.usage
= "usage: %prog [options]"
1465 self
.needsGit
= True
1466 self
.verbose
= False
1468 # This is required for the "append" update_shelve action
1469 def ensure_value(self
, attr
, value
):
1470 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
1471 setattr(self
, attr
, value
)
1472 return getattr(self
, attr
)
1476 self
.userMapFromPerforceServer
= False
1477 self
.myP4UserId
= None
1481 return self
.myP4UserId
1483 results
= p4CmdList("user -o")
1486 self
.myP4UserId
= r
['User']
1488 die("Could not find your p4 user id")
1490 def p4UserIsMe(self
, p4User
):
1491 # return True if the given p4 user is actually me
1492 me
= self
.p4UserId()
1493 if not p4User
or p4User
!= me
:
1498 def getUserCacheFilename(self
):
1499 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
1500 return home
+ "/.gitp4-usercache.txt"
1502 def getUserMapFromPerforceServer(self
):
1503 if self
.userMapFromPerforceServer
:
1508 for output
in p4CmdList("users"):
1509 if "User" not in output
:
1511 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
1512 self
.emails
[output
["Email"]] = output
["User"]
1514 mapUserConfigRegex
= re
.compile(r
"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re
.VERBOSE
)
1515 for mapUserConfig
in gitConfigList("git-p4.mapUser"):
1516 mapUser
= mapUserConfigRegex
.findall(mapUserConfig
)
1517 if mapUser
and len(mapUser
[0]) == 3:
1518 user
= mapUser
[0][0]
1519 fullname
= mapUser
[0][1]
1520 email
= mapUser
[0][2]
1521 self
.users
[user
] = fullname
+ " <" + email
+ ">"
1522 self
.emails
[email
] = user
1525 for (key
, val
) in self
.users
.items():
1526 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
1528 open(self
.getUserCacheFilename(), 'w').write(s
)
1529 self
.userMapFromPerforceServer
= True
1531 def loadUserMapFromCache(self
):
1533 self
.userMapFromPerforceServer
= False
1535 cache
= open(self
.getUserCacheFilename(), 'r')
1536 lines
= cache
.readlines()
1539 entry
= line
.strip().split("\t")
1540 self
.users
[entry
[0]] = entry
[1]
1542 self
.getUserMapFromPerforceServer()
1544 class P4Debug(Command
):
1546 Command
.__init
__(self
)
1548 self
.description
= "A tool to debug the output of p4 -G."
1549 self
.needsGit
= False
1551 def run(self
, args
):
1553 for output
in p4CmdList(args
):
1554 print('Element: %d' % j
)
1559 class P4RollBack(Command
):
1561 Command
.__init
__(self
)
1563 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
1565 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
1566 self
.rollbackLocalBranches
= False
1568 def run(self
, args
):
1571 maxChange
= int(args
[0])
1573 if "p4ExitCode" in p4Cmd("changes -m 1"):
1574 die("Problems executing p4");
1576 if self
.rollbackLocalBranches
:
1577 refPrefix
= "refs/heads/"
1578 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
1580 refPrefix
= "refs/remotes/"
1581 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
1584 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
1586 ref
= refPrefix
+ line
1587 log
= extractLogMessageFromGitCommit(ref
)
1588 settings
= extractSettingsGitLog(log
)
1590 depotPaths
= settings
['depot-paths']
1591 change
= settings
['change']
1595 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
1596 for p
in depotPaths
]))) == 0:
1597 print("Branch %s did not exist at change %s, deleting." % (ref
, maxChange
))
1598 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
1601 while change
and int(change
) > maxChange
:
1604 print("%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
))
1605 system("git update-ref %s \"%s^\"" % (ref
, ref
))
1606 log
= extractLogMessageFromGitCommit(ref
)
1607 settings
= extractSettingsGitLog(log
)
1610 depotPaths
= settings
['depot-paths']
1611 change
= settings
['change']
1614 print("%s rewound to %s" % (ref
, change
))
1618 class P4Submit(Command
, P4UserMap
):
1620 conflict_behavior_choices
= ("ask", "skip", "quit")
1623 Command
.__init
__(self
)
1624 P4UserMap
.__init
__(self
)
1626 optparse
.make_option("--origin", dest
="origin"),
1627 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
1628 # preserve the user, requires relevant p4 permissions
1629 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
1630 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
1631 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
1632 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
1633 optparse
.make_option("--conflict", dest
="conflict_behavior",
1634 choices
=self
.conflict_behavior_choices
),
1635 optparse
.make_option("--branch", dest
="branch"),
1636 optparse
.make_option("--shelve", dest
="shelve", action
="store_true",
1637 help="Shelve instead of submit. Shelved files are reverted, "
1638 "restoring the workspace to the state before the shelve"),
1639 optparse
.make_option("--update-shelve", dest
="update_shelve", action
="append", type="int",
1640 metavar
="CHANGELIST",
1641 help="update an existing shelved changelist, implies --shelve, "
1642 "repeat in-order for multiple shelved changelists"),
1643 optparse
.make_option("--commit", dest
="commit", metavar
="COMMIT",
1644 help="submit only the specified commit(s), one commit or xxx..xxx"),
1645 optparse
.make_option("--disable-rebase", dest
="disable_rebase", action
="store_true",
1646 help="Disable rebase after submit is completed. Can be useful if you "
1647 "work from a local git branch that is not master"),
1648 optparse
.make_option("--disable-p4sync", dest
="disable_p4sync", action
="store_true",
1649 help="Skip Perforce sync of p4/master after submit or shelve"),
1650 optparse
.make_option("--no-verify", dest
="no_verify", action
="store_true",
1651 help="Bypass p4-pre-submit and p4-changelist hooks"),
1653 self
.description
= """Submit changes from git to the perforce depot.\n
1654 The `p4-pre-submit` hook is executed if it exists and is executable. It
1655 can be bypassed with the `--no-verify` command line option. The hook takes
1656 no parameters and nothing from standard input. Exiting with a non-zero status
1657 from this script prevents `git-p4 submit` from launching.
1659 One usage scenario is to run unit tests in the hook.
1661 The `p4-prepare-changelist` hook is executed right after preparing the default
1662 changelist message and before the editor is started. It takes one parameter,
1663 the name of the file that contains the changelist text. Exiting with a non-zero
1664 status from the script will abort the process.
1666 The purpose of the hook is to edit the message file in place, and it is not
1667 supressed by the `--no-verify` option. This hook is called even if
1668 `--prepare-p4-only` is set.
1670 The `p4-changelist` hook is executed after the changelist message has been
1671 edited by the user. It can be bypassed with the `--no-verify` option. It
1672 takes a single parameter, the name of the file that holds the proposed
1673 changelist text. Exiting with a non-zero status causes the command to abort.
1675 The hook is allowed to edit the changelist file and can be used to normalize
1676 the text into some project standard format. It can also be used to refuse the
1677 Submit after inspect the message file.
1679 The `p4-post-changelist` hook is invoked after the submit has successfully
1680 occurred in P4. It takes no parameters and is meant primarily for notification
1681 and cannot affect the outcome of the git p4 submit action.
1684 self
.usage
+= " [name of git branch to submit into perforce depot]"
1686 self
.detectRenames
= False
1687 self
.preserveUser
= gitConfigBool("git-p4.preserveUser")
1688 self
.dry_run
= False
1690 self
.update_shelve
= list()
1692 self
.disable_rebase
= gitConfigBool("git-p4.disableRebase")
1693 self
.disable_p4sync
= gitConfigBool("git-p4.disableP4Sync")
1694 self
.prepare_p4_only
= False
1695 self
.conflict_behavior
= None
1696 self
.isWindows
= (platform
.system() == "Windows")
1697 self
.exportLabels
= False
1698 self
.p4HasMoveCommand
= p4_has_move_command()
1700 self
.no_verify
= False
1702 if gitConfig('git-p4.largeFileSystem'):
1703 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1706 if len(p4CmdList("opened ...")) > 0:
1707 die("You have files opened with perforce! Close them before starting the sync.")
1709 def separate_jobs_from_description(self
, message
):
1710 """Extract and return a possible Jobs field in the commit
1711 message. It goes into a separate section in the p4 change
1714 A jobs line starts with "Jobs:" and looks like a new field
1715 in a form. Values are white-space separated on the same
1716 line or on following lines that start with a tab.
1718 This does not parse and extract the full git commit message
1719 like a p4 form. It just sees the Jobs: line as a marker
1720 to pass everything from then on directly into the p4 form,
1721 but outside the description section.
1723 Return a tuple (stripped log message, jobs string)."""
1725 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
1727 return (message
, None)
1729 jobtext
= message
[m
.start():]
1730 stripped_message
= message
[:m
.start()].rstrip()
1731 return (stripped_message
, jobtext
)
1733 def prepareLogMessage(self
, template
, message
, jobs
):
1734 """Edits the template returned from "p4 change -o" to insert
1735 the message in the Description field, and the jobs text in
1739 inDescriptionSection
= False
1741 for line
in template
.split("\n"):
1742 if line
.startswith("#"):
1743 result
+= line
+ "\n"
1746 if inDescriptionSection
:
1747 if line
.startswith("Files:") or line
.startswith("Jobs:"):
1748 inDescriptionSection
= False
1749 # insert Jobs section
1751 result
+= jobs
+ "\n"
1755 if line
.startswith("Description:"):
1756 inDescriptionSection
= True
1758 for messageLine
in message
.split("\n"):
1759 line
+= "\t" + messageLine
+ "\n"
1761 result
+= line
+ "\n"
1765 def patchRCSKeywords(self
, file, regexp
):
1766 # Attempt to zap the RCS keywords in a p4 controlled file matching the given regex
1767 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
1769 with os
.fdopen(handle
, "wb") as outFile
, open(file, "rb") as inFile
:
1770 for line
in inFile
.readlines():
1771 outFile
.write(regexp
.sub(br
'$\1$', line
))
1772 # Forcibly overwrite the original file
1774 shutil
.move(outFileName
, file)
1776 # cleanup our temporary file
1777 os
.unlink(outFileName
)
1778 print("Failed to strip RCS keywords in %s" % file)
1781 print("Patched up RCS keywords in %s" % file)
1783 def p4UserForCommit(self
,id):
1784 # Return the tuple (perforce user,git email) for a given git commit id
1785 self
.getUserMapFromPerforceServer()
1786 gitEmail
= read_pipe(["git", "log", "--max-count=1",
1787 "--format=%ae", id])
1788 gitEmail
= gitEmail
.strip()
1789 if gitEmail
not in self
.emails
:
1790 return (None,gitEmail
)
1792 return (self
.emails
[gitEmail
],gitEmail
)
1794 def checkValidP4Users(self
,commits
):
1795 # check if any git authors cannot be mapped to p4 users
1797 (user
,email
) = self
.p4UserForCommit(id)
1799 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
1800 if gitConfigBool("git-p4.allowMissingP4Users"):
1803 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
1805 def lastP4Changelist(self
):
1806 # Get back the last changelist number submitted in this client spec. This
1807 # then gets used to patch up the username in the change. If the same
1808 # client spec is being used by multiple processes then this might go
1810 results
= p4CmdList("client -o") # find the current client
1814 client
= r
['Client']
1817 die("could not get client spec")
1818 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
1822 die("Could not get changelist number for last submit - cannot patch up user details")
1824 def modifyChangelistUser(self
, changelist
, newUser
):
1825 # fixup the user field of a changelist after it has been submitted.
1826 changes
= p4CmdList("change -o %s" % changelist
)
1827 if len(changes
) != 1:
1828 die("Bad output from p4 change modifying %s to user %s" %
1829 (changelist
, newUser
))
1832 if c
['User'] == newUser
: return # nothing to do
1834 # p4 does not understand format version 3 and above
1835 input = marshal
.dumps(c
, 2)
1837 result
= p4CmdList("change -f -i", stdin
=input)
1840 if r
['code'] == 'error':
1841 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1843 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1845 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1847 def canChangeChangelists(self
):
1848 # check to see if we have p4 admin or super-user permissions, either of
1849 # which are required to modify changelists.
1850 results
= p4CmdList(["protects", self
.depotPath
])
1853 if r
['perm'] == 'admin':
1855 if r
['perm'] == 'super':
1859 def prepareSubmitTemplate(self
, changelist
=None):
1860 """Run "p4 change -o" to grab a change specification template.
1861 This does not use "p4 -G", as it is nice to keep the submission
1862 template in original order, since a human might edit it.
1864 Remove lines in the Files section that show changes to files
1865 outside the depot path we're committing into."""
1867 [upstream
, settings
] = findUpstreamBranchPoint()
1870 # A Perforce Change Specification.
1872 # Change: The change number. 'new' on a new changelist.
1873 # Date: The date this specification was last modified.
1874 # Client: The client on which the changelist was created. Read-only.
1875 # User: The user who created the changelist.
1876 # Status: Either 'pending' or 'submitted'. Read-only.
1877 # Type: Either 'public' or 'restricted'. Default is 'public'.
1878 # Description: Comments about the changelist. Required.
1879 # Jobs: What opened jobs are to be closed by this changelist.
1880 # You may delete jobs from this list. (New changelists only.)
1881 # Files: What opened files from the default changelist are to be added
1882 # to this changelist. You may delete files from this list.
1883 # (New changelists only.)
1886 inFilesSection
= False
1888 args
= ['change', '-o']
1890 args
.append(str(changelist
))
1891 for entry
in p4CmdList(args
):
1892 if 'code' not in entry
:
1894 if entry
['code'] == 'stat':
1895 change_entry
= entry
1897 if not change_entry
:
1898 die('Failed to decode output of p4 change -o')
1899 for key
, value
in change_entry
.items():
1900 if key
.startswith('File'):
1901 if 'depot-paths' in settings
:
1902 if not [p
for p
in settings
['depot-paths']
1903 if p4PathStartsWith(value
, p
)]:
1906 if not p4PathStartsWith(value
, self
.depotPath
):
1908 files_list
.append(value
)
1910 # Output in the order expected by prepareLogMessage
1911 for key
in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1912 if key
not in change_entry
:
1915 template
+= key
+ ':'
1916 if key
== 'Description':
1918 for field_line
in change_entry
[key
].splitlines():
1919 template
+= '\t'+field_line
+'\n'
1920 if len(files_list
) > 0:
1922 template
+= 'Files:\n'
1923 for path
in files_list
:
1924 template
+= '\t'+path
+'\n'
1927 def edit_template(self
, template_file
):
1928 """Invoke the editor to let the user change the submission
1929 message. Return true if okay to continue with the submit."""
1931 # if configured to skip the editing part, just submit
1932 if gitConfigBool("git-p4.skipSubmitEdit"):
1935 # look at the modification time, to check later if the user saved
1937 mtime
= os
.stat(template_file
).st_mtime
1940 if "P4EDITOR" in os
.environ
and (os
.environ
.get("P4EDITOR") != ""):
1941 editor
= os
.environ
.get("P4EDITOR")
1943 editor
= read_pipe("git var GIT_EDITOR").strip()
1944 system(["sh", "-c", ('%s "$@"' % editor
), editor
, template_file
])
1946 # If the file was not saved, prompt to see if this patch should
1947 # be skipped. But skip this verification step if configured so.
1948 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1951 # modification time updated means user saved the file
1952 if os
.stat(template_file
).st_mtime
> mtime
:
1955 response
= prompt("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1961 def get_diff_description(self
, editedFiles
, filesToAdd
, symlinks
):
1963 if "P4DIFF" in os
.environ
:
1964 del(os
.environ
["P4DIFF"])
1966 for editedFile
in editedFiles
:
1967 diff
+= p4_read_pipe(['diff', '-du',
1968 wildcard_encode(editedFile
)])
1972 for newFile
in filesToAdd
:
1973 newdiff
+= "==== new file ====\n"
1974 newdiff
+= "--- /dev/null\n"
1975 newdiff
+= "+++ %s\n" % newFile
1977 is_link
= os
.path
.islink(newFile
)
1978 expect_link
= newFile
in symlinks
1980 if is_link
and expect_link
:
1981 newdiff
+= "+%s\n" % os
.readlink(newFile
)
1983 f
= open(newFile
, "r")
1985 for line
in f
.readlines():
1986 newdiff
+= "+" + line
1987 except UnicodeDecodeError:
1988 pass # Found non-text data and skip, since diff description should only include text
1991 return (diff
+ newdiff
).replace('\r\n', '\n')
1993 def applyCommit(self
, id):
1994 """Apply one commit, return True if it succeeded."""
1996 print("Applying", read_pipe(["git", "show", "-s",
1997 "--format=format:%h %s", id]))
1999 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
2001 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
2003 filesToChangeType
= set()
2004 filesToDelete
= set()
2006 pureRenameCopy
= set()
2008 filesToChangeExecBit
= {}
2012 diff
= parseDiffTreeEntry(line
)
2013 modifier
= diff
['status']
2015 all_files
.append(path
)
2019 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
2020 filesToChangeExecBit
[path
] = diff
['dst_mode']
2021 editedFiles
.add(path
)
2022 elif modifier
== "A":
2023 filesToAdd
.add(path
)
2024 filesToChangeExecBit
[path
] = diff
['dst_mode']
2025 if path
in filesToDelete
:
2026 filesToDelete
.remove(path
)
2028 dst_mode
= int(diff
['dst_mode'], 8)
2029 if dst_mode
== 0o120000:
2032 elif modifier
== "D":
2033 filesToDelete
.add(path
)
2034 if path
in filesToAdd
:
2035 filesToAdd
.remove(path
)
2036 elif modifier
== "C":
2037 src
, dest
= diff
['src'], diff
['dst']
2038 all_files
.append(dest
)
2039 p4_integrate(src
, dest
)
2040 pureRenameCopy
.add(dest
)
2041 if diff
['src_sha1'] != diff
['dst_sha1']:
2043 pureRenameCopy
.discard(dest
)
2044 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
2046 pureRenameCopy
.discard(dest
)
2047 filesToChangeExecBit
[dest
] = diff
['dst_mode']
2049 # turn off read-only attribute
2050 os
.chmod(dest
, stat
.S_IWRITE
)
2052 editedFiles
.add(dest
)
2053 elif modifier
== "R":
2054 src
, dest
= diff
['src'], diff
['dst']
2055 all_files
.append(dest
)
2056 if self
.p4HasMoveCommand
:
2057 p4_edit(src
) # src must be open before move
2058 p4_move(src
, dest
) # opens for (move/delete, move/add)
2060 p4_integrate(src
, dest
)
2061 if diff
['src_sha1'] != diff
['dst_sha1']:
2064 pureRenameCopy
.add(dest
)
2065 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
2066 if not self
.p4HasMoveCommand
:
2067 p4_edit(dest
) # with move: already open, writable
2068 filesToChangeExecBit
[dest
] = diff
['dst_mode']
2069 if not self
.p4HasMoveCommand
:
2071 os
.chmod(dest
, stat
.S_IWRITE
)
2073 filesToDelete
.add(src
)
2074 editedFiles
.add(dest
)
2075 elif modifier
== "T":
2076 filesToChangeType
.add(path
)
2078 die("unknown modifier %s for %s" % (modifier
, path
))
2080 diffcmd
= "git diff-tree --full-index -p \"%s\"" % (id)
2081 patchcmd
= diffcmd
+ " | git apply "
2082 tryPatchCmd
= patchcmd
+ "--check -"
2083 applyPatchCmd
= patchcmd
+ "--check --apply -"
2084 patch_succeeded
= True
2087 print("TryPatch: %s" % tryPatchCmd
)
2089 if os
.system(tryPatchCmd
) != 0:
2090 fixed_rcs_keywords
= False
2091 patch_succeeded
= False
2092 print("Unfortunately applying the change failed!")
2094 # Patch failed, maybe it's just RCS keyword woes. Look through
2095 # the patch to see if that's possible.
2096 if gitConfigBool("git-p4.attemptRCSCleanup"):
2099 for file in editedFiles | filesToDelete
:
2100 # did this file's delta contain RCS keywords?
2101 regexp
= p4_keywords_regexp_for_file(file)
2103 # this file is a possibility...look for RCS keywords.
2104 for line
in read_pipe_lines(
2105 ["git", "diff", "%s^..%s" % (id, id), file],
2107 if regexp
.search(line
):
2109 print("got keyword match on %s in %s in %s" % (regex
.pattern
, line
, file))
2110 kwfiles
[file] = regexp
2113 for file, regexp
in kwfiles
.items():
2115 print("zapping %s with %s" % (line
, regexp
.pattern
))
2116 # File is being deleted, so not open in p4. Must
2117 # disable the read-only bit on windows.
2118 if self
.isWindows
and file not in editedFiles
:
2119 os
.chmod(file, stat
.S_IWRITE
)
2120 self
.patchRCSKeywords(file, kwfiles
[file])
2121 fixed_rcs_keywords
= True
2123 if fixed_rcs_keywords
:
2124 print("Retrying the patch with RCS keywords cleaned up")
2125 if os
.system(tryPatchCmd
) == 0:
2126 patch_succeeded
= True
2127 print("Patch succeesed this time with RCS keywords cleaned")
2129 if not patch_succeeded
:
2130 for f
in editedFiles
:
2135 # Apply the patch for real, and do add/delete/+x handling.
2137 system(applyPatchCmd
)
2139 for f
in filesToChangeType
:
2140 p4_edit(f
, "-t", "auto")
2141 for f
in filesToAdd
:
2143 for f
in filesToDelete
:
2147 # Set/clear executable bits
2148 for f
in filesToChangeExecBit
.keys():
2149 mode
= filesToChangeExecBit
[f
]
2150 setP4ExecBit(f
, mode
)
2153 if len(self
.update_shelve
) > 0:
2154 update_shelve
= self
.update_shelve
.pop(0)
2155 p4_reopen_in_change(update_shelve
, all_files
)
2158 # Build p4 change description, starting with the contents
2159 # of the git commit message.
2161 logMessage
= extractLogMessageFromGitCommit(id)
2162 logMessage
= logMessage
.strip()
2163 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
2165 template
= self
.prepareSubmitTemplate(update_shelve
)
2166 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
2168 if self
.preserveUser
:
2169 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
2171 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
2172 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
2173 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
2174 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
2176 separatorLine
= "######## everything below this line is just the diff #######\n"
2177 if not self
.prepare_p4_only
:
2178 submitTemplate
+= separatorLine
2179 submitTemplate
+= self
.get_diff_description(editedFiles
, filesToAdd
, symlinks
)
2181 (handle
, fileName
) = tempfile
.mkstemp()
2182 tmpFile
= os
.fdopen(handle
, "w+b")
2184 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
2185 tmpFile
.write(encode_text_stream(submitTemplate
))
2191 # Allow the hook to edit the changelist text before presenting it
2193 if not run_git_hook("p4-prepare-changelist", [fileName
]):
2196 if self
.prepare_p4_only
:
2198 # Leave the p4 tree prepared, and the submit template around
2199 # and let the user decide what to do next
2203 print("P4 workspace prepared for submission.")
2204 print("To submit or revert, go to client workspace")
2205 print(" " + self
.clientPath
)
2207 print("To submit, use \"p4 submit\" to write a new description,")
2208 print("or \"p4 submit -i <%s\" to use the one prepared by" \
2209 " \"git p4\"." % fileName
)
2210 print("You can delete the file \"%s\" when finished." % fileName
)
2212 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
2213 print("To preserve change ownership by user %s, you must\n" \
2214 "do \"p4 change -f <change>\" after submitting and\n" \
2215 "edit the User field.")
2217 print("After submitting, renamed files must be re-synced.")
2218 print("Invoke \"p4 sync -f\" on each of these files:")
2219 for f
in pureRenameCopy
:
2223 print("To revert the changes, use \"p4 revert ...\", and delete")
2224 print("the submit template file \"%s\"" % fileName
)
2226 print("Since the commit adds new files, they must be deleted:")
2227 for f
in filesToAdd
:
2233 if self
.edit_template(fileName
):
2234 if not self
.no_verify
:
2235 if not run_git_hook("p4-changelist", [fileName
]):
2236 print("The p4-changelist hook failed.")
2240 # read the edited message and submit
2241 tmpFile
= open(fileName
, "rb")
2242 message
= decode_text_stream(tmpFile
.read())
2245 message
= message
.replace("\r\n", "\n")
2246 if message
.find(separatorLine
) != -1:
2247 submitTemplate
= message
[:message
.index(separatorLine
)]
2249 submitTemplate
= message
2251 if len(submitTemplate
.strip()) == 0:
2252 print("Changelist is empty, aborting this changelist.")
2257 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate
)
2259 p4_write_pipe(['shelve', '-i'], submitTemplate
)
2261 p4_write_pipe(['submit', '-i'], submitTemplate
)
2262 # The rename/copy happened by applying a patch that created a
2263 # new file. This leaves it writable, which confuses p4.
2264 for f
in pureRenameCopy
:
2267 if self
.preserveUser
:
2269 # Get last changelist number. Cannot easily get it from
2270 # the submit command output as the output is
2272 changelist
= self
.lastP4Changelist()
2273 self
.modifyChangelistUser(changelist
, p4User
)
2277 run_git_hook("p4-post-changelist")
2279 # Revert changes if we skip this patch
2280 if not submitted
or self
.shelve
:
2282 print ("Reverting shelved files.")
2284 print ("Submission cancelled, undoing p4 changes.")
2286 for f
in editedFiles | filesToDelete
:
2288 for f
in filesToAdd
:
2292 if not self
.prepare_p4_only
:
2296 # Export git tags as p4 labels. Create a p4 label and then tag
2298 def exportGitTags(self
, gitTags
):
2299 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
2300 if len(validLabelRegexp
) == 0:
2301 validLabelRegexp
= defaultLabelRegexp
2302 m
= re
.compile(validLabelRegexp
)
2304 for name
in gitTags
:
2306 if not m
.match(name
):
2308 print("tag %s does not match regexp %s" % (name
, validLabelRegexp
))
2311 # Get the p4 commit this corresponds to
2312 logMessage
= extractLogMessageFromGitCommit(name
)
2313 values
= extractSettingsGitLog(logMessage
)
2315 if 'change' not in values
:
2316 # a tag pointing to something not sent to p4; ignore
2318 print("git tag %s does not give a p4 commit" % name
)
2321 changelist
= values
['change']
2323 # Get the tag details.
2327 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
2330 if re
.match(r
'tag\s+', l
):
2332 elif re
.match(r
'\s*$', l
):
2339 body
= ["lightweight tag imported by git p4\n"]
2341 # Create the label - use the same view as the client spec we are using
2342 clientSpec
= getClientSpec()
2344 labelTemplate
= "Label: %s\n" % name
2345 labelTemplate
+= "Description:\n"
2347 labelTemplate
+= "\t" + b
+ "\n"
2348 labelTemplate
+= "View:\n"
2349 for depot_side
in clientSpec
.mappings
:
2350 labelTemplate
+= "\t%s\n" % depot_side
2353 print("Would create p4 label %s for tag" % name
)
2354 elif self
.prepare_p4_only
:
2355 print("Not creating p4 label %s for tag due to option" \
2356 " --prepare-p4-only" % name
)
2358 p4_write_pipe(["label", "-i"], labelTemplate
)
2361 p4_system(["tag", "-l", name
] +
2362 ["%s@%s" % (depot_side
, changelist
) for depot_side
in clientSpec
.mappings
])
2365 print("created p4 label for tag %s" % name
)
2367 def run(self
, args
):
2369 self
.master
= currentGitBranch()
2370 elif len(args
) == 1:
2371 self
.master
= args
[0]
2372 if not branchExists(self
.master
):
2373 die("Branch %s does not exist" % self
.master
)
2377 for i
in self
.update_shelve
:
2379 sys
.exit("invalid changelist %d" % i
)
2382 allowSubmit
= gitConfig("git-p4.allowSubmit")
2383 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
2384 die("%s is not in git-p4.allowSubmit" % self
.master
)
2386 [upstream
, settings
] = findUpstreamBranchPoint()
2387 self
.depotPath
= settings
['depot-paths'][0]
2388 if len(self
.origin
) == 0:
2389 self
.origin
= upstream
2391 if len(self
.update_shelve
) > 0:
2394 if self
.preserveUser
:
2395 if not self
.canChangeChangelists():
2396 die("Cannot preserve user names without p4 super-user or admin permissions")
2398 # if not set from the command line, try the config file
2399 if self
.conflict_behavior
is None:
2400 val
= gitConfig("git-p4.conflict")
2402 if val
not in self
.conflict_behavior_choices
:
2403 die("Invalid value '%s' for config git-p4.conflict" % val
)
2406 self
.conflict_behavior
= val
2409 print("Origin branch is " + self
.origin
)
2411 if len(self
.depotPath
) == 0:
2412 print("Internal error: cannot locate perforce depot path from existing branches")
2415 self
.useClientSpec
= False
2416 if gitConfigBool("git-p4.useclientspec"):
2417 self
.useClientSpec
= True
2418 if self
.useClientSpec
:
2419 self
.clientSpecDirs
= getClientSpec()
2421 # Check for the existence of P4 branches
2422 branchesDetected
= (len(p4BranchesInGit().keys()) > 1)
2424 if self
.useClientSpec
and not branchesDetected
:
2425 # all files are relative to the client spec
2426 self
.clientPath
= getClientRoot()
2428 self
.clientPath
= p4Where(self
.depotPath
)
2430 if self
.clientPath
== "":
2431 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
2433 print("Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
))
2434 self
.oldWorkingDirectory
= os
.getcwd()
2436 # ensure the clientPath exists
2437 new_client_dir
= False
2438 if not os
.path
.exists(self
.clientPath
):
2439 new_client_dir
= True
2440 os
.makedirs(self
.clientPath
)
2442 chdir(self
.clientPath
, is_client_path
=True)
2444 print("Would synchronize p4 checkout in %s" % self
.clientPath
)
2446 print("Synchronizing p4 checkout...")
2448 # old one was destroyed, and maybe nobody told p4
2449 p4_sync("...", "-f")
2456 committish
= self
.master
2460 if self
.commit
!= "":
2461 if self
.commit
.find("..") != -1:
2462 limits_ish
= self
.commit
.split("..")
2463 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish
[0], limits_ish
[1])]):
2464 commits
.append(line
.strip())
2467 commits
.append(self
.commit
)
2469 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self
.origin
, committish
)]):
2470 commits
.append(line
.strip())
2473 if self
.preserveUser
or gitConfigBool("git-p4.skipUserNameCheck"):
2474 self
.checkAuthorship
= False
2476 self
.checkAuthorship
= True
2478 if self
.preserveUser
:
2479 self
.checkValidP4Users(commits
)
2482 # Build up a set of options to be passed to diff when
2483 # submitting each commit to p4.
2485 if self
.detectRenames
:
2486 # command-line -M arg
2487 self
.diffOpts
= "-M"
2489 # If not explicitly set check the config variable
2490 detectRenames
= gitConfig("git-p4.detectRenames")
2492 if detectRenames
.lower() == "false" or detectRenames
== "":
2494 elif detectRenames
.lower() == "true":
2495 self
.diffOpts
= "-M"
2497 self
.diffOpts
= "-M%s" % detectRenames
2499 # no command-line arg for -C or --find-copies-harder, just
2501 detectCopies
= gitConfig("git-p4.detectCopies")
2502 if detectCopies
.lower() == "false" or detectCopies
== "":
2504 elif detectCopies
.lower() == "true":
2505 self
.diffOpts
+= " -C"
2507 self
.diffOpts
+= " -C%s" % detectCopies
2509 if gitConfigBool("git-p4.detectCopiesHarder"):
2510 self
.diffOpts
+= " --find-copies-harder"
2512 num_shelves
= len(self
.update_shelve
)
2513 if num_shelves
> 0 and num_shelves
!= len(commits
):
2514 sys
.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2515 (len(commits
), num_shelves
))
2517 if not self
.no_verify
:
2519 if not run_git_hook("p4-pre-submit"):
2520 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nYou can skip " \
2521 "this pre-submission check by adding\nthe command line option '--no-verify', " \
2522 "however,\nthis will also skip the p4-changelist hook as well.")
2524 except Exception as e
:
2525 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nThe hook failed "\
2526 "with the error '{0}'".format(e
.message
) )
2530 # Apply the commits, one at a time. On failure, ask if should
2531 # continue to try the rest of the patches, or quit.
2534 print("Would apply")
2536 last
= len(commits
) - 1
2537 for i
, commit
in enumerate(commits
):
2539 print(" ", read_pipe(["git", "show", "-s",
2540 "--format=format:%h %s", commit
]))
2543 ok
= self
.applyCommit(commit
)
2545 applied
.append(commit
)
2546 if self
.prepare_p4_only
:
2548 print("Processing only the first commit due to option" \
2549 " --prepare-p4-only")
2553 # prompt for what to do, or use the option/variable
2554 if self
.conflict_behavior
== "ask":
2555 print("What do you want to do?")
2556 response
= prompt("[s]kip this commit but apply the rest, or [q]uit? ")
2557 elif self
.conflict_behavior
== "skip":
2559 elif self
.conflict_behavior
== "quit":
2562 die("Unknown conflict_behavior '%s'" %
2563 self
.conflict_behavior
)
2566 print("Skipping this commit, but applying the rest")
2571 chdir(self
.oldWorkingDirectory
)
2572 shelved_applied
= "shelved" if self
.shelve
else "applied"
2575 elif self
.prepare_p4_only
:
2577 elif len(commits
) == len(applied
):
2578 print("All commits {0}!".format(shelved_applied
))
2582 sync
.branch
= self
.branch
2583 if self
.disable_p4sync
:
2584 sync
.sync_origin_only()
2588 if not self
.disable_rebase
:
2593 if len(applied
) == 0:
2594 print("No commits {0}.".format(shelved_applied
))
2596 print("{0} only the commits marked with '*':".format(shelved_applied
.capitalize()))
2602 print(star
, read_pipe(["git", "show", "-s",
2603 "--format=format:%h %s", c
]))
2604 print("You will have to do 'git p4 sync' and rebase.")
2606 if gitConfigBool("git-p4.exportLabels"):
2607 self
.exportLabels
= True
2609 if self
.exportLabels
:
2610 p4Labels
= getP4Labels(self
.depotPath
)
2611 gitTags
= getGitTags()
2613 missingGitTags
= gitTags
- p4Labels
2614 self
.exportGitTags(missingGitTags
)
2616 # exit with error unless everything applied perfectly
2617 if len(commits
) != len(applied
):
2623 """Represent a p4 view ("p4 help views"), and map files in a
2624 repo according to the view."""
2626 def __init__(self
, client_name
):
2628 self
.client_prefix
= "//%s/" % client_name
2629 # cache results of "p4 where" to lookup client file locations
2630 self
.client_spec_path_cache
= {}
2632 def append(self
, view_line
):
2633 """Parse a view line, splitting it into depot and client
2634 sides. Append to self.mappings, preserving order. This
2635 is only needed for tag creation."""
2637 # Split the view line into exactly two words. P4 enforces
2638 # structure on these lines that simplifies this quite a bit.
2640 # Either or both words may be double-quoted.
2641 # Single quotes do not matter.
2642 # Double-quote marks cannot occur inside the words.
2643 # A + or - prefix is also inside the quotes.
2644 # There are no quotes unless they contain a space.
2645 # The line is already white-space stripped.
2646 # The two words are separated by a single space.
2648 if view_line
[0] == '"':
2649 # First word is double quoted. Find its end.
2650 close_quote_index
= view_line
.find('"', 1)
2651 if close_quote_index
<= 0:
2652 die("No first-word closing quote found: %s" % view_line
)
2653 depot_side
= view_line
[1:close_quote_index
]
2654 # skip closing quote and space
2655 rhs_index
= close_quote_index
+ 1 + 1
2657 space_index
= view_line
.find(" ")
2658 if space_index
<= 0:
2659 die("No word-splitting space found: %s" % view_line
)
2660 depot_side
= view_line
[0:space_index
]
2661 rhs_index
= space_index
+ 1
2663 # prefix + means overlay on previous mapping
2664 if depot_side
.startswith("+"):
2665 depot_side
= depot_side
[1:]
2667 # prefix - means exclude this path, leave out of mappings
2669 if depot_side
.startswith("-"):
2671 depot_side
= depot_side
[1:]
2674 self
.mappings
.append(depot_side
)
2676 def convert_client_path(self
, clientFile
):
2677 # chop off //client/ part to make it relative
2678 if not decode_path(clientFile
).startswith(self
.client_prefix
):
2679 die("No prefix '%s' on clientFile '%s'" %
2680 (self
.client_prefix
, clientFile
))
2681 return clientFile
[len(self
.client_prefix
):]
2683 def update_client_spec_path_cache(self
, files
):
2684 """ Caching file paths by "p4 where" batch query """
2686 # List depot file paths exclude that already cached
2687 fileArgs
= [f
['path'] for f
in files
if decode_path(f
['path']) not in self
.client_spec_path_cache
]
2689 if len(fileArgs
) == 0:
2690 return # All files in cache
2692 where_result
= p4CmdList(["-x", "-", "where"], stdin
=fileArgs
)
2693 for res
in where_result
:
2694 if "code" in res
and res
["code"] == "error":
2695 # assume error is "... file(s) not in client view"
2697 if "clientFile" not in res
:
2698 die("No clientFile in 'p4 where' output")
2700 # it will list all of them, but only one not unmap-ped
2702 depot_path
= decode_path(res
['depotFile'])
2703 if gitConfigBool("core.ignorecase"):
2704 depot_path
= depot_path
.lower()
2705 self
.client_spec_path_cache
[depot_path
] = self
.convert_client_path(res
["clientFile"])
2707 # not found files or unmap files set to ""
2708 for depotFile
in fileArgs
:
2709 depotFile
= decode_path(depotFile
)
2710 if gitConfigBool("core.ignorecase"):
2711 depotFile
= depotFile
.lower()
2712 if depotFile
not in self
.client_spec_path_cache
:
2713 self
.client_spec_path_cache
[depotFile
] = b
''
2715 def map_in_client(self
, depot_path
):
2716 """Return the relative location in the client where this
2717 depot file should live. Returns "" if the file should
2718 not be mapped in the client."""
2720 if gitConfigBool("core.ignorecase"):
2721 depot_path
= depot_path
.lower()
2723 if depot_path
in self
.client_spec_path_cache
:
2724 return self
.client_spec_path_cache
[depot_path
]
2726 die( "Error: %s is not found in client spec path" % depot_path
)
2729 def cloneExcludeCallback(option
, opt_str
, value
, parser
):
2730 # prepend "/" because the first "/" was consumed as part of the option itself.
2731 # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2732 parser
.values
.cloneExclude
+= ["/" + re
.sub(r
"\.\.\.$", "", value
)]
2734 class P4Sync(Command
, P4UserMap
):
2737 Command
.__init
__(self
)
2738 P4UserMap
.__init
__(self
)
2740 optparse
.make_option("--branch", dest
="branch"),
2741 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
2742 optparse
.make_option("--changesfile", dest
="changesFile"),
2743 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
2744 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
2745 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2746 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
2747 help="Import into refs/heads/ , not refs/remotes"),
2748 optparse
.make_option("--max-changes", dest
="maxChanges",
2749 help="Maximum number of changes to import"),
2750 optparse
.make_option("--changes-block-size", dest
="changes_block_size", type="int",
2751 help="Internal block size to use when iteratively calling p4 changes"),
2752 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
2753 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2754 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
2755 help="Only sync files that are included in the Perforce Client Spec"),
2756 optparse
.make_option("-/", dest
="cloneExclude",
2757 action
="callback", callback
=cloneExcludeCallback
, type="string",
2758 help="exclude depot path"),
2760 self
.description
= """Imports from Perforce into a git repository.\n
2762 //depot/my/project/ -- to import the current head
2763 //depot/my/project/@all -- to import everything
2764 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2766 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2768 self
.usage
+= " //depot/path[@revRange]"
2770 self
.createdBranches
= set()
2771 self
.committedChanges
= set()
2773 self
.detectBranches
= False
2774 self
.detectLabels
= False
2775 self
.importLabels
= False
2776 self
.changesFile
= ""
2777 self
.syncWithOrigin
= True
2778 self
.importIntoRemotes
= True
2779 self
.maxChanges
= ""
2780 self
.changes_block_size
= None
2781 self
.keepRepoPath
= False
2782 self
.depotPaths
= None
2783 self
.p4BranchesInGit
= []
2784 self
.cloneExclude
= []
2785 self
.useClientSpec
= False
2786 self
.useClientSpec_from_options
= False
2787 self
.clientSpecDirs
= None
2788 self
.tempBranches
= []
2789 self
.tempBranchLocation
= "refs/git-p4-tmp"
2790 self
.largeFileSystem
= None
2791 self
.suppress_meta_comment
= False
2793 if gitConfig('git-p4.largeFileSystem'):
2794 largeFileSystemConstructor
= globals()[gitConfig('git-p4.largeFileSystem')]
2795 self
.largeFileSystem
= largeFileSystemConstructor(
2796 lambda git_mode
, relPath
, contents
: self
.writeToGitStream(git_mode
, relPath
, contents
)
2799 if gitConfig("git-p4.syncFromOrigin") == "false":
2800 self
.syncWithOrigin
= False
2802 self
.depotPaths
= []
2803 self
.changeRange
= ""
2804 self
.previousDepotPaths
= []
2805 self
.hasOrigin
= False
2807 # map from branch depot path to parent branch
2808 self
.knownBranches
= {}
2809 self
.initialParents
= {}
2811 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
2814 # Force a checkpoint in fast-import and wait for it to finish
2815 def checkpoint(self
):
2816 self
.gitStream
.write("checkpoint\n\n")
2817 self
.gitStream
.write("progress checkpoint\n\n")
2818 self
.gitStream
.flush()
2819 out
= self
.gitOutput
.readline()
2821 print("checkpoint finished: " + out
)
2823 def isPathWanted(self
, path
):
2824 for p
in self
.cloneExclude
:
2826 if p4PathStartsWith(path
, p
):
2828 # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2829 elif path
.lower() == p
.lower():
2831 for p
in self
.depotPaths
:
2832 if p4PathStartsWith(path
, decode_path(p
)):
2836 def extractFilesFromCommit(self
, commit
, shelved
=False, shelved_cl
= 0):
2839 while "depotFile%s" % fnum
in commit
:
2840 path
= commit
["depotFile%s" % fnum
]
2841 found
= self
.isPathWanted(decode_path(path
))
2848 file["rev"] = commit
["rev%s" % fnum
]
2849 file["action"] = commit
["action%s" % fnum
]
2850 file["type"] = commit
["type%s" % fnum
]
2852 file["shelved_cl"] = int(shelved_cl
)
2857 def extractJobsFromCommit(self
, commit
):
2860 while "job%s" % jnum
in commit
:
2861 job
= commit
["job%s" % jnum
]
2866 def stripRepoPath(self
, path
, prefixes
):
2867 """When streaming files, this is called to map a p4 depot path
2868 to where it should go in git. The prefixes are either
2869 self.depotPaths, or self.branchPrefixes in the case of
2870 branch detection."""
2872 if self
.useClientSpec
:
2873 # branch detection moves files up a level (the branch name)
2874 # from what client spec interpretation gives
2875 path
= decode_path(self
.clientSpecDirs
.map_in_client(path
))
2876 if self
.detectBranches
:
2877 for b
in self
.knownBranches
:
2878 if p4PathStartsWith(path
, b
+ "/"):
2879 path
= path
[len(b
)+1:]
2881 elif self
.keepRepoPath
:
2882 # Preserve everything in relative path name except leading
2883 # //depot/; just look at first prefix as they all should
2884 # be in the same depot.
2885 depot
= re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])
2886 if p4PathStartsWith(path
, depot
):
2887 path
= path
[len(depot
):]
2891 if p4PathStartsWith(path
, p
):
2892 path
= path
[len(p
):]
2895 path
= wildcard_decode(path
)
2898 def splitFilesIntoBranches(self
, commit
):
2899 """Look at each depotFile in the commit to figure out to what
2900 branch it belongs."""
2902 if self
.clientSpecDirs
:
2903 files
= self
.extractFilesFromCommit(commit
)
2904 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2908 while "depotFile%s" % fnum
in commit
:
2909 raw_path
= commit
["depotFile%s" % fnum
]
2910 path
= decode_path(raw_path
)
2911 found
= self
.isPathWanted(path
)
2917 file["path"] = raw_path
2918 file["rev"] = commit
["rev%s" % fnum
]
2919 file["action"] = commit
["action%s" % fnum
]
2920 file["type"] = commit
["type%s" % fnum
]
2923 # start with the full relative path where this file would
2925 if self
.useClientSpec
:
2926 relPath
= decode_path(self
.clientSpecDirs
.map_in_client(path
))
2928 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
2930 for branch
in self
.knownBranches
.keys():
2931 # add a trailing slash so that a commit into qt/4.2foo
2932 # doesn't end up in qt/4.2, e.g.
2933 if p4PathStartsWith(relPath
, branch
+ "/"):
2934 if branch
not in branches
:
2935 branches
[branch
] = []
2936 branches
[branch
].append(file)
2941 def writeToGitStream(self
, gitMode
, relPath
, contents
):
2942 self
.gitStream
.write(encode_text_stream(u
'M {} inline {}\n'.format(gitMode
, relPath
)))
2943 self
.gitStream
.write('data %d\n' % sum(len(d
) for d
in contents
))
2945 self
.gitStream
.write(d
)
2946 self
.gitStream
.write('\n')
2948 def encodeWithUTF8(self
, path
):
2950 path
.decode('ascii')
2953 if gitConfig('git-p4.pathEncoding'):
2954 encoding
= gitConfig('git-p4.pathEncoding')
2955 path
= path
.decode(encoding
, 'replace').encode('utf8', 'replace')
2957 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding
, path
))
2960 # output one file from the P4 stream
2961 # - helper for streamP4Files
2963 def streamOneP4File(self
, file, contents
):
2964 file_path
= file['depotFile']
2965 relPath
= self
.stripRepoPath(decode_path(file_path
), self
.branchPrefixes
)
2968 if 'fileSize' in self
.stream_file
:
2969 size
= int(self
.stream_file
['fileSize'])
2971 size
= 0 # deleted files don't get a fileSize apparently
2972 sys
.stdout
.write('\r%s --> %s (%s)\n' % (
2973 file_path
, relPath
, format_size_human_readable(size
)))
2976 (type_base
, type_mods
) = split_p4_type(file["type"])
2979 if "x" in type_mods
:
2981 if type_base
== "symlink":
2983 # p4 print on a symlink sometimes contains "target\n";
2984 # if it does, remove the newline
2985 data
= ''.join(decode_text_stream(c
) for c
in contents
)
2987 # Some version of p4 allowed creating a symlink that pointed
2988 # to nothing. This causes p4 errors when checking out such
2989 # a change, and errors here too. Work around it by ignoring
2990 # the bad symlink; hopefully a future change fixes it.
2991 print("\nIgnoring empty symlink in %s" % file_path
)
2993 elif data
[-1] == '\n':
2994 contents
= [data
[:-1]]
2998 if type_base
== "utf16":
2999 # p4 delivers different text in the python output to -G
3000 # than it does when using "print -o", or normal p4 client
3001 # operations. utf16 is converted to ascii or utf8, perhaps.
3002 # But ascii text saved as -t utf16 is completely mangled.
3003 # Invoke print -o to get the real contents.
3005 # On windows, the newlines will always be mangled by print, so put
3006 # them back too. This is not needed to the cygwin windows version,
3007 # just the native "NT" type.
3010 text
= p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (decode_path(file['depotFile']), file['change'])], raw
=True)
3011 except Exception as e
:
3012 if 'Translation of file content failed' in str(e
):
3013 type_base
= 'binary'
3017 if p4_version_string().find('/NT') >= 0:
3018 text
= text
.replace(b
'\r\n', b
'\n')
3021 if type_base
== "apple":
3022 # Apple filetype files will be streamed as a concatenation of
3023 # its appledouble header and the contents. This is useless
3024 # on both macs and non-macs. If using "print -q -o xx", it
3025 # will create "xx" with the data, and "%xx" with the header.
3026 # This is also not very useful.
3028 # Ideally, someday, this script can learn how to generate
3029 # appledouble files directly and import those to git, but
3030 # non-mac machines can never find a use for apple filetype.
3031 print("\nIgnoring apple filetype file %s" % file['depotFile'])
3034 # Note that we do not try to de-mangle keywords on utf16 files,
3035 # even though in theory somebody may want that.
3036 regexp
= p4_keywords_regexp_for_type(type_base
, type_mods
)
3038 contents
= [regexp
.sub(br
'$\1$', c
) for c
in contents
]
3040 if self
.largeFileSystem
:
3041 (git_mode
, contents
) = self
.largeFileSystem
.processContent(git_mode
, relPath
, contents
)
3043 self
.writeToGitStream(git_mode
, relPath
, contents
)
3045 def streamOneP4Deletion(self
, file):
3046 relPath
= self
.stripRepoPath(decode_path(file['path']), self
.branchPrefixes
)
3048 sys
.stdout
.write("delete %s\n" % relPath
)
3050 self
.gitStream
.write(encode_text_stream(u
'D {}\n'.format(relPath
)))
3052 if self
.largeFileSystem
and self
.largeFileSystem
.isLargeFile(relPath
):
3053 self
.largeFileSystem
.removeLargeFile(relPath
)
3055 # handle another chunk of streaming data
3056 def streamP4FilesCb(self
, marshalled
):
3058 # catch p4 errors and complain
3060 if "code" in marshalled
:
3061 if marshalled
["code"] == "error":
3062 if "data" in marshalled
:
3063 err
= marshalled
["data"].rstrip()
3065 if not err
and 'fileSize' in self
.stream_file
:
3066 required_bytes
= int((4 * int(self
.stream_file
["fileSize"])) - calcDiskFree())
3067 if required_bytes
> 0:
3068 err
= 'Not enough space left on %s! Free at least %s.' % (
3069 os
.getcwd(), format_size_human_readable(required_bytes
))
3073 if self
.stream_have_file_info
:
3074 if "depotFile" in self
.stream_file
:
3075 f
= self
.stream_file
["depotFile"]
3076 # force a failure in fast-import, else an empty
3077 # commit will be made
3078 self
.gitStream
.write("\n")
3079 self
.gitStream
.write("die-now\n")
3080 self
.gitStream
.close()
3081 # ignore errors, but make sure it exits first
3082 self
.importProcess
.wait()
3084 die("Error from p4 print for %s: %s" % (f
, err
))
3086 die("Error from p4 print: %s" % err
)
3088 if 'depotFile' in marshalled
and self
.stream_have_file_info
:
3089 # start of a new file - output the old one first
3090 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
3091 self
.stream_file
= {}
3092 self
.stream_contents
= []
3093 self
.stream_have_file_info
= False
3095 # pick up the new file information... for the
3096 # 'data' field we need to append to our array
3097 for k
in marshalled
.keys():
3099 if 'streamContentSize' not in self
.stream_file
:
3100 self
.stream_file
['streamContentSize'] = 0
3101 self
.stream_file
['streamContentSize'] += len(marshalled
['data'])
3102 self
.stream_contents
.append(marshalled
['data'])
3104 self
.stream_file
[k
] = marshalled
[k
]
3107 'streamContentSize' in self
.stream_file
and
3108 'fileSize' in self
.stream_file
and
3109 'depotFile' in self
.stream_file
):
3110 size
= int(self
.stream_file
["fileSize"])
3112 progress
= 100*self
.stream_file
['streamContentSize']/size
3113 sys
.stdout
.write('\r%s %d%% (%s)' % (
3114 self
.stream_file
['depotFile'], progress
,
3115 format_size_human_readable(size
)))
3118 self
.stream_have_file_info
= True
3120 # Stream directly from "p4 files" into "git fast-import"
3121 def streamP4Files(self
, files
):
3127 filesForCommit
.append(f
)
3128 if f
['action'] in self
.delete_actions
:
3129 filesToDelete
.append(f
)
3131 filesToRead
.append(f
)
3134 for f
in filesToDelete
:
3135 self
.streamOneP4Deletion(f
)
3137 if len(filesToRead
) > 0:
3138 self
.stream_file
= {}
3139 self
.stream_contents
= []
3140 self
.stream_have_file_info
= False
3142 # curry self argument
3143 def streamP4FilesCbSelf(entry
):
3144 self
.streamP4FilesCb(entry
)
3147 for f
in filesToRead
:
3148 if 'shelved_cl' in f
:
3149 # Handle shelved CLs using the "p4 print file@=N" syntax to print
3151 fileArg
= f
['path'] + encode_text_stream('@={}'.format(f
['shelved_cl']))
3153 fileArg
= f
['path'] + encode_text_stream('#{}'.format(f
['rev']))
3155 fileArgs
.append(fileArg
)
3157 p4CmdList(["-x", "-", "print"],
3159 cb
=streamP4FilesCbSelf
)
3162 if 'depotFile' in self
.stream_file
:
3163 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
3165 def make_email(self
, userid
):
3166 if userid
in self
.users
:
3167 return self
.users
[userid
]
3169 return "%s <a@b>" % userid
3171 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
3172 """ Stream a p4 tag.
3173 commit is either a git commit, or a fast-import mark, ":<p4commit>"
3177 print("writing tag %s for commit %s" % (labelName
, commit
))
3178 gitStream
.write("tag %s\n" % labelName
)
3179 gitStream
.write("from %s\n" % commit
)
3181 if 'Owner' in labelDetails
:
3182 owner
= labelDetails
["Owner"]
3186 # Try to use the owner of the p4 label, or failing that,
3187 # the current p4 user id.
3189 email
= self
.make_email(owner
)
3191 email
= self
.make_email(self
.p4UserId())
3192 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
3194 gitStream
.write("tagger %s\n" % tagger
)
3196 print("labelDetails=",labelDetails
)
3197 if 'Description' in labelDetails
:
3198 description
= labelDetails
['Description']
3200 description
= 'Label from git p4'
3202 gitStream
.write("data %d\n" % len(description
))
3203 gitStream
.write(description
)
3204 gitStream
.write("\n")
3206 def inClientSpec(self
, path
):
3207 if not self
.clientSpecDirs
:
3209 inClientSpec
= self
.clientSpecDirs
.map_in_client(path
)
3210 if not inClientSpec
and self
.verbose
:
3211 print('Ignoring file outside of client spec: {0}'.format(path
))
3214 def hasBranchPrefix(self
, path
):
3215 if not self
.branchPrefixes
:
3217 hasPrefix
= [p
for p
in self
.branchPrefixes
3218 if p4PathStartsWith(path
, p
)]
3219 if not hasPrefix
and self
.verbose
:
3220 print('Ignoring file outside of prefix: {0}'.format(path
))
3223 def findShadowedFiles(self
, files
, change
):
3224 # Perforce allows you commit files and directories with the same name,
3225 # so you could have files //depot/foo and //depot/foo/bar both checked
3226 # in. A p4 sync of a repository in this state fails. Deleting one of
3227 # the files recovers the repository.
3229 # Git will not allow the broken state to exist and only the most recent
3230 # of the conflicting names is left in the repository. When one of the
3231 # conflicting files is deleted we need to re-add the other one to make
3232 # sure the git repository recovers in the same way as perforce.
3233 deleted
= [f
for f
in files
if f
['action'] in self
.delete_actions
]
3236 path
= decode_path(f
['path'])
3237 to_check
.add(path
+ '/...')
3239 path
= path
.rsplit("/", 1)[0]
3240 if path
== "/" or path
in to_check
:
3243 to_check
= ['%s@%s' % (wildcard_encode(p
), change
) for p
in to_check
3244 if self
.hasBranchPrefix(p
)]
3246 stat_result
= p4CmdList(["-x", "-", "fstat", "-T",
3247 "depotFile,headAction,headRev,headType"], stdin
=to_check
)
3248 for record
in stat_result
:
3249 if record
['code'] != 'stat':
3251 if record
['headAction'] in self
.delete_actions
:
3255 'path': record
['depotFile'],
3256 'rev': record
['headRev'],
3257 'type': record
['headType']})
3259 def commit(self
, details
, files
, branch
, parent
= "", allow_empty
=False):
3260 epoch
= details
["time"]
3261 author
= details
["user"]
3262 jobs
= self
.extractJobsFromCommit(details
)
3265 print('commit into {0}'.format(branch
))
3267 files
= [f
for f
in files
3268 if self
.hasBranchPrefix(decode_path(f
['path']))]
3269 self
.findShadowedFiles(files
, details
['change'])
3271 if self
.clientSpecDirs
:
3272 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
3274 files
= [f
for f
in files
if self
.inClientSpec(decode_path(f
['path']))]
3276 if gitConfigBool('git-p4.keepEmptyCommits'):
3279 if not files
and not allow_empty
:
3280 print('Ignoring revision {0} as it would produce an empty commit.'
3281 .format(details
['change']))
3284 self
.gitStream
.write("commit %s\n" % branch
)
3285 self
.gitStream
.write("mark :%s\n" % details
["change"])
3286 self
.committedChanges
.add(int(details
["change"]))
3288 if author
not in self
.users
:
3289 self
.getUserMapFromPerforceServer()
3290 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
3292 self
.gitStream
.write("committer %s\n" % committer
)
3294 self
.gitStream
.write("data <<EOT\n")
3295 self
.gitStream
.write(details
["desc"])
3297 self
.gitStream
.write("\nJobs: %s" % (' '.join(jobs
)))
3299 if not self
.suppress_meta_comment
:
3300 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3301 (','.join(self
.branchPrefixes
), details
["change"]))
3302 if len(details
['options']) > 0:
3303 self
.gitStream
.write(": options = %s" % details
['options'])
3304 self
.gitStream
.write("]\n")
3306 self
.gitStream
.write("EOT\n\n")
3310 print("parent %s" % parent
)
3311 self
.gitStream
.write("from %s\n" % parent
)
3313 self
.streamP4Files(files
)
3314 self
.gitStream
.write("\n")
3316 change
= int(details
["change"])
3318 if change
in self
.labels
:
3319 label
= self
.labels
[change
]
3320 labelDetails
= label
[0]
3321 labelRevisions
= label
[1]
3323 print("Change %s is labelled %s" % (change
, labelDetails
))
3325 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
3326 for p
in self
.branchPrefixes
])
3328 if len(files
) == len(labelRevisions
):
3332 if info
["action"] in self
.delete_actions
:
3334 cleanedFiles
[info
["depotFile"]] = info
["rev"]
3336 if cleanedFiles
== labelRevisions
:
3337 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
3341 print("Tag %s does not match with change %s: files do not match."
3342 % (labelDetails
["label"], change
))
3346 print("Tag %s does not match with change %s: file count is different."
3347 % (labelDetails
["label"], change
))
3349 # Build a dictionary of changelists and labels, for "detect-labels" option.
3350 def getLabels(self
):
3353 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
3354 if len(l
) > 0 and not self
.silent
:
3355 print("Finding files belonging to labels in %s" % self
.depotPaths
)
3358 label
= output
["label"]
3362 print("Querying files for label %s" % label
)
3363 for file in p4CmdList(["files"] +
3364 ["%s...@%s" % (p
, label
)
3365 for p
in self
.depotPaths
]):
3366 revisions
[file["depotFile"]] = file["rev"]
3367 change
= int(file["change"])
3368 if change
> newestChange
:
3369 newestChange
= change
3371 self
.labels
[newestChange
] = [output
, revisions
]
3374 print("Label changes: %s" % self
.labels
.keys())
3376 # Import p4 labels as git tags. A direct mapping does not
3377 # exist, so assume that if all the files are at the same revision
3378 # then we can use that, or it's something more complicated we should
3380 def importP4Labels(self
, stream
, p4Labels
):
3382 print("import p4 labels: " + ' '.join(p4Labels
))
3384 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
3385 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
3386 if len(validLabelRegexp
) == 0:
3387 validLabelRegexp
= defaultLabelRegexp
3388 m
= re
.compile(validLabelRegexp
)
3390 for name
in p4Labels
:
3393 if not m
.match(name
):
3395 print("label %s does not match regexp %s" % (name
,validLabelRegexp
))
3398 if name
in ignoredP4Labels
:
3401 labelDetails
= p4CmdList(['label', "-o", name
])[0]
3403 # get the most recent changelist for each file in this label
3404 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
3405 for p
in self
.depotPaths
])
3407 if 'change' in change
:
3408 # find the corresponding git commit; take the oldest commit
3409 changelist
= int(change
['change'])
3410 if changelist
in self
.committedChanges
:
3411 gitCommit
= ":%d" % changelist
# use a fast-import mark
3414 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
3415 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
], ignore_error
=True)
3416 if len(gitCommit
) == 0:
3417 print("importing label %s: could not find git commit for changelist %d" % (name
, changelist
))
3420 gitCommit
= gitCommit
.strip()
3423 # Convert from p4 time format
3425 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
3427 print("Could not convert label time %s" % labelDetails
['Update'])
3430 when
= int(time
.mktime(tmwhen
))
3431 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
3433 print("p4 label %s mapped to git commit %s" % (name
, gitCommit
))
3436 print("Label %s has no changelists - possibly deleted?" % name
)
3439 # We can't import this label; don't try again as it will get very
3440 # expensive repeatedly fetching all the files for labels that will
3441 # never be imported. If the label is moved in the future, the
3442 # ignore will need to be removed manually.
3443 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
3445 def guessProjectName(self
):
3446 for p
in self
.depotPaths
:
3449 p
= p
[p
.strip().rfind("/") + 1:]
3450 if not p
.endswith("/"):
3454 def getBranchMapping(self
):
3455 lostAndFoundBranches
= set()
3457 user
= gitConfig("git-p4.branchUser")
3459 command
= "branches -u %s" % user
3461 command
= "branches"
3463 for info
in p4CmdList(command
):
3464 details
= p4Cmd(["branch", "-o", info
["branch"]])
3466 while "View%s" % viewIdx
in details
:
3467 paths
= details
["View%s" % viewIdx
].split(" ")
3468 viewIdx
= viewIdx
+ 1
3469 # require standard //depot/foo/... //depot/bar/... mapping
3470 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
3473 destination
= paths
[1]
3475 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
3476 source
= source
[len(self
.depotPaths
[0]):-4]
3477 destination
= destination
[len(self
.depotPaths
[0]):-4]
3479 if destination
in self
.knownBranches
:
3481 print("p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
))
3482 print("but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
))
3485 self
.knownBranches
[destination
] = source
3487 lostAndFoundBranches
.discard(destination
)
3489 if source
not in self
.knownBranches
:
3490 lostAndFoundBranches
.add(source
)
3492 # Perforce does not strictly require branches to be defined, so we also
3493 # check git config for a branch list.
3495 # Example of branch definition in git config file:
3497 # branchList=main:branchA
3498 # branchList=main:branchB
3499 # branchList=branchA:branchC
3500 configBranches
= gitConfigList("git-p4.branchList")
3501 for branch
in configBranches
:
3503 (source
, destination
) = branch
.split(":")
3504 self
.knownBranches
[destination
] = source
3506 lostAndFoundBranches
.discard(destination
)
3508 if source
not in self
.knownBranches
:
3509 lostAndFoundBranches
.add(source
)
3512 for branch
in lostAndFoundBranches
:
3513 self
.knownBranches
[branch
] = branch
3515 def getBranchMappingFromGitBranches(self
):
3516 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3517 for branch
in branches
.keys():
3518 if branch
== "master":
3521 branch
= branch
[len(self
.projectName
):]
3522 self
.knownBranches
[branch
] = branch
3524 def updateOptionDict(self
, d
):
3526 if self
.keepRepoPath
:
3527 option_keys
['keepRepoPath'] = 1
3529 d
["options"] = ' '.join(sorted(option_keys
.keys()))
3531 def readOptions(self
, d
):
3532 self
.keepRepoPath
= ('options' in d
3533 and ('keepRepoPath' in d
['options']))
3535 def gitRefForBranch(self
, branch
):
3536 if branch
== "main":
3537 return self
.refPrefix
+ "master"
3539 if len(branch
) <= 0:
3542 return self
.refPrefix
+ self
.projectName
+ branch
3544 def gitCommitByP4Change(self
, ref
, change
):
3546 print("looking in ref " + ref
+ " for change %s using bisect..." % change
)
3549 latestCommit
= parseRevision(ref
)
3553 print("trying: earliest %s latest %s" % (earliestCommit
, latestCommit
))
3554 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
3559 log
= extractLogMessageFromGitCommit(next
)
3560 settings
= extractSettingsGitLog(log
)
3561 currentChange
= int(settings
['change'])
3563 print("current change %s" % currentChange
)
3565 if currentChange
== change
:
3567 print("found %s" % next
)
3570 if currentChange
< change
:
3571 earliestCommit
= "^%s" % next
3573 if next
== latestCommit
:
3574 die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref
, change
))
3575 latestCommit
= "%s^@" % next
3579 def importNewBranch(self
, branch
, maxChange
):
3580 # make fast-import flush all changes to disk and update the refs using the checkpoint
3581 # command so that we can try to find the branch parent in the git history
3582 self
.gitStream
.write("checkpoint\n\n");
3583 self
.gitStream
.flush();
3584 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3585 range = "@1,%s" % maxChange
3586 #print "prefix" + branchPrefix
3587 changes
= p4ChangesForPaths([branchPrefix
], range, self
.changes_block_size
)
3588 if len(changes
) <= 0:
3590 firstChange
= changes
[0]
3591 #print "first change in branch: %s" % firstChange
3592 sourceBranch
= self
.knownBranches
[branch
]
3593 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
3594 sourceRef
= self
.gitRefForBranch(sourceBranch
)
3595 #print "source " + sourceBranch
3597 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
3598 #print "branch parent: %s" % branchParentChange
3599 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
3600 if len(gitParent
) > 0:
3601 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
3602 #print "parent git commit: %s" % gitParent
3604 self
.importChanges(changes
)
3607 def searchParent(self
, parent
, branch
, target
):
3608 targetTree
= read_pipe(["git", "rev-parse",
3609 "{}^{{tree}}".format(target
)]).strip()
3610 for line
in read_pipe_lines(["git", "rev-list", "--format=%H %T",
3611 "--no-merges", parent
]):
3612 if line
.startswith("commit "):
3614 commit
, tree
= line
.strip().split(" ")
3615 if tree
== targetTree
:
3617 print("Found parent of %s in commit %s" % (branch
, commit
))
3621 def importChanges(self
, changes
, origin_revision
=0):
3623 for change
in changes
:
3624 description
= p4_describe(change
)
3625 self
.updateOptionDict(description
)
3628 sys
.stdout
.write("\rImporting revision %s (%d%%)" % (
3629 change
, (cnt
* 100) // len(changes
)))
3634 if self
.detectBranches
:
3635 branches
= self
.splitFilesIntoBranches(description
)
3636 for branch
in branches
.keys():
3638 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3639 self
.branchPrefixes
= [ branchPrefix
]
3643 filesForCommit
= branches
[branch
]
3646 print("branch is %s" % branch
)
3648 self
.updatedBranches
.add(branch
)
3650 if branch
not in self
.createdBranches
:
3651 self
.createdBranches
.add(branch
)
3652 parent
= self
.knownBranches
[branch
]
3653 if parent
== branch
:
3656 fullBranch
= self
.projectName
+ branch
3657 if fullBranch
not in self
.p4BranchesInGit
:
3659 print("\n Importing new branch %s" % fullBranch
);
3660 if self
.importNewBranch(branch
, change
- 1):
3662 self
.p4BranchesInGit
.append(fullBranch
)
3664 print("\n Resuming with change %s" % change
);
3667 print("parent determined through known branches: %s" % parent
)
3669 branch
= self
.gitRefForBranch(branch
)
3670 parent
= self
.gitRefForBranch(parent
)
3673 print("looking for initial parent for %s; current parent is %s" % (branch
, parent
))
3675 if len(parent
) == 0 and branch
in self
.initialParents
:
3676 parent
= self
.initialParents
[branch
]
3677 del self
.initialParents
[branch
]
3681 tempBranch
= "%s/%d" % (self
.tempBranchLocation
, change
)
3683 print("Creating temporary branch: " + tempBranch
)
3684 self
.commit(description
, filesForCommit
, tempBranch
)
3685 self
.tempBranches
.append(tempBranch
)
3687 blob
= self
.searchParent(parent
, branch
, tempBranch
)
3689 self
.commit(description
, filesForCommit
, branch
, blob
)
3692 print("Parent of %s not found. Committing into head of %s" % (branch
, parent
))
3693 self
.commit(description
, filesForCommit
, branch
, parent
)
3695 files
= self
.extractFilesFromCommit(description
)
3696 self
.commit(description
, files
, self
.branch
,
3698 # only needed once, to connect to the previous commit
3699 self
.initialParent
= ""
3701 print(self
.gitError
.read())
3704 def sync_origin_only(self
):
3705 if self
.syncWithOrigin
:
3706 self
.hasOrigin
= originP4BranchesExist()
3709 print('Syncing with origin first, using "git fetch origin"')
3710 system("git fetch origin")
3712 def importHeadRevision(self
, revision
):
3713 print("Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
))
3716 details
["user"] = "git perforce import user"
3717 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
3718 % (' '.join(self
.depotPaths
), revision
))
3719 details
["change"] = revision
3723 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
3725 for info
in p4CmdList(["files"] + fileArgs
):
3727 if 'code' in info
and info
['code'] == 'error':
3728 sys
.stderr
.write("p4 returned an error: %s\n"
3730 if info
['data'].find("must refer to client") >= 0:
3731 sys
.stderr
.write("This particular p4 error is misleading.\n")
3732 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
3733 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
3735 if 'p4ExitCode' in info
:
3736 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
3740 change
= int(info
["change"])
3741 if change
> newestRevision
:
3742 newestRevision
= change
3744 if info
["action"] in self
.delete_actions
:
3745 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3746 #fileCnt = fileCnt + 1
3749 for prop
in ["depotFile", "rev", "action", "type" ]:
3750 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
3752 fileCnt
= fileCnt
+ 1
3754 details
["change"] = newestRevision
3756 # Use time from top-most change so that all git p4 clones of
3757 # the same p4 repo have the same commit SHA1s.
3758 res
= p4_describe(newestRevision
)
3759 details
["time"] = res
["time"]
3761 self
.updateOptionDict(details
)
3763 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
3764 except IOError as err
:
3765 print("IO error with git fast-import. Is your git version recent enough?")
3766 print("IO error details: {}".format(err
))
3767 print(self
.gitError
.read())
3770 def importRevisions(self
, args
, branch_arg_given
):
3773 if len(self
.changesFile
) > 0:
3774 with
open(self
.changesFile
) as f
:
3775 output
= f
.readlines()
3778 changeSet
.add(int(line
))
3780 for change
in changeSet
:
3781 changes
.append(change
)
3785 # catch "git p4 sync" with no new branches, in a repo that
3786 # does not have any existing p4 branches
3788 if not self
.p4BranchesInGit
:
3789 raise P4CommandException("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3791 # The default branch is master, unless --branch is used to
3792 # specify something else. Make sure it exists, or complain
3793 # nicely about how to use --branch.
3794 if not self
.detectBranches
:
3795 if not branch_exists(self
.branch
):
3796 if branch_arg_given
:
3797 raise P4CommandException("Error: branch %s does not exist." % self
.branch
)
3799 raise P4CommandException("Error: no branch %s; perhaps specify one with --branch." %
3803 print("Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
3805 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
, self
.changes_block_size
)
3807 if len(self
.maxChanges
) > 0:
3808 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
3810 if len(changes
) == 0:
3812 print("No changes to import!")
3814 if not self
.silent
and not self
.detectBranches
:
3815 print("Import destination: %s" % self
.branch
)
3817 self
.updatedBranches
= set()
3819 if not self
.detectBranches
:
3821 # start a new branch
3822 self
.initialParent
= ""
3824 # build on a previous revision
3825 self
.initialParent
= parseRevision(self
.branch
)
3827 self
.importChanges(changes
)
3831 if len(self
.updatedBranches
) > 0:
3832 sys
.stdout
.write("Updated branches: ")
3833 for b
in self
.updatedBranches
:
3834 sys
.stdout
.write("%s " % b
)
3835 sys
.stdout
.write("\n")
3837 def openStreams(self
):
3838 self
.importProcess
= subprocess
.Popen(["git", "fast-import"],
3839 stdin
=subprocess
.PIPE
,
3840 stdout
=subprocess
.PIPE
,
3841 stderr
=subprocess
.PIPE
);
3842 self
.gitOutput
= self
.importProcess
.stdout
3843 self
.gitStream
= self
.importProcess
.stdin
3844 self
.gitError
= self
.importProcess
.stderr
3846 if bytes
is not str:
3847 # Wrap gitStream.write() so that it can be called using `str` arguments
3848 def make_encoded_write(write
):
3849 def encoded_write(s
):
3850 return write(s
.encode() if isinstance(s
, str) else s
)
3851 return encoded_write
3853 self
.gitStream
.write
= make_encoded_write(self
.gitStream
.write
)
3855 def closeStreams(self
):
3856 if self
.gitStream
is None:
3858 self
.gitStream
.close()
3859 if self
.importProcess
.wait() != 0:
3860 die("fast-import failed: %s" % self
.gitError
.read())
3861 self
.gitOutput
.close()
3862 self
.gitError
.close()
3863 self
.gitStream
= None
3865 def run(self
, args
):
3866 if self
.importIntoRemotes
:
3867 self
.refPrefix
= "refs/remotes/p4/"
3869 self
.refPrefix
= "refs/heads/p4/"
3871 self
.sync_origin_only()
3873 branch_arg_given
= bool(self
.branch
)
3874 if len(self
.branch
) == 0:
3875 self
.branch
= self
.refPrefix
+ "master"
3876 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
3877 system("git update-ref %s refs/heads/p4" % self
.branch
)
3878 system("git branch -D p4")
3880 # accept either the command-line option, or the configuration variable
3881 if self
.useClientSpec
:
3882 # will use this after clone to set the variable
3883 self
.useClientSpec_from_options
= True
3885 if gitConfigBool("git-p4.useclientspec"):
3886 self
.useClientSpec
= True
3887 if self
.useClientSpec
:
3888 self
.clientSpecDirs
= getClientSpec()
3890 # TODO: should always look at previous commits,
3891 # merge with previous imports, if possible.
3894 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
3896 # branches holds mapping from branch name to sha1
3897 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3899 # restrict to just this one, disabling detect-branches
3900 if branch_arg_given
:
3901 short
= self
.branch
.split("/")[-1]
3902 if short
in branches
:
3903 self
.p4BranchesInGit
= [ short
]
3905 self
.p4BranchesInGit
= branches
.keys()
3907 if len(self
.p4BranchesInGit
) > 1:
3909 print("Importing from/into multiple branches")
3910 self
.detectBranches
= True
3911 for branch
in branches
.keys():
3912 self
.initialParents
[self
.refPrefix
+ branch
] = \
3916 print("branches: %s" % self
.p4BranchesInGit
)
3919 for branch
in self
.p4BranchesInGit
:
3920 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
3922 settings
= extractSettingsGitLog(logMsg
)
3924 self
.readOptions(settings
)
3925 if ('depot-paths' in settings
3926 and 'change' in settings
):
3927 change
= int(settings
['change']) + 1
3928 p4Change
= max(p4Change
, change
)
3930 depotPaths
= sorted(settings
['depot-paths'])
3931 if self
.previousDepotPaths
== []:
3932 self
.previousDepotPaths
= depotPaths
3935 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
3936 prev_list
= prev
.split("/")
3937 cur_list
= cur
.split("/")
3938 for i
in range(0, min(len(cur_list
), len(prev_list
))):
3939 if cur_list
[i
] != prev_list
[i
]:
3943 paths
.append ("/".join(cur_list
[:i
+ 1]))
3945 self
.previousDepotPaths
= paths
3948 self
.depotPaths
= sorted(self
.previousDepotPaths
)
3949 self
.changeRange
= "@%s,#head" % p4Change
3950 if not self
.silent
and not self
.detectBranches
:
3951 print("Performing incremental import into %s git branch" % self
.branch
)
3953 # accept multiple ref name abbreviations:
3954 # refs/foo/bar/branch -> use it exactly
3955 # p4/branch -> prepend refs/remotes/ or refs/heads/
3956 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3957 if not self
.branch
.startswith("refs/"):
3958 if self
.importIntoRemotes
:
3959 prepend
= "refs/remotes/"
3961 prepend
= "refs/heads/"
3962 if not self
.branch
.startswith("p4/"):
3964 self
.branch
= prepend
+ self
.branch
3966 if len(args
) == 0 and self
.depotPaths
:
3968 print("Depot paths: %s" % ' '.join(self
.depotPaths
))
3970 if self
.depotPaths
and self
.depotPaths
!= args
:
3971 print("previous import used depot path %s and now %s was specified. "
3972 "This doesn't work!" % (' '.join (self
.depotPaths
),
3976 self
.depotPaths
= sorted(args
)
3981 # Make sure no revision specifiers are used when --changesfile
3983 bad_changesfile
= False
3984 if len(self
.changesFile
) > 0:
3985 for p
in self
.depotPaths
:
3986 if p
.find("@") >= 0 or p
.find("#") >= 0:
3987 bad_changesfile
= True
3990 die("Option --changesfile is incompatible with revision specifiers")
3993 for p
in self
.depotPaths
:
3994 if p
.find("@") != -1:
3995 atIdx
= p
.index("@")
3996 self
.changeRange
= p
[atIdx
:]
3997 if self
.changeRange
== "@all":
3998 self
.changeRange
= ""
3999 elif ',' not in self
.changeRange
:
4000 revision
= self
.changeRange
4001 self
.changeRange
= ""
4003 elif p
.find("#") != -1:
4004 hashIdx
= p
.index("#")
4005 revision
= p
[hashIdx
:]
4007 elif self
.previousDepotPaths
== []:
4008 # pay attention to changesfile, if given, else import
4009 # the entire p4 tree at the head revision
4010 if len(self
.changesFile
) == 0:
4013 p
= re
.sub ("\.\.\.$", "", p
)
4014 if not p
.endswith("/"):
4019 self
.depotPaths
= newPaths
4021 # --detect-branches may change this for each branch
4022 self
.branchPrefixes
= self
.depotPaths
4024 self
.loadUserMapFromCache()
4026 if self
.detectLabels
:
4029 if self
.detectBranches
:
4030 ## FIXME - what's a P4 projectName ?
4031 self
.projectName
= self
.guessProjectName()
4034 self
.getBranchMappingFromGitBranches()
4036 self
.getBranchMapping()
4038 print("p4-git branches: %s" % self
.p4BranchesInGit
)
4039 print("initial parents: %s" % self
.initialParents
)
4040 for b
in self
.p4BranchesInGit
:
4044 b
= b
[len(self
.projectName
):]
4045 self
.createdBranches
.add(b
)
4055 self
.importHeadRevision(revision
)
4057 self
.importRevisions(args
, branch_arg_given
)
4059 if gitConfigBool("git-p4.importLabels"):
4060 self
.importLabels
= True
4062 if self
.importLabels
:
4063 p4Labels
= getP4Labels(self
.depotPaths
)
4064 gitTags
= getGitTags()
4066 missingP4Labels
= p4Labels
- gitTags
4067 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
4069 except P4CommandException
as e
:
4078 # Cleanup temporary branches created during import
4079 if self
.tempBranches
!= []:
4080 for branch
in self
.tempBranches
:
4081 read_pipe("git update-ref -d %s" % branch
)
4082 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
4084 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
4085 # a convenient shortcut refname "p4".
4086 if self
.importIntoRemotes
:
4087 head_ref
= self
.refPrefix
+ "HEAD"
4088 if not gitBranchExists(head_ref
) and gitBranchExists(self
.branch
):
4089 system(["git", "symbolic-ref", head_ref
, self
.branch
])
4093 class P4Rebase(Command
):
4095 Command
.__init
__(self
)
4097 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
4099 self
.importLabels
= False
4100 self
.description
= ("Fetches the latest revision from perforce and "
4101 + "rebases the current work (branch) against it")
4103 def run(self
, args
):
4105 sync
.importLabels
= self
.importLabels
4108 return self
.rebase()
4111 if os
.system("git update-index --refresh") != 0:
4112 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
4113 if len(read_pipe("git diff-index HEAD --")) > 0:
4114 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
4116 [upstream
, settings
] = findUpstreamBranchPoint()
4117 if len(upstream
) == 0:
4118 die("Cannot find upstream branchpoint for rebase")
4120 # the branchpoint may be p4/foo~3, so strip off the parent
4121 upstream
= re
.sub("~[0-9]+$", "", upstream
)
4123 print("Rebasing the current branch onto %s" % upstream
)
4124 oldHead
= read_pipe("git rev-parse HEAD").strip()
4125 system("git rebase %s" % upstream
)
4126 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead
)
4129 class P4Clone(P4Sync
):
4131 P4Sync
.__init
__(self
)
4132 self
.description
= "Creates a new git repository and imports from Perforce into it"
4133 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
4135 optparse
.make_option("--destination", dest
="cloneDestination",
4136 action
='store', default
=None,
4137 help="where to leave result of the clone"),
4138 optparse
.make_option("--bare", dest
="cloneBare",
4139 action
="store_true", default
=False),
4141 self
.cloneDestination
= None
4142 self
.needsGit
= False
4143 self
.cloneBare
= False
4145 def defaultDestination(self
, args
):
4146 ## TODO: use common prefix of args?
4148 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
4149 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
4150 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
4151 depotDir
= re
.sub(r
"/$", "", depotDir
)
4152 return os
.path
.split(depotDir
)[1]
4154 def run(self
, args
):
4158 if self
.keepRepoPath
and not self
.cloneDestination
:
4159 sys
.stderr
.write("Must specify destination for --keep-path\n")
4164 if not self
.cloneDestination
and len(depotPaths
) > 1:
4165 self
.cloneDestination
= depotPaths
[-1]
4166 depotPaths
= depotPaths
[:-1]
4168 for p
in depotPaths
:
4169 if not p
.startswith("//"):
4170 sys
.stderr
.write('Depot paths must start with "//": %s\n' % p
)
4173 if not self
.cloneDestination
:
4174 self
.cloneDestination
= self
.defaultDestination(args
)
4176 print("Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
))
4178 if not os
.path
.exists(self
.cloneDestination
):
4179 os
.makedirs(self
.cloneDestination
)
4180 chdir(self
.cloneDestination
)
4182 init_cmd
= [ "git", "init" ]
4184 init_cmd
.append("--bare")
4185 retcode
= subprocess
.call(init_cmd
)
4187 raise CalledProcessError(retcode
, init_cmd
)
4189 if not P4Sync
.run(self
, depotPaths
):
4192 # create a master branch and check out a work tree
4193 if gitBranchExists(self
.branch
):
4194 system([ "git", "branch", currentGitBranch(), self
.branch
])
4195 if not self
.cloneBare
:
4196 system([ "git", "checkout", "-f" ])
4198 print('Not checking out any branch, use ' \
4199 '"git checkout -q -b master <branch>"')
4201 # auto-set this variable if invoked with --use-client-spec
4202 if self
.useClientSpec_from_options
:
4203 system("git config --bool git-p4.useclientspec true")
4207 class P4Unshelve(Command
):
4209 Command
.__init
__(self
)
4211 self
.origin
= "HEAD"
4212 self
.description
= "Unshelve a P4 changelist into a git commit"
4213 self
.usage
= "usage: %prog [options] changelist"
4215 optparse
.make_option("--origin", dest
="origin",
4216 help="Use this base revision instead of the default (%s)" % self
.origin
),
4218 self
.verbose
= False
4219 self
.noCommit
= False
4220 self
.destbranch
= "refs/remotes/p4-unshelved"
4222 def renameBranch(self
, branch_name
):
4223 """ Rename the existing branch to branch_name.N
4227 for i
in range(0,1000):
4228 backup_branch_name
= "{0}.{1}".format(branch_name
, i
)
4229 if not gitBranchExists(backup_branch_name
):
4230 gitUpdateRef(backup_branch_name
, branch_name
) # copy ref to backup
4231 gitDeleteRef(branch_name
)
4233 print("renamed old unshelve branch to {0}".format(backup_branch_name
))
4237 sys
.exit("gave up trying to rename existing branch {0}".format(sync
.branch
))
4239 def findLastP4Revision(self
, starting_point
):
4240 """ Look back from starting_point for the first commit created by git-p4
4241 to find the P4 commit we are based on, and the depot-paths.
4244 for parent
in (range(65535)):
4245 log
= extractLogMessageFromGitCommit("{0}~{1}".format(starting_point
, parent
))
4246 settings
= extractSettingsGitLog(log
)
4247 if 'change' in settings
:
4250 sys
.exit("could not find git-p4 commits in {0}".format(self
.origin
))
4252 def createShelveParent(self
, change
, branch_name
, sync
, origin
):
4253 """ Create a commit matching the parent of the shelved changelist 'change'
4255 parent_description
= p4_describe(change
, shelved
=True)
4256 parent_description
['desc'] = 'parent for shelved changelist {}\n'.format(change
)
4257 files
= sync
.extractFilesFromCommit(parent_description
, shelved
=False, shelved_cl
=change
)
4261 # if it was added in the shelved changelist, it won't exist in the parent
4262 if f
['action'] in self
.add_actions
:
4265 # if it was deleted in the shelved changelist it must not be deleted
4266 # in the parent - we might even need to create it if the origin branch
4268 if f
['action'] in self
.delete_actions
:
4271 parent_files
.append(f
)
4273 sync
.commit(parent_description
, parent_files
, branch_name
,
4274 parent
=origin
, allow_empty
=True)
4275 print("created parent commit for {0} based on {1} in {2}".format(
4276 change
, self
.origin
, branch_name
))
4278 def run(self
, args
):
4282 if not gitBranchExists(self
.origin
):
4283 sys
.exit("origin branch {0} does not exist".format(self
.origin
))
4288 # only one change at a time
4291 # if the target branch already exists, rename it
4292 branch_name
= "{0}/{1}".format(self
.destbranch
, change
)
4293 if gitBranchExists(branch_name
):
4294 self
.renameBranch(branch_name
)
4295 sync
.branch
= branch_name
4297 sync
.verbose
= self
.verbose
4298 sync
.suppress_meta_comment
= True
4300 settings
= self
.findLastP4Revision(self
.origin
)
4301 sync
.depotPaths
= settings
['depot-paths']
4302 sync
.branchPrefixes
= sync
.depotPaths
4305 sync
.loadUserMapFromCache()
4308 # create a commit for the parent of the shelved changelist
4309 self
.createShelveParent(change
, branch_name
, sync
, self
.origin
)
4311 # create the commit for the shelved changelist itself
4312 description
= p4_describe(change
, True)
4313 files
= sync
.extractFilesFromCommit(description
, True, change
)
4315 sync
.commit(description
, files
, branch_name
, "")
4318 print("unshelved changelist {0} into {1}".format(change
, branch_name
))
4322 class P4Branches(Command
):
4324 Command
.__init
__(self
)
4326 self
.description
= ("Shows the git branches that hold imports and their "
4327 + "corresponding perforce depot paths")
4328 self
.verbose
= False
4330 def run(self
, args
):
4331 if originP4BranchesExist():
4332 createOrUpdateBranchesFromOrigin()
4334 cmdline
= "git rev-parse --symbolic "
4335 cmdline
+= " --remotes"
4337 for line
in read_pipe_lines(cmdline
):
4340 if not line
.startswith('p4/') or line
== "p4/HEAD":
4344 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
4345 settings
= extractSettingsGitLog(log
)
4347 print("%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"]))
4350 class HelpFormatter(optparse
.IndentedHelpFormatter
):
4352 optparse
.IndentedHelpFormatter
.__init
__(self
)
4354 def format_description(self
, description
):
4356 return description
+ "\n"
4360 def printUsage(commands
):
4361 print("usage: %s <command> [options]" % sys
.argv
[0])
4363 print("valid commands: %s" % ", ".join(commands
))
4365 print("Try %s <command> --help for command specific help." % sys
.argv
[0])
4370 "submit" : P4Submit
,
4371 "commit" : P4Submit
,
4373 "rebase" : P4Rebase
,
4375 "rollback" : P4RollBack
,
4376 "branches" : P4Branches
,
4377 "unshelve" : P4Unshelve
,
4381 if len(sys
.argv
[1:]) == 0:
4382 printUsage(commands
.keys())
4385 cmdName
= sys
.argv
[1]
4387 klass
= commands
[cmdName
]
4390 print("unknown command %s" % cmdName
)
4392 printUsage(commands
.keys())
4395 options
= cmd
.options
4396 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
4400 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
4402 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
4404 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
4406 description
= cmd
.description
,
4407 formatter
= HelpFormatter())
4410 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
4416 verbose
= cmd
.verbose
4418 if cmd
.gitdir
== None:
4419 cmd
.gitdir
= os
.path
.abspath(".git")
4420 if not isValidGitDir(cmd
.gitdir
):
4421 # "rev-parse --git-dir" without arguments will try $PWD/.git
4422 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
4423 if os
.path
.exists(cmd
.gitdir
):
4424 cdup
= read_pipe("git rev-parse --show-cdup").strip()
4428 if not isValidGitDir(cmd
.gitdir
):
4429 if isValidGitDir(cmd
.gitdir
+ "/.git"):
4430 cmd
.gitdir
+= "/.git"
4432 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
4434 # so git commands invoked from the P4 workspace will succeed
4435 os
.environ
["GIT_DIR"] = cmd
.gitdir
4437 if not cmd
.run(args
):
4442 if __name__
== '__main__':