]> git.ipfire.org Git - thirdparty/git.git/blob - git-p4.py
Sync with 2.39.3
[thirdparty/git.git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 # pylint: disable=bad-whitespace
11 # pylint: disable=broad-except
12 # pylint: disable=consider-iterating-dictionary
13 # pylint: disable=disable
14 # pylint: disable=fixme
15 # pylint: disable=invalid-name
16 # pylint: disable=line-too-long
17 # pylint: disable=missing-docstring
18 # pylint: disable=no-self-use
19 # pylint: disable=superfluous-parens
20 # pylint: disable=too-few-public-methods
21 # pylint: disable=too-many-arguments
22 # pylint: disable=too-many-branches
23 # pylint: disable=too-many-instance-attributes
24 # pylint: disable=too-many-lines
25 # pylint: disable=too-many-locals
26 # pylint: disable=too-many-nested-blocks
27 # pylint: disable=too-many-statements
28 # pylint: disable=ungrouped-imports
29 # pylint: disable=unused-import
30 # pylint: disable=wrong-import-order
31 # pylint: disable=wrong-import-position
32 #
33
34 import struct
35 import sys
36 if sys.version_info.major < 3 and sys.version_info.minor < 7:
37 sys.stderr.write("git-p4: requires Python 2.7 or later.\n")
38 sys.exit(1)
39
40 import ctypes
41 import errno
42 import functools
43 import glob
44 import marshal
45 import optparse
46 import os
47 import platform
48 import re
49 import shutil
50 import stat
51 import subprocess
52 import tempfile
53 import time
54 import zipfile
55 import zlib
56
57 # On python2.7 where raw_input() and input() are both availble,
58 # we want raw_input's semantics, but aliased to input for python3
59 # compatibility
60 # support basestring in python3
61 try:
62 if raw_input and input:
63 input = raw_input
64 except:
65 pass
66
67 verbose = False
68
69 # Only labels/tags matching this will be imported/exported
70 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
71
72 # The block size is reduced automatically if required
73 defaultBlockSize = 1 << 20
74
75 defaultMetadataDecodingStrategy = 'passthrough' if sys.version_info.major == 2 else 'fallback'
76 defaultFallbackMetadataEncoding = 'cp1252'
77
78 p4_access_checked = False
79
80 re_ko_keywords = re.compile(br'\$(Id|Header)(:[^$\n]+)?\$')
81 re_k_keywords = re.compile(br'\$(Id|Header|Author|Date|DateTime|Change|File|Revision)(:[^$\n]+)?\$')
82
83
84 def format_size_human_readable(num):
85 """Returns a number of units (typically bytes) formatted as a
86 human-readable string.
87 """
88 if num < 1024:
89 return '{:d} B'.format(num)
90 for unit in ["Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
91 num /= 1024.0
92 if num < 1024.0:
93 return "{:3.1f} {}B".format(num, unit)
94 return "{:.1f} YiB".format(num)
95
96
97 def p4_build_cmd(cmd):
98 """Build a suitable p4 command line.
99
100 This consolidates building and returning a p4 command line into one
101 location. It means that hooking into the environment, or other
102 configuration can be done more easily.
103 """
104 real_cmd = ["p4"]
105
106 user = gitConfig("git-p4.user")
107 if len(user) > 0:
108 real_cmd += ["-u", user]
109
110 password = gitConfig("git-p4.password")
111 if len(password) > 0:
112 real_cmd += ["-P", password]
113
114 port = gitConfig("git-p4.port")
115 if len(port) > 0:
116 real_cmd += ["-p", port]
117
118 host = gitConfig("git-p4.host")
119 if len(host) > 0:
120 real_cmd += ["-H", host]
121
122 client = gitConfig("git-p4.client")
123 if len(client) > 0:
124 real_cmd += ["-c", client]
125
126 retries = gitConfigInt("git-p4.retries")
127 if retries is None:
128 # Perform 3 retries by default
129 retries = 3
130 if retries > 0:
131 # Provide a way to not pass this option by setting git-p4.retries to 0
132 real_cmd += ["-r", str(retries)]
133
134 real_cmd += cmd
135
136 # now check that we can actually talk to the server
137 global p4_access_checked
138 if not p4_access_checked:
139 p4_access_checked = True # suppress access checks in p4_check_access itself
140 p4_check_access()
141
142 return real_cmd
143
144
145 def git_dir(path):
146 """Return TRUE if the given path is a git directory (/path/to/dir/.git).
147 This won't automatically add ".git" to a directory.
148 """
149 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
150 if not d or len(d) == 0:
151 return None
152 else:
153 return d
154
155
156 def chdir(path, is_client_path=False):
157 """Do chdir to the given path, and set the PWD environment variable for use
158 by P4. It does not look at getcwd() output. Since we're not using the
159 shell, it is necessary to set the PWD environment variable explicitly.
160
161 Normally, expand the path to force it to be absolute. This addresses
162 the use of relative path names inside P4 settings, e.g.
163 P4CONFIG=.p4config. P4 does not simply open the filename as given; it
164 looks for .p4config using PWD.
165
166 If is_client_path, the path was handed to us directly by p4, and may be
167 a symbolic link. Do not call os.getcwd() in this case, because it will
168 cause p4 to think that PWD is not inside the client path.
169 """
170
171 os.chdir(path)
172 if not is_client_path:
173 path = os.getcwd()
174 os.environ['PWD'] = path
175
176
177 def calcDiskFree():
178 """Return free space in bytes on the disk of the given dirname."""
179 if platform.system() == 'Windows':
180 free_bytes = ctypes.c_ulonglong(0)
181 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
182 return free_bytes.value
183 else:
184 st = os.statvfs(os.getcwd())
185 return st.f_bavail * st.f_frsize
186
187
188 def die(msg):
189 """Terminate execution. Make sure that any running child processes have
190 been wait()ed for before calling this.
191 """
192 if verbose:
193 raise Exception(msg)
194 else:
195 sys.stderr.write(msg + "\n")
196 sys.exit(1)
197
198
199 def prompt(prompt_text):
200 """Prompt the user to choose one of the choices.
201
202 Choices are identified in the prompt_text by square brackets around a
203 single letter option.
204 """
205 choices = set(m.group(1) for m in re.finditer(r"\[(.)\]", prompt_text))
206 while True:
207 sys.stderr.flush()
208 sys.stdout.write(prompt_text)
209 sys.stdout.flush()
210 response = sys.stdin.readline().strip().lower()
211 if not response:
212 continue
213 response = response[0]
214 if response in choices:
215 return response
216
217
218 # We need different encoding/decoding strategies for text data being passed
219 # around in pipes depending on python version
220 if bytes is not str:
221 # For python3, always encode and decode as appropriate
222 def decode_text_stream(s):
223 return s.decode() if isinstance(s, bytes) else s
224
225 def encode_text_stream(s):
226 return s.encode() if isinstance(s, str) else s
227 else:
228 # For python2.7, pass read strings as-is, but also allow writing unicode
229 def decode_text_stream(s):
230 return s
231
232 def encode_text_stream(s):
233 return s.encode('utf_8') if isinstance(s, unicode) else s
234
235
236 class MetadataDecodingException(Exception):
237 def __init__(self, input_string):
238 self.input_string = input_string
239
240 def __str__(self):
241 return """Decoding perforce metadata failed!
242 The failing string was:
243 ---
244 {}
245 ---
246 Consider setting the git-p4.metadataDecodingStrategy config option to
247 'fallback', to allow metadata to be decoded using a fallback encoding,
248 defaulting to cp1252.""".format(self.input_string)
249
250
251 encoding_fallback_warning_issued = False
252 encoding_escape_warning_issued = False
253 def metadata_stream_to_writable_bytes(s):
254 encodingStrategy = gitConfig('git-p4.metadataDecodingStrategy') or defaultMetadataDecodingStrategy
255 fallbackEncoding = gitConfig('git-p4.metadataFallbackEncoding') or defaultFallbackMetadataEncoding
256 if not isinstance(s, bytes):
257 return s.encode('utf_8')
258 if encodingStrategy == 'passthrough':
259 return s
260 try:
261 s.decode('utf_8')
262 return s
263 except UnicodeDecodeError:
264 if encodingStrategy == 'fallback' and fallbackEncoding:
265 global encoding_fallback_warning_issued
266 global encoding_escape_warning_issued
267 try:
268 if not encoding_fallback_warning_issued:
269 print("\nCould not decode value as utf-8; using configured fallback encoding %s: %s" % (fallbackEncoding, s))
270 print("\n(this warning is only displayed once during an import)")
271 encoding_fallback_warning_issued = True
272 return s.decode(fallbackEncoding).encode('utf_8')
273 except Exception as exc:
274 if not encoding_escape_warning_issued:
275 print("\nCould not decode value with configured fallback encoding %s; escaping bytes over 127: %s" % (fallbackEncoding, s))
276 print("\n(this warning is only displayed once during an import)")
277 encoding_escape_warning_issued = True
278 escaped_bytes = b''
279 # bytes and strings work very differently in python2 vs python3...
280 if str is bytes:
281 for byte in s:
282 byte_number = struct.unpack('>B', byte)[0]
283 if byte_number > 127:
284 escaped_bytes += b'%'
285 escaped_bytes += hex(byte_number)[2:].upper()
286 else:
287 escaped_bytes += byte
288 else:
289 for byte_number in s:
290 if byte_number > 127:
291 escaped_bytes += b'%'
292 escaped_bytes += hex(byte_number).upper().encode()[2:]
293 else:
294 escaped_bytes += bytes([byte_number])
295 return escaped_bytes
296
297 raise MetadataDecodingException(s)
298
299
300 def decode_path(path):
301 """Decode a given string (bytes or otherwise) using configured path
302 encoding options.
303 """
304
305 encoding = gitConfig('git-p4.pathEncoding') or 'utf_8'
306 if bytes is not str:
307 return path.decode(encoding, errors='replace') if isinstance(path, bytes) else path
308 else:
309 try:
310 path.decode('ascii')
311 except:
312 path = path.decode(encoding, errors='replace')
313 if verbose:
314 print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding, path))
315 return path
316
317
318 def run_git_hook(cmd, param=[]):
319 """Execute a hook if the hook exists."""
320 args = ['git', 'hook', 'run', '--ignore-missing', cmd]
321 if param:
322 args.append("--")
323 for p in param:
324 args.append(p)
325 return subprocess.call(args) == 0
326
327
328 def write_pipe(c, stdin, *k, **kw):
329 if verbose:
330 sys.stderr.write('Writing pipe: {}\n'.format(' '.join(c)))
331
332 p = subprocess.Popen(c, stdin=subprocess.PIPE, *k, **kw)
333 pipe = p.stdin
334 val = pipe.write(stdin)
335 pipe.close()
336 if p.wait():
337 die('Command failed: {}'.format(' '.join(c)))
338
339 return val
340
341
342 def p4_write_pipe(c, stdin, *k, **kw):
343 real_cmd = p4_build_cmd(c)
344 if bytes is not str and isinstance(stdin, str):
345 stdin = encode_text_stream(stdin)
346 return write_pipe(real_cmd, stdin, *k, **kw)
347
348
349 def read_pipe_full(c, *k, **kw):
350 """Read output from command. Returns a tuple of the return status, stdout
351 text and stderr text.
352 """
353 if verbose:
354 sys.stderr.write('Reading pipe: {}\n'.format(' '.join(c)))
355
356 p = subprocess.Popen(
357 c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, *k, **kw)
358 out, err = p.communicate()
359 return (p.returncode, out, decode_text_stream(err))
360
361
362 def read_pipe(c, ignore_error=False, raw=False, *k, **kw):
363 """Read output from command. Returns the output text on success. On
364 failure, terminates execution, unless ignore_error is True, when it
365 returns an empty string.
366
367 If raw is True, do not attempt to decode output text.
368 """
369 retcode, out, err = read_pipe_full(c, *k, **kw)
370 if retcode != 0:
371 if ignore_error:
372 out = ""
373 else:
374 die('Command failed: {}\nError: {}'.format(' '.join(c), err))
375 if not raw:
376 out = decode_text_stream(out)
377 return out
378
379
380 def read_pipe_text(c, *k, **kw):
381 """Read output from a command with trailing whitespace stripped. On error,
382 returns None.
383 """
384 retcode, out, err = read_pipe_full(c, *k, **kw)
385 if retcode != 0:
386 return None
387 else:
388 return decode_text_stream(out).rstrip()
389
390
391 def p4_read_pipe(c, ignore_error=False, raw=False, *k, **kw):
392 real_cmd = p4_build_cmd(c)
393 return read_pipe(real_cmd, ignore_error, raw=raw, *k, **kw)
394
395
396 def read_pipe_lines(c, raw=False, *k, **kw):
397 if verbose:
398 sys.stderr.write('Reading pipe: {}\n'.format(' '.join(c)))
399
400 p = subprocess.Popen(c, stdout=subprocess.PIPE, *k, **kw)
401 pipe = p.stdout
402 lines = pipe.readlines()
403 if not raw:
404 lines = [decode_text_stream(line) for line in lines]
405 if pipe.close() or p.wait():
406 die('Command failed: {}'.format(' '.join(c)))
407 return lines
408
409
410 def p4_read_pipe_lines(c, *k, **kw):
411 """Specifically invoke p4 on the command supplied."""
412 real_cmd = p4_build_cmd(c)
413 return read_pipe_lines(real_cmd, *k, **kw)
414
415
416 def p4_has_command(cmd):
417 """Ask p4 for help on this command. If it returns an error, the command
418 does not exist in this version of p4.
419 """
420 real_cmd = p4_build_cmd(["help", cmd])
421 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
422 stderr=subprocess.PIPE)
423 p.communicate()
424 return p.returncode == 0
425
426
427 def p4_has_move_command():
428 """See if the move command exists, that it supports -k, and that it has not
429 been administratively disabled. The arguments must be correct, but the
430 filenames do not have to exist. Use ones with wildcards so even if they
431 exist, it will fail.
432 """
433
434 if not p4_has_command("move"):
435 return False
436 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
437 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
438 out, err = p.communicate()
439 err = decode_text_stream(err)
440 # return code will be 1 in either case
441 if err.find("Invalid option") >= 0:
442 return False
443 if err.find("disabled") >= 0:
444 return False
445 # assume it failed because @... was invalid changelist
446 return True
447
448
449 def system(cmd, ignore_error=False, *k, **kw):
450 if verbose:
451 sys.stderr.write("executing {}\n".format(
452 ' '.join(cmd) if isinstance(cmd, list) else cmd))
453 retcode = subprocess.call(cmd, *k, **kw)
454 if retcode and not ignore_error:
455 raise subprocess.CalledProcessError(retcode, cmd)
456
457 return retcode
458
459
460 def p4_system(cmd, *k, **kw):
461 """Specifically invoke p4 as the system command."""
462 real_cmd = p4_build_cmd(cmd)
463 retcode = subprocess.call(real_cmd, *k, **kw)
464 if retcode:
465 raise subprocess.CalledProcessError(retcode, real_cmd)
466
467
468 def die_bad_access(s):
469 die("failure accessing depot: {0}".format(s.rstrip()))
470
471
472 def p4_check_access(min_expiration=1):
473 """Check if we can access Perforce - account still logged in."""
474
475 results = p4CmdList(["login", "-s"])
476
477 if len(results) == 0:
478 # should never get here: always get either some results, or a p4ExitCode
479 assert("could not parse response from perforce")
480
481 result = results[0]
482
483 if 'p4ExitCode' in result:
484 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
485 die_bad_access("could not run p4")
486
487 code = result.get("code")
488 if not code:
489 # we get here if we couldn't connect and there was nothing to unmarshal
490 die_bad_access("could not connect")
491
492 elif code == "stat":
493 expiry = result.get("TicketExpiration")
494 if expiry:
495 expiry = int(expiry)
496 if expiry > min_expiration:
497 # ok to carry on
498 return
499 else:
500 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
501
502 else:
503 # account without a timeout - all ok
504 return
505
506 elif code == "error":
507 data = result.get("data")
508 if data:
509 die_bad_access("p4 error: {0}".format(data))
510 else:
511 die_bad_access("unknown error")
512 elif code == "info":
513 return
514 else:
515 die_bad_access("unknown error code {0}".format(code))
516
517
518 _p4_version_string = None
519
520
521 def p4_version_string():
522 """Read the version string, showing just the last line, which hopefully is
523 the interesting version bit.
524
525 $ p4 -V
526 Perforce - The Fast Software Configuration Management System.
527 Copyright 1995-2011 Perforce Software. All rights reserved.
528 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
529 """
530 global _p4_version_string
531 if not _p4_version_string:
532 a = p4_read_pipe_lines(["-V"])
533 _p4_version_string = a[-1].rstrip()
534 return _p4_version_string
535
536
537 def p4_integrate(src, dest):
538 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
539
540
541 def p4_sync(f, *options):
542 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
543
544
545 def p4_add(f):
546 """Forcibly add file names with wildcards."""
547 if wildcard_present(f):
548 p4_system(["add", "-f", f])
549 else:
550 p4_system(["add", f])
551
552
553 def p4_delete(f):
554 p4_system(["delete", wildcard_encode(f)])
555
556
557 def p4_edit(f, *options):
558 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
559
560
561 def p4_revert(f):
562 p4_system(["revert", wildcard_encode(f)])
563
564
565 def p4_reopen(type, f):
566 p4_system(["reopen", "-t", type, wildcard_encode(f)])
567
568
569 def p4_reopen_in_change(changelist, files):
570 cmd = ["reopen", "-c", str(changelist)] + files
571 p4_system(cmd)
572
573
574 def p4_move(src, dest):
575 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
576
577
578 def p4_last_change():
579 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
580 return int(results[0]['change'])
581
582
583 def p4_describe(change, shelved=False):
584 """Make sure it returns a valid result by checking for the presence of
585 field "time".
586
587 Return a dict of the results.
588 """
589
590 cmd = ["describe", "-s"]
591 if shelved:
592 cmd += ["-S"]
593 cmd += [str(change)]
594
595 ds = p4CmdList(cmd, skip_info=True)
596 if len(ds) != 1:
597 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
598
599 d = ds[0]
600
601 if "p4ExitCode" in d:
602 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
603 str(d)))
604 if "code" in d:
605 if d["code"] == "error":
606 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
607
608 if "time" not in d:
609 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
610
611 return d
612
613
614 def split_p4_type(p4type):
615 """Canonicalize the p4 type and return a tuple of the base type, plus any
616 modifiers. See "p4 help filetypes" for a list and explanation.
617 """
618
619 p4_filetypes_historical = {
620 "ctempobj": "binary+Sw",
621 "ctext": "text+C",
622 "cxtext": "text+Cx",
623 "ktext": "text+k",
624 "kxtext": "text+kx",
625 "ltext": "text+F",
626 "tempobj": "binary+FSw",
627 "ubinary": "binary+F",
628 "uresource": "resource+F",
629 "uxbinary": "binary+Fx",
630 "xbinary": "binary+x",
631 "xltext": "text+Fx",
632 "xtempobj": "binary+Swx",
633 "xtext": "text+x",
634 "xunicode": "unicode+x",
635 "xutf16": "utf16+x",
636 }
637 if p4type in p4_filetypes_historical:
638 p4type = p4_filetypes_historical[p4type]
639 mods = ""
640 s = p4type.split("+")
641 base = s[0]
642 mods = ""
643 if len(s) > 1:
644 mods = s[1]
645 return (base, mods)
646
647
648 def p4_type(f):
649 """Return the raw p4 type of a file (text, text+ko, etc)."""
650
651 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
652 return results[0]['headType']
653
654
655 def p4_keywords_regexp_for_type(base, type_mods):
656 """Given a type base and modifier, return a regexp matching the keywords
657 that can be expanded in the file.
658 """
659
660 if base in ("text", "unicode", "binary"):
661 if "ko" in type_mods:
662 return re_ko_keywords
663 elif "k" in type_mods:
664 return re_k_keywords
665 else:
666 return None
667 else:
668 return None
669
670
671 def p4_keywords_regexp_for_file(file):
672 """Given a file, return a regexp matching the possible RCS keywords that
673 will be expanded, or None for files with kw expansion turned off.
674 """
675
676 if not os.path.exists(file):
677 return None
678 else:
679 type_base, type_mods = split_p4_type(p4_type(file))
680 return p4_keywords_regexp_for_type(type_base, type_mods)
681
682
683 def setP4ExecBit(file, mode):
684 """Reopens an already open file and changes the execute bit to match the
685 execute bit setting in the passed in mode.
686 """
687
688 p4Type = "+x"
689
690 if not isModeExec(mode):
691 p4Type = getP4OpenedType(file)
692 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
693 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
694 if p4Type[-1] == "+":
695 p4Type = p4Type[0:-1]
696
697 p4_reopen(p4Type, file)
698
699
700 def getP4OpenedType(file):
701 """Returns the perforce file type for the given file."""
702
703 result = p4_read_pipe(["opened", wildcard_encode(file)])
704 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
705 if match:
706 return match.group(1)
707 else:
708 die("Could not determine file type for %s (result: '%s')" % (file, result))
709
710
711 def getP4Labels(depotPaths):
712 """Return the set of all p4 labels."""
713
714 labels = set()
715 if not isinstance(depotPaths, list):
716 depotPaths = [depotPaths]
717
718 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
719 label = l['label']
720 labels.add(label)
721
722 return labels
723
724
725 def getGitTags():
726 """Return the set of all git tags."""
727
728 gitTags = set()
729 for line in read_pipe_lines(["git", "tag"]):
730 tag = line.strip()
731 gitTags.add(tag)
732 return gitTags
733
734
735 _diff_tree_pattern = None
736
737
738 def parseDiffTreeEntry(entry):
739 """Parses a single diff tree entry into its component elements.
740
741 See git-diff-tree(1) manpage for details about the format of the diff
742 output. This method returns a dictionary with the following elements:
743
744 src_mode - The mode of the source file
745 dst_mode - The mode of the destination file
746 src_sha1 - The sha1 for the source file
747 dst_sha1 - The sha1 fr the destination file
748 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
749 status_score - The score for the status (applicable for 'C' and 'R'
750 statuses). This is None if there is no score.
751 src - The path for the source file.
752 dst - The path for the destination file. This is only present for
753 copy or renames. If it is not present, this is None.
754
755 If the pattern is not matched, None is returned.
756 """
757
758 global _diff_tree_pattern
759 if not _diff_tree_pattern:
760 _diff_tree_pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
761
762 match = _diff_tree_pattern.match(entry)
763 if match:
764 return {
765 'src_mode': match.group(1),
766 'dst_mode': match.group(2),
767 'src_sha1': match.group(3),
768 'dst_sha1': match.group(4),
769 'status': match.group(5),
770 'status_score': match.group(6),
771 'src': match.group(7),
772 'dst': match.group(10)
773 }
774 return None
775
776
777 def isModeExec(mode):
778 """Returns True if the given git mode represents an executable file,
779 otherwise False.
780 """
781 return mode[-3:] == "755"
782
783
784 class P4Exception(Exception):
785 """Base class for exceptions from the p4 client."""
786
787 def __init__(self, exit_code):
788 self.p4ExitCode = exit_code
789
790
791 class P4ServerException(P4Exception):
792 """Base class for exceptions where we get some kind of marshalled up result
793 from the server.
794 """
795
796 def __init__(self, exit_code, p4_result):
797 super(P4ServerException, self).__init__(exit_code)
798 self.p4_result = p4_result
799 self.code = p4_result[0]['code']
800 self.data = p4_result[0]['data']
801
802
803 class P4RequestSizeException(P4ServerException):
804 """One of the maxresults or maxscanrows errors."""
805
806 def __init__(self, exit_code, p4_result, limit):
807 super(P4RequestSizeException, self).__init__(exit_code, p4_result)
808 self.limit = limit
809
810
811 class P4CommandException(P4Exception):
812 """Something went wrong calling p4 which means we have to give up."""
813
814 def __init__(self, msg):
815 self.msg = msg
816
817 def __str__(self):
818 return self.msg
819
820
821 def isModeExecChanged(src_mode, dst_mode):
822 return isModeExec(src_mode) != isModeExec(dst_mode)
823
824
825 def p4KeysContainingNonUtf8Chars():
826 """Returns all keys which may contain non UTF-8 encoded strings
827 for which a fallback strategy has to be applied.
828 """
829 return ['desc', 'client', 'FullName']
830
831
832 def p4KeysContainingBinaryData():
833 """Returns all keys which may contain arbitrary binary data
834 """
835 return ['data']
836
837
838 def p4KeyContainsFilePaths(key):
839 """Returns True if the key contains file paths. These are handled by decode_path().
840 Otherwise False.
841 """
842 return key.startswith('depotFile') or key in ['path', 'clientFile']
843
844
845 def p4KeyWhichCanBeDirectlyDecoded(key):
846 """Returns True if the key can be directly decoded as UTF-8 string
847 Otherwise False.
848
849 Keys which can not be encoded directly:
850 - `data` which may contain arbitrary binary data
851 - `desc` or `client` or `FullName` which may contain non-UTF8 encoded text
852 - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text, handled by decode_path()
853 """
854 if key in p4KeysContainingNonUtf8Chars() or \
855 key in p4KeysContainingBinaryData() or \
856 p4KeyContainsFilePaths(key):
857 return False
858 return True
859
860
861 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
862 errors_as_exceptions=False, *k, **kw):
863
864 cmd = p4_build_cmd(["-G"] + cmd)
865 if verbose:
866 sys.stderr.write("Opening pipe: {}\n".format(' '.join(cmd)))
867
868 # Use a temporary file to avoid deadlocks without
869 # subprocess.communicate(), which would put another copy
870 # of stdout into memory.
871 stdin_file = None
872 if stdin is not None:
873 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
874 if not isinstance(stdin, list):
875 stdin_file.write(stdin)
876 else:
877 for i in stdin:
878 stdin_file.write(encode_text_stream(i))
879 stdin_file.write(b'\n')
880 stdin_file.flush()
881 stdin_file.seek(0)
882
883 p4 = subprocess.Popen(
884 cmd, stdin=stdin_file, stdout=subprocess.PIPE, *k, **kw)
885
886 result = []
887 try:
888 while True:
889 entry = marshal.load(p4.stdout)
890
891 if bytes is not str:
892 # Decode unmarshalled dict to use str keys and values. Special cases are handled below.
893 decoded_entry = {}
894 for key, value in entry.items():
895 key = key.decode()
896 if isinstance(value, bytes) and p4KeyWhichCanBeDirectlyDecoded(key):
897 value = value.decode()
898 decoded_entry[key] = value
899 # Parse out data if it's an error response
900 if decoded_entry.get('code') == 'error' and 'data' in decoded_entry:
901 decoded_entry['data'] = decoded_entry['data'].decode()
902 entry = decoded_entry
903 if skip_info:
904 if 'code' in entry and entry['code'] == 'info':
905 continue
906 for key in p4KeysContainingNonUtf8Chars():
907 if key in entry:
908 entry[key] = metadata_stream_to_writable_bytes(entry[key])
909 if cb is not None:
910 cb(entry)
911 else:
912 result.append(entry)
913 except EOFError:
914 pass
915 exitCode = p4.wait()
916 if exitCode != 0:
917 if errors_as_exceptions:
918 if len(result) > 0:
919 data = result[0].get('data')
920 if data:
921 m = re.search('Too many rows scanned \(over (\d+)\)', data)
922 if not m:
923 m = re.search('Request too large \(over (\d+)\)', data)
924
925 if m:
926 limit = int(m.group(1))
927 raise P4RequestSizeException(exitCode, result, limit)
928
929 raise P4ServerException(exitCode, result)
930 else:
931 raise P4Exception(exitCode)
932 else:
933 entry = {}
934 entry["p4ExitCode"] = exitCode
935 result.append(entry)
936
937 return result
938
939
940 def p4Cmd(cmd, *k, **kw):
941 list = p4CmdList(cmd, *k, **kw)
942 result = {}
943 for entry in list:
944 result.update(entry)
945 return result
946
947
948 def p4Where(depotPath):
949 if not depotPath.endswith("/"):
950 depotPath += "/"
951 depotPathLong = depotPath + "..."
952 outputList = p4CmdList(["where", depotPathLong])
953 output = None
954 for entry in outputList:
955 if "depotFile" in entry:
956 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
957 # The base path always ends with "/...".
958 entry_path = decode_path(entry['depotFile'])
959 if entry_path.find(depotPath) == 0 and entry_path[-4:] == "/...":
960 output = entry
961 break
962 elif "data" in entry:
963 data = entry.get("data")
964 space = data.find(" ")
965 if data[:space] == depotPath:
966 output = entry
967 break
968 if output is None:
969 return ""
970 if output["code"] == "error":
971 return ""
972 clientPath = ""
973 if "path" in output:
974 clientPath = decode_path(output['path'])
975 elif "data" in output:
976 data = output.get("data")
977 lastSpace = data.rfind(b" ")
978 clientPath = decode_path(data[lastSpace + 1:])
979
980 if clientPath.endswith("..."):
981 clientPath = clientPath[:-3]
982 return clientPath
983
984
985 def currentGitBranch():
986 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
987
988
989 def isValidGitDir(path):
990 return git_dir(path) is not None
991
992
993 def parseRevision(ref):
994 return read_pipe(["git", "rev-parse", ref]).strip()
995
996
997 def branchExists(ref):
998 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
999 ignore_error=True)
1000 return len(rev) > 0
1001
1002
1003 def extractLogMessageFromGitCommit(commit):
1004 logMessage = ""
1005
1006 # fixme: title is first line of commit, not 1st paragraph.
1007 foundTitle = False
1008 for log in read_pipe_lines(["git", "cat-file", "commit", commit]):
1009 if not foundTitle:
1010 if len(log) == 1:
1011 foundTitle = True
1012 continue
1013
1014 logMessage += log
1015 return logMessage
1016
1017
1018 def extractSettingsGitLog(log):
1019 values = {}
1020 for line in log.split("\n"):
1021 line = line.strip()
1022 m = re.search(r"^ *\[git-p4: (.*)\]$", line)
1023 if not m:
1024 continue
1025
1026 assignments = m.group(1).split(':')
1027 for a in assignments:
1028 vals = a.split('=')
1029 key = vals[0].strip()
1030 val = ('='.join(vals[1:])).strip()
1031 if val.endswith('\"') and val.startswith('"'):
1032 val = val[1:-1]
1033
1034 values[key] = val
1035
1036 paths = values.get("depot-paths")
1037 if not paths:
1038 paths = values.get("depot-path")
1039 if paths:
1040 values['depot-paths'] = paths.split(',')
1041 return values
1042
1043
1044 def gitBranchExists(branch):
1045 proc = subprocess.Popen(["git", "rev-parse", branch],
1046 stderr=subprocess.PIPE, stdout=subprocess.PIPE)
1047 return proc.wait() == 0
1048
1049
1050 def gitUpdateRef(ref, newvalue):
1051 subprocess.check_call(["git", "update-ref", ref, newvalue])
1052
1053
1054 def gitDeleteRef(ref):
1055 subprocess.check_call(["git", "update-ref", "-d", ref])
1056
1057
1058 _gitConfig = {}
1059
1060
1061 def gitConfig(key, typeSpecifier=None):
1062 if key not in _gitConfig:
1063 cmd = ["git", "config"]
1064 if typeSpecifier:
1065 cmd += [typeSpecifier]
1066 cmd += [key]
1067 s = read_pipe(cmd, ignore_error=True)
1068 _gitConfig[key] = s.strip()
1069 return _gitConfig[key]
1070
1071
1072 def gitConfigBool(key):
1073 """Return a bool, using git config --bool. It is True only if the
1074 variable is set to true, and False if set to false or not present
1075 in the config.
1076 """
1077
1078 if key not in _gitConfig:
1079 _gitConfig[key] = gitConfig(key, '--bool') == "true"
1080 return _gitConfig[key]
1081
1082
1083 def gitConfigInt(key):
1084 if key not in _gitConfig:
1085 cmd = ["git", "config", "--int", key]
1086 s = read_pipe(cmd, ignore_error=True)
1087 v = s.strip()
1088 try:
1089 _gitConfig[key] = int(gitConfig(key, '--int'))
1090 except ValueError:
1091 _gitConfig[key] = None
1092 return _gitConfig[key]
1093
1094
1095 def gitConfigList(key):
1096 if key not in _gitConfig:
1097 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
1098 _gitConfig[key] = s.strip().splitlines()
1099 if _gitConfig[key] == ['']:
1100 _gitConfig[key] = []
1101 return _gitConfig[key]
1102
1103 def fullP4Ref(incomingRef, importIntoRemotes=True):
1104 """Standardize a given provided p4 ref value to a full git ref:
1105 refs/foo/bar/branch -> use it exactly
1106 p4/branch -> prepend refs/remotes/ or refs/heads/
1107 branch -> prepend refs/remotes/p4/ or refs/heads/p4/"""
1108 if incomingRef.startswith("refs/"):
1109 return incomingRef
1110 if importIntoRemotes:
1111 prepend = "refs/remotes/"
1112 else:
1113 prepend = "refs/heads/"
1114 if not incomingRef.startswith("p4/"):
1115 prepend += "p4/"
1116 return prepend + incomingRef
1117
1118 def shortP4Ref(incomingRef, importIntoRemotes=True):
1119 """Standardize to a "short ref" if possible:
1120 refs/foo/bar/branch -> ignore
1121 refs/remotes/p4/branch or refs/heads/p4/branch -> shorten
1122 p4/branch -> shorten"""
1123 if importIntoRemotes:
1124 longprefix = "refs/remotes/p4/"
1125 else:
1126 longprefix = "refs/heads/p4/"
1127 if incomingRef.startswith(longprefix):
1128 return incomingRef[len(longprefix):]
1129 if incomingRef.startswith("p4/"):
1130 return incomingRef[3:]
1131 return incomingRef
1132
1133 def p4BranchesInGit(branchesAreInRemotes=True):
1134 """Find all the branches whose names start with "p4/", looking
1135 in remotes or heads as specified by the argument. Return
1136 a dictionary of { branch: revision } for each one found.
1137 The branch names are the short names, without any
1138 "p4/" prefix.
1139 """
1140
1141 branches = {}
1142
1143 cmdline = ["git", "rev-parse", "--symbolic"]
1144 if branchesAreInRemotes:
1145 cmdline.append("--remotes")
1146 else:
1147 cmdline.append("--branches")
1148
1149 for line in read_pipe_lines(cmdline):
1150 line = line.strip()
1151
1152 # only import to p4/
1153 if not line.startswith('p4/'):
1154 continue
1155 # special symbolic ref to p4/master
1156 if line == "p4/HEAD":
1157 continue
1158
1159 # strip off p4/ prefix
1160 branch = line[len("p4/"):]
1161
1162 branches[branch] = parseRevision(line)
1163
1164 return branches
1165
1166
1167 def branch_exists(branch):
1168 """Make sure that the given ref name really exists."""
1169
1170 cmd = ["git", "rev-parse", "--symbolic", "--verify", branch]
1171 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
1172 out, _ = p.communicate()
1173 out = decode_text_stream(out)
1174 if p.returncode:
1175 return False
1176 # expect exactly one line of output: the branch name
1177 return out.rstrip() == branch
1178
1179
1180 def findUpstreamBranchPoint(head="HEAD"):
1181 branches = p4BranchesInGit()
1182 # map from depot-path to branch name
1183 branchByDepotPath = {}
1184 for branch in branches.keys():
1185 tip = branches[branch]
1186 log = extractLogMessageFromGitCommit(tip)
1187 settings = extractSettingsGitLog(log)
1188 if "depot-paths" in settings:
1189 git_branch = "remotes/p4/" + branch
1190 paths = ",".join(settings["depot-paths"])
1191 branchByDepotPath[paths] = git_branch
1192 if "change" in settings:
1193 paths = paths + ";" + settings["change"]
1194 branchByDepotPath[paths] = git_branch
1195
1196 settings = None
1197 parent = 0
1198 while parent < 65535:
1199 commit = head + "~%s" % parent
1200 log = extractLogMessageFromGitCommit(commit)
1201 settings = extractSettingsGitLog(log)
1202 if "depot-paths" in settings:
1203 paths = ",".join(settings["depot-paths"])
1204 if "change" in settings:
1205 expaths = paths + ";" + settings["change"]
1206 if expaths in branchByDepotPath:
1207 return [branchByDepotPath[expaths], settings]
1208 if paths in branchByDepotPath:
1209 return [branchByDepotPath[paths], settings]
1210
1211 parent = parent + 1
1212
1213 return ["", settings]
1214
1215
1216 def createOrUpdateBranchesFromOrigin(localRefPrefix="refs/remotes/p4/", silent=True):
1217 if not silent:
1218 print("Creating/updating branch(es) in %s based on origin branch(es)"
1219 % localRefPrefix)
1220
1221 originPrefix = "origin/p4/"
1222
1223 for line in read_pipe_lines(["git", "rev-parse", "--symbolic", "--remotes"]):
1224 line = line.strip()
1225 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
1226 continue
1227
1228 headName = line[len(originPrefix):]
1229 remoteHead = localRefPrefix + headName
1230 originHead = line
1231
1232 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
1233 if 'depot-paths' not in original or 'change' not in original:
1234 continue
1235
1236 update = False
1237 if not gitBranchExists(remoteHead):
1238 if verbose:
1239 print("creating %s" % remoteHead)
1240 update = True
1241 else:
1242 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
1243 if 'change' in settings:
1244 if settings['depot-paths'] == original['depot-paths']:
1245 originP4Change = int(original['change'])
1246 p4Change = int(settings['change'])
1247 if originP4Change > p4Change:
1248 print("%s (%s) is newer than %s (%s). "
1249 "Updating p4 branch from origin."
1250 % (originHead, originP4Change,
1251 remoteHead, p4Change))
1252 update = True
1253 else:
1254 print("Ignoring: %s was imported from %s while "
1255 "%s was imported from %s"
1256 % (originHead, ','.join(original['depot-paths']),
1257 remoteHead, ','.join(settings['depot-paths'])))
1258
1259 if update:
1260 system(["git", "update-ref", remoteHead, originHead])
1261
1262
1263 def originP4BranchesExist():
1264 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
1265
1266
1267 def p4ParseNumericChangeRange(parts):
1268 changeStart = int(parts[0][1:])
1269 if parts[1] == '#head':
1270 changeEnd = p4_last_change()
1271 else:
1272 changeEnd = int(parts[1])
1273
1274 return (changeStart, changeEnd)
1275
1276
1277 def chooseBlockSize(blockSize):
1278 if blockSize:
1279 return blockSize
1280 else:
1281 return defaultBlockSize
1282
1283
1284 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
1285 assert depotPaths
1286
1287 # Parse the change range into start and end. Try to find integer
1288 # revision ranges as these can be broken up into blocks to avoid
1289 # hitting server-side limits (maxrows, maxscanresults). But if
1290 # that doesn't work, fall back to using the raw revision specifier
1291 # strings, without using block mode.
1292
1293 if changeRange is None or changeRange == '':
1294 changeStart = 1
1295 changeEnd = p4_last_change()
1296 block_size = chooseBlockSize(requestedBlockSize)
1297 else:
1298 parts = changeRange.split(',')
1299 assert len(parts) == 2
1300 try:
1301 changeStart, changeEnd = p4ParseNumericChangeRange(parts)
1302 block_size = chooseBlockSize(requestedBlockSize)
1303 except ValueError:
1304 changeStart = parts[0][1:]
1305 changeEnd = parts[1]
1306 if requestedBlockSize:
1307 die("cannot use --changes-block-size with non-numeric revisions")
1308 block_size = None
1309
1310 changes = set()
1311
1312 # Retrieve changes a block at a time, to prevent running
1313 # into a MaxResults/MaxScanRows error from the server. If
1314 # we _do_ hit one of those errors, turn down the block size
1315
1316 while True:
1317 cmd = ['changes']
1318
1319 if block_size:
1320 end = min(changeEnd, changeStart + block_size)
1321 revisionRange = "%d,%d" % (changeStart, end)
1322 else:
1323 revisionRange = "%s,%s" % (changeStart, changeEnd)
1324
1325 for p in depotPaths:
1326 cmd += ["%s...@%s" % (p, revisionRange)]
1327
1328 # fetch the changes
1329 try:
1330 result = p4CmdList(cmd, errors_as_exceptions=True)
1331 except P4RequestSizeException as e:
1332 if not block_size:
1333 block_size = e.limit
1334 elif block_size > e.limit:
1335 block_size = e.limit
1336 else:
1337 block_size = max(2, block_size // 2)
1338
1339 if verbose:
1340 print("block size error, retrying with block size {0}".format(block_size))
1341 continue
1342 except P4Exception as e:
1343 die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1344
1345 # Insert changes in chronological order
1346 for entry in reversed(result):
1347 if 'change' not in entry:
1348 continue
1349 changes.add(int(entry['change']))
1350
1351 if not block_size:
1352 break
1353
1354 if end >= changeEnd:
1355 break
1356
1357 changeStart = end + 1
1358
1359 changes = sorted(changes)
1360 return changes
1361
1362
1363 def p4PathStartsWith(path, prefix):
1364 """This method tries to remedy a potential mixed-case issue:
1365
1366 If UserA adds //depot/DirA/file1
1367 and UserB adds //depot/dira/file2
1368
1369 we may or may not have a problem. If you have core.ignorecase=true,
1370 we treat DirA and dira as the same directory.
1371 """
1372 if gitConfigBool("core.ignorecase"):
1373 return path.lower().startswith(prefix.lower())
1374 return path.startswith(prefix)
1375
1376
1377 def getClientSpec():
1378 """Look at the p4 client spec, create a View() object that contains
1379 all the mappings, and return it.
1380 """
1381
1382 specList = p4CmdList(["client", "-o"])
1383 if len(specList) != 1:
1384 die('Output from "client -o" is %d lines, expecting 1' %
1385 len(specList))
1386
1387 # dictionary of all client parameters
1388 entry = specList[0]
1389
1390 # the //client/ name
1391 client_name = entry["Client"]
1392
1393 # just the keys that start with "View"
1394 view_keys = [k for k in entry.keys() if k.startswith("View")]
1395
1396 # hold this new View
1397 view = View(client_name)
1398
1399 # append the lines, in order, to the view
1400 for view_num in range(len(view_keys)):
1401 k = "View%d" % view_num
1402 if k not in view_keys:
1403 die("Expected view key %s missing" % k)
1404 view.append(entry[k])
1405
1406 return view
1407
1408
1409 def getClientRoot():
1410 """Grab the client directory."""
1411
1412 output = p4CmdList(["client", "-o"])
1413 if len(output) != 1:
1414 die('Output from "client -o" is %d lines, expecting 1' % len(output))
1415
1416 entry = output[0]
1417 if "Root" not in entry:
1418 die('Client has no "Root"')
1419
1420 return entry["Root"]
1421
1422
1423 def wildcard_decode(path):
1424 """Decode P4 wildcards into %xx encoding
1425
1426 P4 wildcards are not allowed in filenames. P4 complains if you simply
1427 add them, but you can force it with "-f", in which case it translates
1428 them into %xx encoding internally.
1429 """
1430
1431 # Search for and fix just these four characters. Do % last so
1432 # that fixing it does not inadvertently create new %-escapes.
1433 # Cannot have * in a filename in windows; untested as to
1434 # what p4 would do in such a case.
1435 if not platform.system() == "Windows":
1436 path = path.replace("%2A", "*")
1437 path = path.replace("%23", "#") \
1438 .replace("%40", "@") \
1439 .replace("%25", "%")
1440 return path
1441
1442
1443 def wildcard_encode(path):
1444 """Encode %xx coded wildcards into P4 coding."""
1445
1446 # do % first to avoid double-encoding the %s introduced here
1447 path = path.replace("%", "%25") \
1448 .replace("*", "%2A") \
1449 .replace("#", "%23") \
1450 .replace("@", "%40")
1451 return path
1452
1453
1454 def wildcard_present(path):
1455 m = re.search("[*#@%]", path)
1456 return m is not None
1457
1458
1459 class LargeFileSystem(object):
1460 """Base class for large file system support."""
1461
1462 def __init__(self, writeToGitStream):
1463 self.largeFiles = set()
1464 self.writeToGitStream = writeToGitStream
1465
1466 def generatePointer(self, cloneDestination, contentFile):
1467 """Return the content of a pointer file that is stored in Git instead
1468 of the actual content.
1469 """
1470 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1471
1472 def pushFile(self, localLargeFile):
1473 """Push the actual content which is not stored in the Git repository to
1474 a server.
1475 """
1476 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1477
1478 def hasLargeFileExtension(self, relPath):
1479 return functools.reduce(
1480 lambda a, b: a or b,
1481 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1482 False
1483 )
1484
1485 def generateTempFile(self, contents):
1486 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1487 for d in contents:
1488 contentFile.write(d)
1489 contentFile.close()
1490 return contentFile.name
1491
1492 def exceedsLargeFileThreshold(self, relPath, contents):
1493 if gitConfigInt('git-p4.largeFileThreshold'):
1494 contentsSize = sum(len(d) for d in contents)
1495 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1496 return True
1497 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1498 contentsSize = sum(len(d) for d in contents)
1499 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1500 return False
1501 contentTempFile = self.generateTempFile(contents)
1502 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=True)
1503 with zipfile.ZipFile(compressedContentFile, mode='w') as zf:
1504 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1505 compressedContentsSize = zf.infolist()[0].compress_size
1506 os.remove(contentTempFile)
1507 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1508 return True
1509 return False
1510
1511 def addLargeFile(self, relPath):
1512 self.largeFiles.add(relPath)
1513
1514 def removeLargeFile(self, relPath):
1515 self.largeFiles.remove(relPath)
1516
1517 def isLargeFile(self, relPath):
1518 return relPath in self.largeFiles
1519
1520 def processContent(self, git_mode, relPath, contents):
1521 """Processes the content of git fast import. This method decides if a
1522 file is stored in the large file system and handles all necessary
1523 steps.
1524 """
1525 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1526 contentTempFile = self.generateTempFile(contents)
1527 pointer_git_mode, contents, localLargeFile = self.generatePointer(contentTempFile)
1528 if pointer_git_mode:
1529 git_mode = pointer_git_mode
1530 if localLargeFile:
1531 # Move temp file to final location in large file system
1532 largeFileDir = os.path.dirname(localLargeFile)
1533 if not os.path.isdir(largeFileDir):
1534 os.makedirs(largeFileDir)
1535 shutil.move(contentTempFile, localLargeFile)
1536 self.addLargeFile(relPath)
1537 if gitConfigBool('git-p4.largeFilePush'):
1538 self.pushFile(localLargeFile)
1539 if verbose:
1540 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1541 return (git_mode, contents)
1542
1543
1544 class MockLFS(LargeFileSystem):
1545 """Mock large file system for testing."""
1546
1547 def generatePointer(self, contentFile):
1548 """The pointer content is the original content prefixed with "pointer-".
1549 The local filename of the large file storage is derived from the
1550 file content.
1551 """
1552 with open(contentFile, 'r') as f:
1553 content = next(f)
1554 gitMode = '100644'
1555 pointerContents = 'pointer-' + content
1556 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1557 return (gitMode, pointerContents, localLargeFile)
1558
1559 def pushFile(self, localLargeFile):
1560 """The remote filename of the large file storage is the same as the
1561 local one but in a different directory.
1562 """
1563 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1564 if not os.path.exists(remotePath):
1565 os.makedirs(remotePath)
1566 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1567
1568
1569 class GitLFS(LargeFileSystem):
1570 """Git LFS as backend for the git-p4 large file system.
1571 See https://git-lfs.github.com/ for details.
1572 """
1573
1574 def __init__(self, *args):
1575 LargeFileSystem.__init__(self, *args)
1576 self.baseGitAttributes = []
1577
1578 def generatePointer(self, contentFile):
1579 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1580 mode and content which is stored in the Git repository instead of
1581 the actual content. Return also the new location of the actual
1582 content.
1583 """
1584 if os.path.getsize(contentFile) == 0:
1585 return (None, '', None)
1586
1587 pointerProcess = subprocess.Popen(
1588 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1589 stdout=subprocess.PIPE
1590 )
1591 pointerFile = decode_text_stream(pointerProcess.stdout.read())
1592 if pointerProcess.wait():
1593 os.remove(contentFile)
1594 die('git-lfs pointer command failed. Did you install the extension?')
1595
1596 # Git LFS removed the preamble in the output of the 'pointer' command
1597 # starting from version 1.2.0. Check for the preamble here to support
1598 # earlier versions.
1599 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1600 if pointerFile.startswith('Git LFS pointer for'):
1601 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1602
1603 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1604 # if someone use external lfs.storage ( not in local repo git )
1605 lfs_path = gitConfig('lfs.storage')
1606 if not lfs_path:
1607 lfs_path = 'lfs'
1608 if not os.path.isabs(lfs_path):
1609 lfs_path = os.path.join(os.getcwd(), '.git', lfs_path)
1610 localLargeFile = os.path.join(
1611 lfs_path,
1612 'objects', oid[:2], oid[2:4],
1613 oid,
1614 )
1615 # LFS Spec states that pointer files should not have the executable bit set.
1616 gitMode = '100644'
1617 return (gitMode, pointerFile, localLargeFile)
1618
1619 def pushFile(self, localLargeFile):
1620 uploadProcess = subprocess.Popen(
1621 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1622 )
1623 if uploadProcess.wait():
1624 die('git-lfs push command failed. Did you define a remote?')
1625
1626 def generateGitAttributes(self):
1627 return (
1628 self.baseGitAttributes +
1629 [
1630 '\n',
1631 '#\n',
1632 '# Git LFS (see https://git-lfs.github.com/)\n',
1633 '#\n',
1634 ] +
1635 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1636 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1637 ] +
1638 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1639 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1640 ]
1641 )
1642
1643 def addLargeFile(self, relPath):
1644 LargeFileSystem.addLargeFile(self, relPath)
1645 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1646
1647 def removeLargeFile(self, relPath):
1648 LargeFileSystem.removeLargeFile(self, relPath)
1649 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1650
1651 def processContent(self, git_mode, relPath, contents):
1652 if relPath == '.gitattributes':
1653 self.baseGitAttributes = contents
1654 return (git_mode, self.generateGitAttributes())
1655 else:
1656 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1657
1658
1659 class Command:
1660 delete_actions = ("delete", "move/delete", "purge")
1661 add_actions = ("add", "branch", "move/add")
1662
1663 def __init__(self):
1664 self.usage = "usage: %prog [options]"
1665 self.needsGit = True
1666 self.verbose = False
1667
1668 # This is required for the "append" update_shelve action
1669 def ensure_value(self, attr, value):
1670 if not hasattr(self, attr) or getattr(self, attr) is None:
1671 setattr(self, attr, value)
1672 return getattr(self, attr)
1673
1674
1675 class P4UserMap:
1676 def __init__(self):
1677 self.userMapFromPerforceServer = False
1678 self.myP4UserId = None
1679
1680 def p4UserId(self):
1681 if self.myP4UserId:
1682 return self.myP4UserId
1683
1684 results = p4CmdList(["user", "-o"])
1685 for r in results:
1686 if 'User' in r:
1687 self.myP4UserId = r['User']
1688 return r['User']
1689 die("Could not find your p4 user id")
1690
1691 def p4UserIsMe(self, p4User):
1692 """Return True if the given p4 user is actually me."""
1693 me = self.p4UserId()
1694 if not p4User or p4User != me:
1695 return False
1696 else:
1697 return True
1698
1699 def getUserCacheFilename(self):
1700 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1701 return home + "/.gitp4-usercache.txt"
1702
1703 def getUserMapFromPerforceServer(self):
1704 if self.userMapFromPerforceServer:
1705 return
1706 self.users = {}
1707 self.emails = {}
1708
1709 for output in p4CmdList(["users"]):
1710 if "User" not in output:
1711 continue
1712 # "FullName" is bytes. "Email" on the other hand might be bytes
1713 # or unicode string depending on whether we are running under
1714 # python2 or python3. To support
1715 # git-p4.metadataDecodingStrategy=fallback, self.users dict values
1716 # are always bytes, ready to be written to git.
1717 emailbytes = metadata_stream_to_writable_bytes(output["Email"])
1718 self.users[output["User"]] = output["FullName"] + b" <" + emailbytes + b">"
1719 self.emails[output["Email"]] = output["User"]
1720
1721 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1722 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1723 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1724 if mapUser and len(mapUser[0]) == 3:
1725 user = mapUser[0][0]
1726 fullname = mapUser[0][1]
1727 email = mapUser[0][2]
1728 fulluser = fullname + " <" + email + ">"
1729 self.users[user] = metadata_stream_to_writable_bytes(fulluser)
1730 self.emails[email] = user
1731
1732 s = b''
1733 for (key, val) in self.users.items():
1734 keybytes = metadata_stream_to_writable_bytes(key)
1735 s += b"%s\t%s\n" % (keybytes.expandtabs(1), val.expandtabs(1))
1736
1737 open(self.getUserCacheFilename(), 'wb').write(s)
1738 self.userMapFromPerforceServer = True
1739
1740 def loadUserMapFromCache(self):
1741 self.users = {}
1742 self.userMapFromPerforceServer = False
1743 try:
1744 cache = open(self.getUserCacheFilename(), 'rb')
1745 lines = cache.readlines()
1746 cache.close()
1747 for line in lines:
1748 entry = line.strip().split(b"\t")
1749 self.users[entry[0].decode('utf_8')] = entry[1]
1750 except IOError:
1751 self.getUserMapFromPerforceServer()
1752
1753
1754 class P4Submit(Command, P4UserMap):
1755
1756 conflict_behavior_choices = ("ask", "skip", "quit")
1757
1758 def __init__(self):
1759 Command.__init__(self)
1760 P4UserMap.__init__(self)
1761 self.options = [
1762 optparse.make_option("--origin", dest="origin"),
1763 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1764 # preserve the user, requires relevant p4 permissions
1765 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1766 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1767 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1768 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1769 optparse.make_option("--conflict", dest="conflict_behavior",
1770 choices=self.conflict_behavior_choices),
1771 optparse.make_option("--branch", dest="branch"),
1772 optparse.make_option("--shelve", dest="shelve", action="store_true",
1773 help="Shelve instead of submit. Shelved files are reverted, "
1774 "restoring the workspace to the state before the shelve"),
1775 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1776 metavar="CHANGELIST",
1777 help="update an existing shelved changelist, implies --shelve, "
1778 "repeat in-order for multiple shelved changelists"),
1779 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1780 help="submit only the specified commit(s), one commit or xxx..xxx"),
1781 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1782 help="Disable rebase after submit is completed. Can be useful if you "
1783 "work from a local git branch that is not master"),
1784 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1785 help="Skip Perforce sync of p4/master after submit or shelve"),
1786 optparse.make_option("--no-verify", dest="no_verify", action="store_true",
1787 help="Bypass p4-pre-submit and p4-changelist hooks"),
1788 ]
1789 self.description = """Submit changes from git to the perforce depot.\n
1790 The `p4-pre-submit` hook is executed if it exists and is executable. It
1791 can be bypassed with the `--no-verify` command line option. The hook takes
1792 no parameters and nothing from standard input. Exiting with a non-zero status
1793 from this script prevents `git-p4 submit` from launching.
1794
1795 One usage scenario is to run unit tests in the hook.
1796
1797 The `p4-prepare-changelist` hook is executed right after preparing the default
1798 changelist message and before the editor is started. It takes one parameter,
1799 the name of the file that contains the changelist text. Exiting with a non-zero
1800 status from the script will abort the process.
1801
1802 The purpose of the hook is to edit the message file in place, and it is not
1803 supressed by the `--no-verify` option. This hook is called even if
1804 `--prepare-p4-only` is set.
1805
1806 The `p4-changelist` hook is executed after the changelist message has been
1807 edited by the user. It can be bypassed with the `--no-verify` option. It
1808 takes a single parameter, the name of the file that holds the proposed
1809 changelist text. Exiting with a non-zero status causes the command to abort.
1810
1811 The hook is allowed to edit the changelist file and can be used to normalize
1812 the text into some project standard format. It can also be used to refuse the
1813 Submit after inspect the message file.
1814
1815 The `p4-post-changelist` hook is invoked after the submit has successfully
1816 occurred in P4. It takes no parameters and is meant primarily for notification
1817 and cannot affect the outcome of the git p4 submit action.
1818 """
1819
1820 self.usage += " [name of git branch to submit into perforce depot]"
1821 self.origin = ""
1822 self.detectRenames = False
1823 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1824 self.dry_run = False
1825 self.shelve = False
1826 self.update_shelve = list()
1827 self.commit = ""
1828 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1829 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1830 self.prepare_p4_only = False
1831 self.conflict_behavior = None
1832 self.isWindows = (platform.system() == "Windows")
1833 self.exportLabels = False
1834 self.p4HasMoveCommand = p4_has_move_command()
1835 self.branch = None
1836 self.no_verify = False
1837
1838 if gitConfig('git-p4.largeFileSystem'):
1839 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1840
1841 def check(self):
1842 if len(p4CmdList(["opened", "..."])) > 0:
1843 die("You have files opened with perforce! Close them before starting the sync.")
1844
1845 def separate_jobs_from_description(self, message):
1846 """Extract and return a possible Jobs field in the commit message. It
1847 goes into a separate section in the p4 change specification.
1848
1849 A jobs line starts with "Jobs:" and looks like a new field in a
1850 form. Values are white-space separated on the same line or on
1851 following lines that start with a tab.
1852
1853 This does not parse and extract the full git commit message like a
1854 p4 form. It just sees the Jobs: line as a marker to pass everything
1855 from then on directly into the p4 form, but outside the description
1856 section.
1857
1858 Return a tuple (stripped log message, jobs string).
1859 """
1860
1861 m = re.search(r'^Jobs:', message, re.MULTILINE)
1862 if m is None:
1863 return (message, None)
1864
1865 jobtext = message[m.start():]
1866 stripped_message = message[:m.start()].rstrip()
1867 return (stripped_message, jobtext)
1868
1869 def prepareLogMessage(self, template, message, jobs):
1870 """Edits the template returned from "p4 change -o" to insert the
1871 message in the Description field, and the jobs text in the Jobs
1872 field.
1873 """
1874 result = ""
1875
1876 inDescriptionSection = False
1877
1878 for line in template.split("\n"):
1879 if line.startswith("#"):
1880 result += line + "\n"
1881 continue
1882
1883 if inDescriptionSection:
1884 if line.startswith("Files:") or line.startswith("Jobs:"):
1885 inDescriptionSection = False
1886 # insert Jobs section
1887 if jobs:
1888 result += jobs + "\n"
1889 else:
1890 continue
1891 else:
1892 if line.startswith("Description:"):
1893 inDescriptionSection = True
1894 line += "\n"
1895 for messageLine in message.split("\n"):
1896 line += "\t" + messageLine + "\n"
1897
1898 result += line + "\n"
1899
1900 return result
1901
1902 def patchRCSKeywords(self, file, regexp):
1903 """Attempt to zap the RCS keywords in a p4 controlled file matching the
1904 given regex.
1905 """
1906 handle, outFileName = tempfile.mkstemp(dir='.')
1907 try:
1908 with os.fdopen(handle, "wb") as outFile, open(file, "rb") as inFile:
1909 for line in inFile.readlines():
1910 outFile.write(regexp.sub(br'$\1$', line))
1911 # Forcibly overwrite the original file
1912 os.unlink(file)
1913 shutil.move(outFileName, file)
1914 except:
1915 # cleanup our temporary file
1916 os.unlink(outFileName)
1917 print("Failed to strip RCS keywords in %s" % file)
1918 raise
1919
1920 print("Patched up RCS keywords in %s" % file)
1921
1922 def p4UserForCommit(self, id):
1923 """Return the tuple (perforce user,git email) for a given git commit
1924 id.
1925 """
1926 self.getUserMapFromPerforceServer()
1927 gitEmail = read_pipe(["git", "log", "--max-count=1",
1928 "--format=%ae", id])
1929 gitEmail = gitEmail.strip()
1930 if gitEmail not in self.emails:
1931 return (None, gitEmail)
1932 else:
1933 return (self.emails[gitEmail], gitEmail)
1934
1935 def checkValidP4Users(self, commits):
1936 """Check if any git authors cannot be mapped to p4 users."""
1937 for id in commits:
1938 user, email = self.p4UserForCommit(id)
1939 if not user:
1940 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1941 if gitConfigBool("git-p4.allowMissingP4Users"):
1942 print("%s" % msg)
1943 else:
1944 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1945
1946 def lastP4Changelist(self):
1947 """Get back the last changelist number submitted in this client spec.
1948
1949 This then gets used to patch up the username in the change. If the
1950 same client spec is being used by multiple processes then this might
1951 go wrong.
1952 """
1953 results = p4CmdList(["client", "-o"]) # find the current client
1954 client = None
1955 for r in results:
1956 if 'Client' in r:
1957 client = r['Client']
1958 break
1959 if not client:
1960 die("could not get client spec")
1961 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1962 for r in results:
1963 if 'change' in r:
1964 return r['change']
1965 die("Could not get changelist number for last submit - cannot patch up user details")
1966
1967 def modifyChangelistUser(self, changelist, newUser):
1968 """Fixup the user field of a changelist after it has been submitted."""
1969 changes = p4CmdList(["change", "-o", changelist])
1970 if len(changes) != 1:
1971 die("Bad output from p4 change modifying %s to user %s" %
1972 (changelist, newUser))
1973
1974 c = changes[0]
1975 if c['User'] == newUser:
1976 # Nothing to do
1977 return
1978 c['User'] = newUser
1979 # p4 does not understand format version 3 and above
1980 input = marshal.dumps(c, 2)
1981
1982 result = p4CmdList(["change", "-f", "-i"], stdin=input)
1983 for r in result:
1984 if 'code' in r:
1985 if r['code'] == 'error':
1986 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1987 if 'data' in r:
1988 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1989 return
1990 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1991
1992 def canChangeChangelists(self):
1993 """Check to see if we have p4 admin or super-user permissions, either
1994 of which are required to modify changelists.
1995 """
1996 results = p4CmdList(["protects", self.depotPath])
1997 for r in results:
1998 if 'perm' in r:
1999 if r['perm'] == 'admin':
2000 return 1
2001 if r['perm'] == 'super':
2002 return 1
2003 return 0
2004
2005 def prepareSubmitTemplate(self, changelist=None):
2006 """Run "p4 change -o" to grab a change specification template.
2007
2008 This does not use "p4 -G", as it is nice to keep the submission
2009 template in original order, since a human might edit it.
2010
2011 Remove lines in the Files section that show changes to files
2012 outside the depot path we're committing into.
2013 """
2014
2015 upstream, settings = findUpstreamBranchPoint()
2016
2017 template = """\
2018 # A Perforce Change Specification.
2019 #
2020 # Change: The change number. 'new' on a new changelist.
2021 # Date: The date this specification was last modified.
2022 # Client: The client on which the changelist was created. Read-only.
2023 # User: The user who created the changelist.
2024 # Status: Either 'pending' or 'submitted'. Read-only.
2025 # Type: Either 'public' or 'restricted'. Default is 'public'.
2026 # Description: Comments about the changelist. Required.
2027 # Jobs: What opened jobs are to be closed by this changelist.
2028 # You may delete jobs from this list. (New changelists only.)
2029 # Files: What opened files from the default changelist are to be added
2030 # to this changelist. You may delete files from this list.
2031 # (New changelists only.)
2032 """
2033 files_list = []
2034 inFilesSection = False
2035 change_entry = None
2036 args = ['change', '-o']
2037 if changelist:
2038 args.append(str(changelist))
2039 for entry in p4CmdList(args):
2040 if 'code' not in entry:
2041 continue
2042 if entry['code'] == 'stat':
2043 change_entry = entry
2044 break
2045 if not change_entry:
2046 die('Failed to decode output of p4 change -o')
2047 for key, value in change_entry.items():
2048 if key.startswith('File'):
2049 if 'depot-paths' in settings:
2050 if not [p for p in settings['depot-paths']
2051 if p4PathStartsWith(value, p)]:
2052 continue
2053 else:
2054 if not p4PathStartsWith(value, self.depotPath):
2055 continue
2056 files_list.append(value)
2057 continue
2058 # Output in the order expected by prepareLogMessage
2059 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
2060 if key not in change_entry:
2061 continue
2062 template += '\n'
2063 template += key + ':'
2064 if key == 'Description':
2065 template += '\n'
2066 for field_line in change_entry[key].splitlines():
2067 template += '\t'+field_line+'\n'
2068 if len(files_list) > 0:
2069 template += '\n'
2070 template += 'Files:\n'
2071 for path in files_list:
2072 template += '\t'+path+'\n'
2073 return template
2074
2075 def edit_template(self, template_file):
2076 """Invoke the editor to let the user change the submission message.
2077
2078 Return true if okay to continue with the submit.
2079 """
2080
2081 # if configured to skip the editing part, just submit
2082 if gitConfigBool("git-p4.skipSubmitEdit"):
2083 return True
2084
2085 # look at the modification time, to check later if the user saved
2086 # the file
2087 mtime = os.stat(template_file).st_mtime
2088
2089 # invoke the editor
2090 if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
2091 editor = os.environ.get("P4EDITOR")
2092 else:
2093 editor = read_pipe(["git", "var", "GIT_EDITOR"]).strip()
2094 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
2095
2096 # If the file was not saved, prompt to see if this patch should
2097 # be skipped. But skip this verification step if configured so.
2098 if gitConfigBool("git-p4.skipSubmitEditCheck"):
2099 return True
2100
2101 # modification time updated means user saved the file
2102 if os.stat(template_file).st_mtime > mtime:
2103 return True
2104
2105 response = prompt("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
2106 if response == 'y':
2107 return True
2108 if response == 'n':
2109 return False
2110
2111 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
2112 # diff
2113 if "P4DIFF" in os.environ:
2114 del(os.environ["P4DIFF"])
2115 diff = ""
2116 for editedFile in editedFiles:
2117 diff += p4_read_pipe(['diff', '-du',
2118 wildcard_encode(editedFile)])
2119
2120 # new file diff
2121 newdiff = ""
2122 for newFile in filesToAdd:
2123 newdiff += "==== new file ====\n"
2124 newdiff += "--- /dev/null\n"
2125 newdiff += "+++ %s\n" % newFile
2126
2127 is_link = os.path.islink(newFile)
2128 expect_link = newFile in symlinks
2129
2130 if is_link and expect_link:
2131 newdiff += "+%s\n" % os.readlink(newFile)
2132 else:
2133 f = open(newFile, "r")
2134 try:
2135 for line in f.readlines():
2136 newdiff += "+" + line
2137 except UnicodeDecodeError:
2138 # Found non-text data and skip, since diff description
2139 # should only include text
2140 pass
2141 f.close()
2142
2143 return (diff + newdiff).replace('\r\n', '\n')
2144
2145 def applyCommit(self, id):
2146 """Apply one commit, return True if it succeeded."""
2147
2148 print("Applying", read_pipe(["git", "show", "-s",
2149 "--format=format:%h %s", id]))
2150
2151 p4User, gitEmail = self.p4UserForCommit(id)
2152
2153 diff = read_pipe_lines(
2154 ["git", "diff-tree", "-r"] + self.diffOpts + ["{}^".format(id), id])
2155 filesToAdd = set()
2156 filesToChangeType = set()
2157 filesToDelete = set()
2158 editedFiles = set()
2159 pureRenameCopy = set()
2160 symlinks = set()
2161 filesToChangeExecBit = {}
2162 all_files = list()
2163
2164 for line in diff:
2165 diff = parseDiffTreeEntry(line)
2166 modifier = diff['status']
2167 path = diff['src']
2168 all_files.append(path)
2169
2170 if modifier == "M":
2171 p4_edit(path)
2172 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
2173 filesToChangeExecBit[path] = diff['dst_mode']
2174 editedFiles.add(path)
2175 elif modifier == "A":
2176 filesToAdd.add(path)
2177 filesToChangeExecBit[path] = diff['dst_mode']
2178 if path in filesToDelete:
2179 filesToDelete.remove(path)
2180
2181 dst_mode = int(diff['dst_mode'], 8)
2182 if dst_mode == 0o120000:
2183 symlinks.add(path)
2184
2185 elif modifier == "D":
2186 filesToDelete.add(path)
2187 if path in filesToAdd:
2188 filesToAdd.remove(path)
2189 elif modifier == "C":
2190 src, dest = diff['src'], diff['dst']
2191 all_files.append(dest)
2192 p4_integrate(src, dest)
2193 pureRenameCopy.add(dest)
2194 if diff['src_sha1'] != diff['dst_sha1']:
2195 p4_edit(dest)
2196 pureRenameCopy.discard(dest)
2197 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
2198 p4_edit(dest)
2199 pureRenameCopy.discard(dest)
2200 filesToChangeExecBit[dest] = diff['dst_mode']
2201 if self.isWindows:
2202 # turn off read-only attribute
2203 os.chmod(dest, stat.S_IWRITE)
2204 os.unlink(dest)
2205 editedFiles.add(dest)
2206 elif modifier == "R":
2207 src, dest = diff['src'], diff['dst']
2208 all_files.append(dest)
2209 if self.p4HasMoveCommand:
2210 p4_edit(src) # src must be open before move
2211 p4_move(src, dest) # opens for (move/delete, move/add)
2212 else:
2213 p4_integrate(src, dest)
2214 if diff['src_sha1'] != diff['dst_sha1']:
2215 p4_edit(dest)
2216 else:
2217 pureRenameCopy.add(dest)
2218 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
2219 if not self.p4HasMoveCommand:
2220 p4_edit(dest) # with move: already open, writable
2221 filesToChangeExecBit[dest] = diff['dst_mode']
2222 if not self.p4HasMoveCommand:
2223 if self.isWindows:
2224 os.chmod(dest, stat.S_IWRITE)
2225 os.unlink(dest)
2226 filesToDelete.add(src)
2227 editedFiles.add(dest)
2228 elif modifier == "T":
2229 filesToChangeType.add(path)
2230 else:
2231 die("unknown modifier %s for %s" % (modifier, path))
2232
2233 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
2234 patchcmd = diffcmd + " | git apply "
2235 tryPatchCmd = patchcmd + "--check -"
2236 applyPatchCmd = patchcmd + "--check --apply -"
2237 patch_succeeded = True
2238
2239 if verbose:
2240 print("TryPatch: %s" % tryPatchCmd)
2241
2242 if os.system(tryPatchCmd) != 0:
2243 fixed_rcs_keywords = False
2244 patch_succeeded = False
2245 print("Unfortunately applying the change failed!")
2246
2247 # Patch failed, maybe it's just RCS keyword woes. Look through
2248 # the patch to see if that's possible.
2249 if gitConfigBool("git-p4.attemptRCSCleanup"):
2250 file = None
2251 kwfiles = {}
2252 for file in editedFiles | filesToDelete:
2253 # did this file's delta contain RCS keywords?
2254 regexp = p4_keywords_regexp_for_file(file)
2255 if regexp:
2256 # this file is a possibility...look for RCS keywords.
2257 for line in read_pipe_lines(
2258 ["git", "diff", "%s^..%s" % (id, id), file],
2259 raw=True):
2260 if regexp.search(line):
2261 if verbose:
2262 print("got keyword match on %s in %s in %s" % (regexp.pattern, line, file))
2263 kwfiles[file] = regexp
2264 break
2265
2266 for file, regexp in kwfiles.items():
2267 if verbose:
2268 print("zapping %s with %s" % (line, regexp.pattern))
2269 # File is being deleted, so not open in p4. Must
2270 # disable the read-only bit on windows.
2271 if self.isWindows and file not in editedFiles:
2272 os.chmod(file, stat.S_IWRITE)
2273 self.patchRCSKeywords(file, kwfiles[file])
2274 fixed_rcs_keywords = True
2275
2276 if fixed_rcs_keywords:
2277 print("Retrying the patch with RCS keywords cleaned up")
2278 if os.system(tryPatchCmd) == 0:
2279 patch_succeeded = True
2280 print("Patch succeesed this time with RCS keywords cleaned")
2281
2282 if not patch_succeeded:
2283 for f in editedFiles:
2284 p4_revert(f)
2285 return False
2286
2287 #
2288 # Apply the patch for real, and do add/delete/+x handling.
2289 #
2290 system(applyPatchCmd, shell=True)
2291
2292 for f in filesToChangeType:
2293 p4_edit(f, "-t", "auto")
2294 for f in filesToAdd:
2295 p4_add(f)
2296 for f in filesToDelete:
2297 p4_revert(f)
2298 p4_delete(f)
2299
2300 # Set/clear executable bits
2301 for f in filesToChangeExecBit.keys():
2302 mode = filesToChangeExecBit[f]
2303 setP4ExecBit(f, mode)
2304
2305 update_shelve = 0
2306 if len(self.update_shelve) > 0:
2307 update_shelve = self.update_shelve.pop(0)
2308 p4_reopen_in_change(update_shelve, all_files)
2309
2310 #
2311 # Build p4 change description, starting with the contents
2312 # of the git commit message.
2313 #
2314 logMessage = extractLogMessageFromGitCommit(id)
2315 logMessage = logMessage.strip()
2316 logMessage, jobs = self.separate_jobs_from_description(logMessage)
2317
2318 template = self.prepareSubmitTemplate(update_shelve)
2319 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
2320
2321 if self.preserveUser:
2322 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
2323
2324 if self.checkAuthorship and not self.p4UserIsMe(p4User):
2325 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
2326 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
2327 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
2328
2329 separatorLine = "######## everything below this line is just the diff #######\n"
2330 if not self.prepare_p4_only:
2331 submitTemplate += separatorLine
2332 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
2333
2334 handle, fileName = tempfile.mkstemp()
2335 tmpFile = os.fdopen(handle, "w+b")
2336 if self.isWindows:
2337 submitTemplate = submitTemplate.replace("\n", "\r\n")
2338 tmpFile.write(encode_text_stream(submitTemplate))
2339 tmpFile.close()
2340
2341 submitted = False
2342
2343 try:
2344 # Allow the hook to edit the changelist text before presenting it
2345 # to the user.
2346 if not run_git_hook("p4-prepare-changelist", [fileName]):
2347 return False
2348
2349 if self.prepare_p4_only:
2350 #
2351 # Leave the p4 tree prepared, and the submit template around
2352 # and let the user decide what to do next
2353 #
2354 submitted = True
2355 print("")
2356 print("P4 workspace prepared for submission.")
2357 print("To submit or revert, go to client workspace")
2358 print(" " + self.clientPath)
2359 print("")
2360 print("To submit, use \"p4 submit\" to write a new description,")
2361 print("or \"p4 submit -i <%s\" to use the one prepared by"
2362 " \"git p4\"." % fileName)
2363 print("You can delete the file \"%s\" when finished." % fileName)
2364
2365 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2366 print("To preserve change ownership by user %s, you must\n"
2367 "do \"p4 change -f <change>\" after submitting and\n"
2368 "edit the User field.")
2369 if pureRenameCopy:
2370 print("After submitting, renamed files must be re-synced.")
2371 print("Invoke \"p4 sync -f\" on each of these files:")
2372 for f in pureRenameCopy:
2373 print(" " + f)
2374
2375 print("")
2376 print("To revert the changes, use \"p4 revert ...\", and delete")
2377 print("the submit template file \"%s\"" % fileName)
2378 if filesToAdd:
2379 print("Since the commit adds new files, they must be deleted:")
2380 for f in filesToAdd:
2381 print(" " + f)
2382 print("")
2383 sys.stdout.flush()
2384 return True
2385
2386 if self.edit_template(fileName):
2387 if not self.no_verify:
2388 if not run_git_hook("p4-changelist", [fileName]):
2389 print("The p4-changelist hook failed.")
2390 sys.stdout.flush()
2391 return False
2392
2393 # read the edited message and submit
2394 tmpFile = open(fileName, "rb")
2395 message = decode_text_stream(tmpFile.read())
2396 tmpFile.close()
2397 if self.isWindows:
2398 message = message.replace("\r\n", "\n")
2399 if message.find(separatorLine) != -1:
2400 submitTemplate = message[:message.index(separatorLine)]
2401 else:
2402 submitTemplate = message
2403
2404 if len(submitTemplate.strip()) == 0:
2405 print("Changelist is empty, aborting this changelist.")
2406 sys.stdout.flush()
2407 return False
2408
2409 if update_shelve:
2410 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2411 elif self.shelve:
2412 p4_write_pipe(['shelve', '-i'], submitTemplate)
2413 else:
2414 p4_write_pipe(['submit', '-i'], submitTemplate)
2415 # The rename/copy happened by applying a patch that created a
2416 # new file. This leaves it writable, which confuses p4.
2417 for f in pureRenameCopy:
2418 p4_sync(f, "-f")
2419
2420 if self.preserveUser:
2421 if p4User:
2422 # Get last changelist number. Cannot easily get it from
2423 # the submit command output as the output is
2424 # unmarshalled.
2425 changelist = self.lastP4Changelist()
2426 self.modifyChangelistUser(changelist, p4User)
2427
2428 submitted = True
2429
2430 run_git_hook("p4-post-changelist")
2431 finally:
2432 # Revert changes if we skip this patch
2433 if not submitted or self.shelve:
2434 if self.shelve:
2435 print("Reverting shelved files.")
2436 else:
2437 print("Submission cancelled, undoing p4 changes.")
2438 sys.stdout.flush()
2439 for f in editedFiles | filesToDelete:
2440 p4_revert(f)
2441 for f in filesToAdd:
2442 p4_revert(f)
2443 os.remove(f)
2444
2445 if not self.prepare_p4_only:
2446 os.remove(fileName)
2447 return submitted
2448
2449 def exportGitTags(self, gitTags):
2450 """Export git tags as p4 labels. Create a p4 label and then tag with
2451 that.
2452 """
2453
2454 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2455 if len(validLabelRegexp) == 0:
2456 validLabelRegexp = defaultLabelRegexp
2457 m = re.compile(validLabelRegexp)
2458
2459 for name in gitTags:
2460
2461 if not m.match(name):
2462 if verbose:
2463 print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2464 continue
2465
2466 # Get the p4 commit this corresponds to
2467 logMessage = extractLogMessageFromGitCommit(name)
2468 values = extractSettingsGitLog(logMessage)
2469
2470 if 'change' not in values:
2471 # a tag pointing to something not sent to p4; ignore
2472 if verbose:
2473 print("git tag %s does not give a p4 commit" % name)
2474 continue
2475 else:
2476 changelist = values['change']
2477
2478 # Get the tag details.
2479 inHeader = True
2480 isAnnotated = False
2481 body = []
2482 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2483 l = l.strip()
2484 if inHeader:
2485 if re.match(r'tag\s+', l):
2486 isAnnotated = True
2487 elif re.match(r'\s*$', l):
2488 inHeader = False
2489 continue
2490 else:
2491 body.append(l)
2492
2493 if not isAnnotated:
2494 body = ["lightweight tag imported by git p4\n"]
2495
2496 # Create the label - use the same view as the client spec we are using
2497 clientSpec = getClientSpec()
2498
2499 labelTemplate = "Label: %s\n" % name
2500 labelTemplate += "Description:\n"
2501 for b in body:
2502 labelTemplate += "\t" + b + "\n"
2503 labelTemplate += "View:\n"
2504 for depot_side in clientSpec.mappings:
2505 labelTemplate += "\t%s\n" % depot_side
2506
2507 if self.dry_run:
2508 print("Would create p4 label %s for tag" % name)
2509 elif self.prepare_p4_only:
2510 print("Not creating p4 label %s for tag due to option"
2511 " --prepare-p4-only" % name)
2512 else:
2513 p4_write_pipe(["label", "-i"], labelTemplate)
2514
2515 # Use the label
2516 p4_system(["tag", "-l", name] +
2517 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2518
2519 if verbose:
2520 print("created p4 label for tag %s" % name)
2521
2522 def run(self, args):
2523 if len(args) == 0:
2524 self.master = currentGitBranch()
2525 elif len(args) == 1:
2526 self.master = args[0]
2527 if not branchExists(self.master):
2528 die("Branch %s does not exist" % self.master)
2529 else:
2530 return False
2531
2532 for i in self.update_shelve:
2533 if i <= 0:
2534 sys.exit("invalid changelist %d" % i)
2535
2536 if self.master:
2537 allowSubmit = gitConfig("git-p4.allowSubmit")
2538 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2539 die("%s is not in git-p4.allowSubmit" % self.master)
2540
2541 upstream, settings = findUpstreamBranchPoint()
2542 self.depotPath = settings['depot-paths'][0]
2543 if len(self.origin) == 0:
2544 self.origin = upstream
2545
2546 if len(self.update_shelve) > 0:
2547 self.shelve = True
2548
2549 if self.preserveUser:
2550 if not self.canChangeChangelists():
2551 die("Cannot preserve user names without p4 super-user or admin permissions")
2552
2553 # if not set from the command line, try the config file
2554 if self.conflict_behavior is None:
2555 val = gitConfig("git-p4.conflict")
2556 if val:
2557 if val not in self.conflict_behavior_choices:
2558 die("Invalid value '%s' for config git-p4.conflict" % val)
2559 else:
2560 val = "ask"
2561 self.conflict_behavior = val
2562
2563 if self.verbose:
2564 print("Origin branch is " + self.origin)
2565
2566 if len(self.depotPath) == 0:
2567 print("Internal error: cannot locate perforce depot path from existing branches")
2568 sys.exit(128)
2569
2570 self.useClientSpec = False
2571 if gitConfigBool("git-p4.useclientspec"):
2572 self.useClientSpec = True
2573 if self.useClientSpec:
2574 self.clientSpecDirs = getClientSpec()
2575
2576 # Check for the existence of P4 branches
2577 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2578
2579 if self.useClientSpec and not branchesDetected:
2580 # all files are relative to the client spec
2581 self.clientPath = getClientRoot()
2582 else:
2583 self.clientPath = p4Where(self.depotPath)
2584
2585 if self.clientPath == "":
2586 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2587
2588 print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2589 self.oldWorkingDirectory = os.getcwd()
2590
2591 # ensure the clientPath exists
2592 new_client_dir = False
2593 if not os.path.exists(self.clientPath):
2594 new_client_dir = True
2595 os.makedirs(self.clientPath)
2596
2597 chdir(self.clientPath, is_client_path=True)
2598 if self.dry_run:
2599 print("Would synchronize p4 checkout in %s" % self.clientPath)
2600 else:
2601 print("Synchronizing p4 checkout...")
2602 if new_client_dir:
2603 # old one was destroyed, and maybe nobody told p4
2604 p4_sync("...", "-f")
2605 else:
2606 p4_sync("...")
2607 self.check()
2608
2609 commits = []
2610 if self.master:
2611 committish = self.master
2612 else:
2613 committish = 'HEAD'
2614
2615 if self.commit != "":
2616 if self.commit.find("..") != -1:
2617 limits_ish = self.commit.split("..")
2618 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2619 commits.append(line.strip())
2620 commits.reverse()
2621 else:
2622 commits.append(self.commit)
2623 else:
2624 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2625 commits.append(line.strip())
2626 commits.reverse()
2627
2628 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2629 self.checkAuthorship = False
2630 else:
2631 self.checkAuthorship = True
2632
2633 if self.preserveUser:
2634 self.checkValidP4Users(commits)
2635
2636 #
2637 # Build up a set of options to be passed to diff when
2638 # submitting each commit to p4.
2639 #
2640 if self.detectRenames:
2641 # command-line -M arg
2642 self.diffOpts = ["-M"]
2643 else:
2644 # If not explicitly set check the config variable
2645 detectRenames = gitConfig("git-p4.detectRenames")
2646
2647 if detectRenames.lower() == "false" or detectRenames == "":
2648 self.diffOpts = []
2649 elif detectRenames.lower() == "true":
2650 self.diffOpts = ["-M"]
2651 else:
2652 self.diffOpts = ["-M{}".format(detectRenames)]
2653
2654 # no command-line arg for -C or --find-copies-harder, just
2655 # config variables
2656 detectCopies = gitConfig("git-p4.detectCopies")
2657 if detectCopies.lower() == "false" or detectCopies == "":
2658 pass
2659 elif detectCopies.lower() == "true":
2660 self.diffOpts.append("-C")
2661 else:
2662 self.diffOpts.append("-C{}".format(detectCopies))
2663
2664 if gitConfigBool("git-p4.detectCopiesHarder"):
2665 self.diffOpts.append("--find-copies-harder")
2666
2667 num_shelves = len(self.update_shelve)
2668 if num_shelves > 0 and num_shelves != len(commits):
2669 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2670 (len(commits), num_shelves))
2671
2672 if not self.no_verify:
2673 try:
2674 if not run_git_hook("p4-pre-submit"):
2675 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nYou can skip "
2676 "this pre-submission check by adding\nthe command line option '--no-verify', "
2677 "however,\nthis will also skip the p4-changelist hook as well.")
2678 sys.exit(1)
2679 except Exception as e:
2680 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nThe hook failed "
2681 "with the error '{0}'".format(e.message))
2682 sys.exit(1)
2683
2684 #
2685 # Apply the commits, one at a time. On failure, ask if should
2686 # continue to try the rest of the patches, or quit.
2687 #
2688 if self.dry_run:
2689 print("Would apply")
2690 applied = []
2691 last = len(commits) - 1
2692 for i, commit in enumerate(commits):
2693 if self.dry_run:
2694 print(" ", read_pipe(["git", "show", "-s",
2695 "--format=format:%h %s", commit]))
2696 ok = True
2697 else:
2698 ok = self.applyCommit(commit)
2699 if ok:
2700 applied.append(commit)
2701 if self.prepare_p4_only:
2702 if i < last:
2703 print("Processing only the first commit due to option"
2704 " --prepare-p4-only")
2705 break
2706 else:
2707 if i < last:
2708 # prompt for what to do, or use the option/variable
2709 if self.conflict_behavior == "ask":
2710 print("What do you want to do?")
2711 response = prompt("[s]kip this commit but apply the rest, or [q]uit? ")
2712 elif self.conflict_behavior == "skip":
2713 response = "s"
2714 elif self.conflict_behavior == "quit":
2715 response = "q"
2716 else:
2717 die("Unknown conflict_behavior '%s'" %
2718 self.conflict_behavior)
2719
2720 if response == "s":
2721 print("Skipping this commit, but applying the rest")
2722 if response == "q":
2723 print("Quitting")
2724 break
2725
2726 chdir(self.oldWorkingDirectory)
2727 shelved_applied = "shelved" if self.shelve else "applied"
2728 if self.dry_run:
2729 pass
2730 elif self.prepare_p4_only:
2731 pass
2732 elif len(commits) == len(applied):
2733 print("All commits {0}!".format(shelved_applied))
2734
2735 sync = P4Sync()
2736 if self.branch:
2737 sync.branch = self.branch
2738 if self.disable_p4sync:
2739 sync.sync_origin_only()
2740 else:
2741 sync.run([])
2742
2743 if not self.disable_rebase:
2744 rebase = P4Rebase()
2745 rebase.rebase()
2746
2747 else:
2748 if len(applied) == 0:
2749 print("No commits {0}.".format(shelved_applied))
2750 else:
2751 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2752 for c in commits:
2753 if c in applied:
2754 star = "*"
2755 else:
2756 star = " "
2757 print(star, read_pipe(["git", "show", "-s",
2758 "--format=format:%h %s", c]))
2759 print("You will have to do 'git p4 sync' and rebase.")
2760
2761 if gitConfigBool("git-p4.exportLabels"):
2762 self.exportLabels = True
2763
2764 if self.exportLabels:
2765 p4Labels = getP4Labels(self.depotPath)
2766 gitTags = getGitTags()
2767
2768 missingGitTags = gitTags - p4Labels
2769 self.exportGitTags(missingGitTags)
2770
2771 # exit with error unless everything applied perfectly
2772 if len(commits) != len(applied):
2773 sys.exit(1)
2774
2775 return True
2776
2777
2778 class View(object):
2779 """Represent a p4 view ("p4 help views"), and map files in a repo according
2780 to the view.
2781 """
2782
2783 def __init__(self, client_name):
2784 self.mappings = []
2785 self.client_prefix = "//%s/" % client_name
2786 # cache results of "p4 where" to lookup client file locations
2787 self.client_spec_path_cache = {}
2788
2789 def append(self, view_line):
2790 """Parse a view line, splitting it into depot and client sides. Append
2791 to self.mappings, preserving order. This is only needed for tag
2792 creation.
2793 """
2794
2795 # Split the view line into exactly two words. P4 enforces
2796 # structure on these lines that simplifies this quite a bit.
2797 #
2798 # Either or both words may be double-quoted.
2799 # Single quotes do not matter.
2800 # Double-quote marks cannot occur inside the words.
2801 # A + or - prefix is also inside the quotes.
2802 # There are no quotes unless they contain a space.
2803 # The line is already white-space stripped.
2804 # The two words are separated by a single space.
2805 #
2806 if view_line[0] == '"':
2807 # First word is double quoted. Find its end.
2808 close_quote_index = view_line.find('"', 1)
2809 if close_quote_index <= 0:
2810 die("No first-word closing quote found: %s" % view_line)
2811 depot_side = view_line[1:close_quote_index]
2812 # skip closing quote and space
2813 rhs_index = close_quote_index + 1 + 1
2814 else:
2815 space_index = view_line.find(" ")
2816 if space_index <= 0:
2817 die("No word-splitting space found: %s" % view_line)
2818 depot_side = view_line[0:space_index]
2819 rhs_index = space_index + 1
2820
2821 # prefix + means overlay on previous mapping
2822 if depot_side.startswith("+"):
2823 depot_side = depot_side[1:]
2824
2825 # prefix - means exclude this path, leave out of mappings
2826 exclude = False
2827 if depot_side.startswith("-"):
2828 exclude = True
2829 depot_side = depot_side[1:]
2830
2831 if not exclude:
2832 self.mappings.append(depot_side)
2833
2834 def convert_client_path(self, clientFile):
2835 # chop off //client/ part to make it relative
2836 if not decode_path(clientFile).startswith(self.client_prefix):
2837 die("No prefix '%s' on clientFile '%s'" %
2838 (self.client_prefix, clientFile))
2839 return clientFile[len(self.client_prefix):]
2840
2841 def update_client_spec_path_cache(self, files):
2842 """Caching file paths by "p4 where" batch query."""
2843
2844 # List depot file paths exclude that already cached
2845 fileArgs = [f['path'] for f in files if decode_path(f['path']) not in self.client_spec_path_cache]
2846
2847 if len(fileArgs) == 0:
2848 return # All files in cache
2849
2850 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2851 for res in where_result:
2852 if "code" in res and res["code"] == "error":
2853 # assume error is "... file(s) not in client view"
2854 continue
2855 if "clientFile" not in res:
2856 die("No clientFile in 'p4 where' output")
2857 if "unmap" in res:
2858 # it will list all of them, but only one not unmap-ped
2859 continue
2860 depot_path = decode_path(res['depotFile'])
2861 if gitConfigBool("core.ignorecase"):
2862 depot_path = depot_path.lower()
2863 self.client_spec_path_cache[depot_path] = self.convert_client_path(res["clientFile"])
2864
2865 # not found files or unmap files set to ""
2866 for depotFile in fileArgs:
2867 depotFile = decode_path(depotFile)
2868 if gitConfigBool("core.ignorecase"):
2869 depotFile = depotFile.lower()
2870 if depotFile not in self.client_spec_path_cache:
2871 self.client_spec_path_cache[depotFile] = b''
2872
2873 def map_in_client(self, depot_path):
2874 """Return the relative location in the client where this depot file
2875 should live.
2876
2877 Returns "" if the file should not be mapped in the client.
2878 """
2879
2880 if gitConfigBool("core.ignorecase"):
2881 depot_path = depot_path.lower()
2882
2883 if depot_path in self.client_spec_path_cache:
2884 return self.client_spec_path_cache[depot_path]
2885
2886 die("Error: %s is not found in client spec path" % depot_path)
2887 return ""
2888
2889
2890 def cloneExcludeCallback(option, opt_str, value, parser):
2891 # prepend "/" because the first "/" was consumed as part of the option itself.
2892 # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2893 parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)]
2894
2895
2896 class P4Sync(Command, P4UserMap):
2897
2898 def __init__(self):
2899 Command.__init__(self)
2900 P4UserMap.__init__(self)
2901 self.options = [
2902 optparse.make_option("--branch", dest="branch"),
2903 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2904 optparse.make_option("--changesfile", dest="changesFile"),
2905 optparse.make_option("--silent", dest="silent", action="store_true"),
2906 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2907 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2908 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2909 help="Import into refs/heads/ , not refs/remotes"),
2910 optparse.make_option("--max-changes", dest="maxChanges",
2911 help="Maximum number of changes to import"),
2912 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2913 help="Internal block size to use when iteratively calling p4 changes"),
2914 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2915 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2916 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2917 help="Only sync files that are included in the Perforce Client Spec"),
2918 optparse.make_option("-/", dest="cloneExclude",
2919 action="callback", callback=cloneExcludeCallback, type="string",
2920 help="exclude depot path"),
2921 ]
2922 self.description = """Imports from Perforce into a git repository.\n
2923 example:
2924 //depot/my/project/ -- to import the current head
2925 //depot/my/project/@all -- to import everything
2926 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2927
2928 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2929
2930 self.usage += " //depot/path[@revRange]"
2931 self.silent = False
2932 self.createdBranches = set()
2933 self.committedChanges = set()
2934 self.branch = ""
2935 self.detectBranches = False
2936 self.detectLabels = False
2937 self.importLabels = False
2938 self.changesFile = ""
2939 self.syncWithOrigin = True
2940 self.importIntoRemotes = True
2941 self.maxChanges = ""
2942 self.changes_block_size = None
2943 self.keepRepoPath = False
2944 self.depotPaths = None
2945 self.p4BranchesInGit = []
2946 self.cloneExclude = []
2947 self.useClientSpec = False
2948 self.useClientSpec_from_options = False
2949 self.clientSpecDirs = None
2950 self.tempBranches = []
2951 self.tempBranchLocation = "refs/git-p4-tmp"
2952 self.largeFileSystem = None
2953 self.suppress_meta_comment = False
2954
2955 if gitConfig('git-p4.largeFileSystem'):
2956 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2957 self.largeFileSystem = largeFileSystemConstructor(
2958 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2959 )
2960
2961 if gitConfig("git-p4.syncFromOrigin") == "false":
2962 self.syncWithOrigin = False
2963
2964 self.depotPaths = []
2965 self.changeRange = ""
2966 self.previousDepotPaths = []
2967 self.hasOrigin = False
2968
2969 # map from branch depot path to parent branch
2970 self.knownBranches = {}
2971 self.initialParents = {}
2972
2973 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2974 self.labels = {}
2975
2976 def checkpoint(self):
2977 """Force a checkpoint in fast-import and wait for it to finish."""
2978 self.gitStream.write("checkpoint\n\n")
2979 self.gitStream.write("progress checkpoint\n\n")
2980 self.gitStream.flush()
2981 out = self.gitOutput.readline()
2982 if self.verbose:
2983 print("checkpoint finished: " + out)
2984
2985 def isPathWanted(self, path):
2986 for p in self.cloneExclude:
2987 if p.endswith("/"):
2988 if p4PathStartsWith(path, p):
2989 return False
2990 # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2991 elif path.lower() == p.lower():
2992 return False
2993 for p in self.depotPaths:
2994 if p4PathStartsWith(path, decode_path(p)):
2995 return True
2996 return False
2997
2998 def extractFilesFromCommit(self, commit, shelved=False, shelved_cl=0):
2999 files = []
3000 fnum = 0
3001 while "depotFile%s" % fnum in commit:
3002 path = commit["depotFile%s" % fnum]
3003 found = self.isPathWanted(decode_path(path))
3004 if not found:
3005 fnum = fnum + 1
3006 continue
3007
3008 file = {}
3009 file["path"] = path
3010 file["rev"] = commit["rev%s" % fnum]
3011 file["action"] = commit["action%s" % fnum]
3012 file["type"] = commit["type%s" % fnum]
3013 if shelved:
3014 file["shelved_cl"] = int(shelved_cl)
3015 files.append(file)
3016 fnum = fnum + 1
3017 return files
3018
3019 def extractJobsFromCommit(self, commit):
3020 jobs = []
3021 jnum = 0
3022 while "job%s" % jnum in commit:
3023 job = commit["job%s" % jnum]
3024 jobs.append(job)
3025 jnum = jnum + 1
3026 return jobs
3027
3028 def stripRepoPath(self, path, prefixes):
3029 """When streaming files, this is called to map a p4 depot path to where
3030 it should go in git. The prefixes are either self.depotPaths, or
3031 self.branchPrefixes in the case of branch detection.
3032 """
3033
3034 if self.useClientSpec:
3035 # branch detection moves files up a level (the branch name)
3036 # from what client spec interpretation gives
3037 path = decode_path(self.clientSpecDirs.map_in_client(path))
3038 if self.detectBranches:
3039 for b in self.knownBranches:
3040 if p4PathStartsWith(path, b + "/"):
3041 path = path[len(b)+1:]
3042
3043 elif self.keepRepoPath:
3044 # Preserve everything in relative path name except leading
3045 # //depot/; just look at first prefix as they all should
3046 # be in the same depot.
3047 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
3048 if p4PathStartsWith(path, depot):
3049 path = path[len(depot):]
3050
3051 else:
3052 for p in prefixes:
3053 if p4PathStartsWith(path, p):
3054 path = path[len(p):]
3055 break
3056
3057 path = wildcard_decode(path)
3058 return path
3059
3060 def splitFilesIntoBranches(self, commit):
3061 """Look at each depotFile in the commit to figure out to what branch it
3062 belongs.
3063 """
3064
3065 if self.clientSpecDirs:
3066 files = self.extractFilesFromCommit(commit)
3067 self.clientSpecDirs.update_client_spec_path_cache(files)
3068
3069 branches = {}
3070 fnum = 0
3071 while "depotFile%s" % fnum in commit:
3072 raw_path = commit["depotFile%s" % fnum]
3073 path = decode_path(raw_path)
3074 found = self.isPathWanted(path)
3075 if not found:
3076 fnum = fnum + 1
3077 continue
3078
3079 file = {}
3080 file["path"] = raw_path
3081 file["rev"] = commit["rev%s" % fnum]
3082 file["action"] = commit["action%s" % fnum]
3083 file["type"] = commit["type%s" % fnum]
3084 fnum = fnum + 1
3085
3086 # start with the full relative path where this file would
3087 # go in a p4 client
3088 if self.useClientSpec:
3089 relPath = decode_path(self.clientSpecDirs.map_in_client(path))
3090 else:
3091 relPath = self.stripRepoPath(path, self.depotPaths)
3092
3093 for branch in self.knownBranches.keys():
3094 # add a trailing slash so that a commit into qt/4.2foo
3095 # doesn't end up in qt/4.2, e.g.
3096 if p4PathStartsWith(relPath, branch + "/"):
3097 if branch not in branches:
3098 branches[branch] = []
3099 branches[branch].append(file)
3100 break
3101
3102 return branches
3103
3104 def writeToGitStream(self, gitMode, relPath, contents):
3105 self.gitStream.write(encode_text_stream(u'M {} inline {}\n'.format(gitMode, relPath)))
3106 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
3107 for d in contents:
3108 self.gitStream.write(d)
3109 self.gitStream.write('\n')
3110
3111 def encodeWithUTF8(self, path):
3112 try:
3113 path.decode('ascii')
3114 except:
3115 encoding = 'utf8'
3116 if gitConfig('git-p4.pathEncoding'):
3117 encoding = gitConfig('git-p4.pathEncoding')
3118 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
3119 if self.verbose:
3120 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
3121 return path
3122
3123 def streamOneP4File(self, file, contents):
3124 """Output one file from the P4 stream.
3125
3126 This is a helper for streamP4Files().
3127 """
3128
3129 file_path = file['depotFile']
3130 relPath = self.stripRepoPath(decode_path(file_path), self.branchPrefixes)
3131
3132 if verbose:
3133 if 'fileSize' in self.stream_file:
3134 size = int(self.stream_file['fileSize'])
3135 else:
3136 # Deleted files don't get a fileSize apparently
3137 size = 0
3138 sys.stdout.write('\r%s --> %s (%s)\n' % (
3139 file_path, relPath, format_size_human_readable(size)))
3140 sys.stdout.flush()
3141
3142 type_base, type_mods = split_p4_type(file["type"])
3143
3144 git_mode = "100644"
3145 if "x" in type_mods:
3146 git_mode = "100755"
3147 if type_base == "symlink":
3148 git_mode = "120000"
3149 # p4 print on a symlink sometimes contains "target\n";
3150 # if it does, remove the newline
3151 data = ''.join(decode_text_stream(c) for c in contents)
3152 if not data:
3153 # Some version of p4 allowed creating a symlink that pointed
3154 # to nothing. This causes p4 errors when checking out such
3155 # a change, and errors here too. Work around it by ignoring
3156 # the bad symlink; hopefully a future change fixes it.
3157 print("\nIgnoring empty symlink in %s" % file_path)
3158 return
3159 elif data[-1] == '\n':
3160 contents = [data[:-1]]
3161 else:
3162 contents = [data]
3163
3164 if type_base == "utf16":
3165 # p4 delivers different text in the python output to -G
3166 # than it does when using "print -o", or normal p4 client
3167 # operations. utf16 is converted to ascii or utf8, perhaps.
3168 # But ascii text saved as -t utf16 is completely mangled.
3169 # Invoke print -o to get the real contents.
3170 #
3171 # On windows, the newlines will always be mangled by print, so put
3172 # them back too. This is not needed to the cygwin windows version,
3173 # just the native "NT" type.
3174 #
3175 try:
3176 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (decode_path(file['depotFile']), file['change'])], raw=True)
3177 except Exception as e:
3178 if 'Translation of file content failed' in str(e):
3179 type_base = 'binary'
3180 else:
3181 raise e
3182 else:
3183 if p4_version_string().find('/NT') >= 0:
3184 text = text.replace(b'\x0d\x00\x0a\x00', b'\x0a\x00')
3185 contents = [text]
3186
3187 if type_base == "apple":
3188 # Apple filetype files will be streamed as a concatenation of
3189 # its appledouble header and the contents. This is useless
3190 # on both macs and non-macs. If using "print -q -o xx", it
3191 # will create "xx" with the data, and "%xx" with the header.
3192 # This is also not very useful.
3193 #
3194 # Ideally, someday, this script can learn how to generate
3195 # appledouble files directly and import those to git, but
3196 # non-mac machines can never find a use for apple filetype.
3197 print("\nIgnoring apple filetype file %s" % file['depotFile'])
3198 return
3199
3200 if type_base == "utf8":
3201 # The type utf8 explicitly means utf8 *with BOM*. These are
3202 # streamed just like regular text files, however, without
3203 # the BOM in the stream.
3204 # Therefore, to accurately import these files into git, we
3205 # need to explicitly re-add the BOM before writing.
3206 # 'contents' is a set of bytes in this case, so create the
3207 # BOM prefix as a b'' literal.
3208 contents = [b'\xef\xbb\xbf' + contents[0]] + contents[1:]
3209
3210 # Note that we do not try to de-mangle keywords on utf16 files,
3211 # even though in theory somebody may want that.
3212 regexp = p4_keywords_regexp_for_type(type_base, type_mods)
3213 if regexp:
3214 contents = [regexp.sub(br'$\1$', c) for c in contents]
3215
3216 if self.largeFileSystem:
3217 git_mode, contents = self.largeFileSystem.processContent(git_mode, relPath, contents)
3218
3219 self.writeToGitStream(git_mode, relPath, contents)
3220
3221 def streamOneP4Deletion(self, file):
3222 relPath = self.stripRepoPath(decode_path(file['path']), self.branchPrefixes)
3223 if verbose:
3224 sys.stdout.write("delete %s\n" % relPath)
3225 sys.stdout.flush()
3226 self.gitStream.write(encode_text_stream(u'D {}\n'.format(relPath)))
3227
3228 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
3229 self.largeFileSystem.removeLargeFile(relPath)
3230
3231 def streamP4FilesCb(self, marshalled):
3232 """Handle another chunk of streaming data."""
3233
3234 # catch p4 errors and complain
3235 err = None
3236 if "code" in marshalled:
3237 if marshalled["code"] == "error":
3238 if "data" in marshalled:
3239 err = marshalled["data"].rstrip()
3240
3241 if not err and 'fileSize' in self.stream_file:
3242 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
3243 if required_bytes > 0:
3244 err = 'Not enough space left on %s! Free at least %s.' % (
3245 os.getcwd(), format_size_human_readable(required_bytes))
3246
3247 if err:
3248 f = None
3249 if self.stream_have_file_info:
3250 if "depotFile" in self.stream_file:
3251 f = self.stream_file["depotFile"]
3252 # force a failure in fast-import, else an empty
3253 # commit will be made
3254 self.gitStream.write("\n")
3255 self.gitStream.write("die-now\n")
3256 self.gitStream.close()
3257 # ignore errors, but make sure it exits first
3258 self.importProcess.wait()
3259 if f:
3260 die("Error from p4 print for %s: %s" % (f, err))
3261 else:
3262 die("Error from p4 print: %s" % err)
3263
3264 if 'depotFile' in marshalled and self.stream_have_file_info:
3265 # start of a new file - output the old one first
3266 self.streamOneP4File(self.stream_file, self.stream_contents)
3267 self.stream_file = {}
3268 self.stream_contents = []
3269 self.stream_have_file_info = False
3270
3271 # pick up the new file information... for the
3272 # 'data' field we need to append to our array
3273 for k in marshalled.keys():
3274 if k == 'data':
3275 if 'streamContentSize' not in self.stream_file:
3276 self.stream_file['streamContentSize'] = 0
3277 self.stream_file['streamContentSize'] += len(marshalled['data'])
3278 self.stream_contents.append(marshalled['data'])
3279 else:
3280 self.stream_file[k] = marshalled[k]
3281
3282 if (verbose and
3283 'streamContentSize' in self.stream_file and
3284 'fileSize' in self.stream_file and
3285 'depotFile' in self.stream_file):
3286 size = int(self.stream_file["fileSize"])
3287 if size > 0:
3288 progress = 100*self.stream_file['streamContentSize']/size
3289 sys.stdout.write('\r%s %d%% (%s)' % (
3290 self.stream_file['depotFile'], progress,
3291 format_size_human_readable(size)))
3292 sys.stdout.flush()
3293
3294 self.stream_have_file_info = True
3295
3296 def streamP4Files(self, files):
3297 """Stream directly from "p4 files" into "git fast-import."""
3298
3299 filesForCommit = []
3300 filesToRead = []
3301 filesToDelete = []
3302
3303 for f in files:
3304 filesForCommit.append(f)
3305 if f['action'] in self.delete_actions:
3306 filesToDelete.append(f)
3307 else:
3308 filesToRead.append(f)
3309
3310 # deleted files...
3311 for f in filesToDelete:
3312 self.streamOneP4Deletion(f)
3313
3314 if len(filesToRead) > 0:
3315 self.stream_file = {}
3316 self.stream_contents = []
3317 self.stream_have_file_info = False
3318
3319 # curry self argument
3320 def streamP4FilesCbSelf(entry):
3321 self.streamP4FilesCb(entry)
3322
3323 fileArgs = []
3324 for f in filesToRead:
3325 if 'shelved_cl' in f:
3326 # Handle shelved CLs using the "p4 print file@=N" syntax to print
3327 # the contents
3328 fileArg = f['path'] + encode_text_stream('@={}'.format(f['shelved_cl']))
3329 else:
3330 fileArg = f['path'] + encode_text_stream('#{}'.format(f['rev']))
3331
3332 fileArgs.append(fileArg)
3333
3334 p4CmdList(["-x", "-", "print"],
3335 stdin=fileArgs,
3336 cb=streamP4FilesCbSelf)
3337
3338 # do the last chunk
3339 if 'depotFile' in self.stream_file:
3340 self.streamOneP4File(self.stream_file, self.stream_contents)
3341
3342 def make_email(self, userid):
3343 if userid in self.users:
3344 return self.users[userid]
3345 else:
3346 userid_bytes = metadata_stream_to_writable_bytes(userid)
3347 return b"%s <a@b>" % userid_bytes
3348
3349 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
3350 """Stream a p4 tag.
3351
3352 Commit is either a git commit, or a fast-import mark, ":<p4commit>".
3353 """
3354
3355 if verbose:
3356 print("writing tag %s for commit %s" % (labelName, commit))
3357 gitStream.write("tag %s\n" % labelName)
3358 gitStream.write("from %s\n" % commit)
3359
3360 if 'Owner' in labelDetails:
3361 owner = labelDetails["Owner"]
3362 else:
3363 owner = None
3364
3365 # Try to use the owner of the p4 label, or failing that,
3366 # the current p4 user id.
3367 if owner:
3368 email = self.make_email(owner)
3369 else:
3370 email = self.make_email(self.p4UserId())
3371
3372 gitStream.write("tagger ")
3373 gitStream.write(email)
3374 gitStream.write(" %s %s\n" % (epoch, self.tz))
3375
3376 print("labelDetails=", labelDetails)
3377 if 'Description' in labelDetails:
3378 description = labelDetails['Description']
3379 else:
3380 description = 'Label from git p4'
3381
3382 gitStream.write("data %d\n" % len(description))
3383 gitStream.write(description)
3384 gitStream.write("\n")
3385
3386 def inClientSpec(self, path):
3387 if not self.clientSpecDirs:
3388 return True
3389 inClientSpec = self.clientSpecDirs.map_in_client(path)
3390 if not inClientSpec and self.verbose:
3391 print('Ignoring file outside of client spec: {0}'.format(path))
3392 return inClientSpec
3393
3394 def hasBranchPrefix(self, path):
3395 if not self.branchPrefixes:
3396 return True
3397 hasPrefix = [p for p in self.branchPrefixes
3398 if p4PathStartsWith(path, p)]
3399 if not hasPrefix and self.verbose:
3400 print('Ignoring file outside of prefix: {0}'.format(path))
3401 return hasPrefix
3402
3403 def findShadowedFiles(self, files, change):
3404 """Perforce allows you commit files and directories with the same name,
3405 so you could have files //depot/foo and //depot/foo/bar both checked
3406 in. A p4 sync of a repository in this state fails. Deleting one of
3407 the files recovers the repository.
3408
3409 Git will not allow the broken state to exist and only the most
3410 recent of the conflicting names is left in the repository. When one
3411 of the conflicting files is deleted we need to re-add the other one
3412 to make sure the git repository recovers in the same way as
3413 perforce.
3414 """
3415
3416 deleted = [f for f in files if f['action'] in self.delete_actions]
3417 to_check = set()
3418 for f in deleted:
3419 path = decode_path(f['path'])
3420 to_check.add(path + '/...')
3421 while True:
3422 path = path.rsplit("/", 1)[0]
3423 if path == "/" or path in to_check:
3424 break
3425 to_check.add(path)
3426 to_check = ['%s@%s' % (wildcard_encode(p), change) for p in to_check
3427 if self.hasBranchPrefix(p)]
3428 if to_check:
3429 stat_result = p4CmdList(["-x", "-", "fstat", "-T",
3430 "depotFile,headAction,headRev,headType"], stdin=to_check)
3431 for record in stat_result:
3432 if record['code'] != 'stat':
3433 continue
3434 if record['headAction'] in self.delete_actions:
3435 continue
3436 files.append({
3437 'action': 'add',
3438 'path': record['depotFile'],
3439 'rev': record['headRev'],
3440 'type': record['headType']})
3441
3442 def commit(self, details, files, branch, parent="", allow_empty=False):
3443 epoch = details["time"]
3444 author = details["user"]
3445 jobs = self.extractJobsFromCommit(details)
3446
3447 if self.verbose:
3448 print('commit into {0}'.format(branch))
3449
3450 files = [f for f in files
3451 if self.hasBranchPrefix(decode_path(f['path']))]
3452 self.findShadowedFiles(files, details['change'])
3453
3454 if self.clientSpecDirs:
3455 self.clientSpecDirs.update_client_spec_path_cache(files)
3456
3457 files = [f for f in files if self.inClientSpec(decode_path(f['path']))]
3458
3459 if gitConfigBool('git-p4.keepEmptyCommits'):
3460 allow_empty = True
3461
3462 if not files and not allow_empty:
3463 print('Ignoring revision {0} as it would produce an empty commit.'
3464 .format(details['change']))
3465 return
3466
3467 self.gitStream.write("commit %s\n" % branch)
3468 self.gitStream.write("mark :%s\n" % details["change"])
3469 self.committedChanges.add(int(details["change"]))
3470 if author not in self.users:
3471 self.getUserMapFromPerforceServer()
3472
3473 self.gitStream.write("committer ")
3474 self.gitStream.write(self.make_email(author))
3475 self.gitStream.write(" %s %s\n" % (epoch, self.tz))
3476
3477 self.gitStream.write("data <<EOT\n")
3478 self.gitStream.write(details["desc"])
3479 if len(jobs) > 0:
3480 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3481
3482 if not self.suppress_meta_comment:
3483 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3484 (','.join(self.branchPrefixes), details["change"]))
3485 if len(details['options']) > 0:
3486 self.gitStream.write(": options = %s" % details['options'])
3487 self.gitStream.write("]\n")
3488
3489 self.gitStream.write("EOT\n\n")
3490
3491 if len(parent) > 0:
3492 if self.verbose:
3493 print("parent %s" % parent)
3494 self.gitStream.write("from %s\n" % parent)
3495
3496 self.streamP4Files(files)
3497 self.gitStream.write("\n")
3498
3499 change = int(details["change"])
3500
3501 if change in self.labels:
3502 label = self.labels[change]
3503 labelDetails = label[0]
3504 labelRevisions = label[1]
3505 if self.verbose:
3506 print("Change %s is labelled %s" % (change, labelDetails))
3507
3508 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3509 for p in self.branchPrefixes])
3510
3511 if len(files) == len(labelRevisions):
3512
3513 cleanedFiles = {}
3514 for info in files:
3515 if info["action"] in self.delete_actions:
3516 continue
3517 cleanedFiles[info["depotFile"]] = info["rev"]
3518
3519 if cleanedFiles == labelRevisions:
3520 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3521
3522 else:
3523 if not self.silent:
3524 print("Tag %s does not match with change %s: files do not match."
3525 % (labelDetails["label"], change))
3526
3527 else:
3528 if not self.silent:
3529 print("Tag %s does not match with change %s: file count is different."
3530 % (labelDetails["label"], change))
3531
3532 def getLabels(self):
3533 """Build a dictionary of changelists and labels, for "detect-labels"
3534 option.
3535 """
3536
3537 self.labels = {}
3538
3539 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3540 if len(l) > 0 and not self.silent:
3541 print("Finding files belonging to labels in %s" % self.depotPaths)
3542
3543 for output in l:
3544 label = output["label"]
3545 revisions = {}
3546 newestChange = 0
3547 if self.verbose:
3548 print("Querying files for label %s" % label)
3549 for file in p4CmdList(["files"] +
3550 ["%s...@%s" % (p, label)
3551 for p in self.depotPaths]):
3552 revisions[file["depotFile"]] = file["rev"]
3553 change = int(file["change"])
3554 if change > newestChange:
3555 newestChange = change
3556
3557 self.labels[newestChange] = [output, revisions]
3558
3559 if self.verbose:
3560 print("Label changes: %s" % self.labels.keys())
3561
3562 def importP4Labels(self, stream, p4Labels):
3563 """Import p4 labels as git tags. A direct mapping does not exist, so
3564 assume that if all the files are at the same revision then we can
3565 use that, or it's something more complicated we should just ignore.
3566 """
3567
3568 if verbose:
3569 print("import p4 labels: " + ' '.join(p4Labels))
3570
3571 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3572 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3573 if len(validLabelRegexp) == 0:
3574 validLabelRegexp = defaultLabelRegexp
3575 m = re.compile(validLabelRegexp)
3576
3577 for name in p4Labels:
3578 commitFound = False
3579
3580 if not m.match(name):
3581 if verbose:
3582 print("label %s does not match regexp %s" % (name, validLabelRegexp))
3583 continue
3584
3585 if name in ignoredP4Labels:
3586 continue
3587
3588 labelDetails = p4CmdList(['label', "-o", name])[0]
3589
3590 # get the most recent changelist for each file in this label
3591 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3592 for p in self.depotPaths])
3593
3594 if 'change' in change:
3595 # find the corresponding git commit; take the oldest commit
3596 changelist = int(change['change'])
3597 if changelist in self.committedChanges:
3598 gitCommit = ":%d" % changelist # use a fast-import mark
3599 commitFound = True
3600 else:
3601 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3602 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3603 if len(gitCommit) == 0:
3604 print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3605 else:
3606 commitFound = True
3607 gitCommit = gitCommit.strip()
3608
3609 if commitFound:
3610 # Convert from p4 time format
3611 try:
3612 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3613 except ValueError:
3614 print("Could not convert label time %s" % labelDetails['Update'])
3615 tmwhen = 1
3616
3617 when = int(time.mktime(tmwhen))
3618 self.streamTag(stream, name, labelDetails, gitCommit, when)
3619 if verbose:
3620 print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3621 else:
3622 if verbose:
3623 print("Label %s has no changelists - possibly deleted?" % name)
3624
3625 if not commitFound:
3626 # We can't import this label; don't try again as it will get very
3627 # expensive repeatedly fetching all the files for labels that will
3628 # never be imported. If the label is moved in the future, the
3629 # ignore will need to be removed manually.
3630 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3631
3632 def guessProjectName(self):
3633 for p in self.depotPaths:
3634 if p.endswith("/"):
3635 p = p[:-1]
3636 p = p[p.strip().rfind("/") + 1:]
3637 if not p.endswith("/"):
3638 p += "/"
3639 return p
3640
3641 def getBranchMapping(self):
3642 lostAndFoundBranches = set()
3643
3644 user = gitConfig("git-p4.branchUser")
3645
3646 for info in p4CmdList(
3647 ["branches"] + (["-u", user] if len(user) > 0 else [])):
3648 details = p4Cmd(["branch", "-o", info["branch"]])
3649 viewIdx = 0
3650 while "View%s" % viewIdx in details:
3651 paths = details["View%s" % viewIdx].split(" ")
3652 viewIdx = viewIdx + 1
3653 # require standard //depot/foo/... //depot/bar/... mapping
3654 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3655 continue
3656 source = paths[0]
3657 destination = paths[1]
3658 # HACK
3659 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3660 source = source[len(self.depotPaths[0]):-4]
3661 destination = destination[len(self.depotPaths[0]):-4]
3662
3663 if destination in self.knownBranches:
3664 if not self.silent:
3665 print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3666 print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3667 continue
3668
3669 self.knownBranches[destination] = source
3670
3671 lostAndFoundBranches.discard(destination)
3672
3673 if source not in self.knownBranches:
3674 lostAndFoundBranches.add(source)
3675
3676 # Perforce does not strictly require branches to be defined, so we also
3677 # check git config for a branch list.
3678 #
3679 # Example of branch definition in git config file:
3680 # [git-p4]
3681 # branchList=main:branchA
3682 # branchList=main:branchB
3683 # branchList=branchA:branchC
3684 configBranches = gitConfigList("git-p4.branchList")
3685 for branch in configBranches:
3686 if branch:
3687 source, destination = branch.split(":")
3688 self.knownBranches[destination] = source
3689
3690 lostAndFoundBranches.discard(destination)
3691
3692 if source not in self.knownBranches:
3693 lostAndFoundBranches.add(source)
3694
3695 for branch in lostAndFoundBranches:
3696 self.knownBranches[branch] = branch
3697
3698 def getBranchMappingFromGitBranches(self):
3699 branches = p4BranchesInGit(self.importIntoRemotes)
3700 for branch in branches.keys():
3701 if branch == "master":
3702 branch = "main"
3703 else:
3704 branch = branch[len(self.projectName):]
3705 self.knownBranches[branch] = branch
3706
3707 def updateOptionDict(self, d):
3708 option_keys = {}
3709 if self.keepRepoPath:
3710 option_keys['keepRepoPath'] = 1
3711
3712 d["options"] = ' '.join(sorted(option_keys.keys()))
3713
3714 def readOptions(self, d):
3715 self.keepRepoPath = ('options' in d
3716 and ('keepRepoPath' in d['options']))
3717
3718 def gitRefForBranch(self, branch):
3719 if branch == "main":
3720 return self.refPrefix + "master"
3721
3722 if len(branch) <= 0:
3723 return branch
3724
3725 return self.refPrefix + self.projectName + branch
3726
3727 def gitCommitByP4Change(self, ref, change):
3728 if self.verbose:
3729 print("looking in ref " + ref + " for change %s using bisect..." % change)
3730
3731 earliestCommit = ""
3732 latestCommit = parseRevision(ref)
3733
3734 while True:
3735 if self.verbose:
3736 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3737 next = read_pipe(["git", "rev-list", "--bisect",
3738 latestCommit, earliestCommit]).strip()
3739 if len(next) == 0:
3740 if self.verbose:
3741 print("argh")
3742 return ""
3743 log = extractLogMessageFromGitCommit(next)
3744 settings = extractSettingsGitLog(log)
3745 currentChange = int(settings['change'])
3746 if self.verbose:
3747 print("current change %s" % currentChange)
3748
3749 if currentChange == change:
3750 if self.verbose:
3751 print("found %s" % next)
3752 return next
3753
3754 if currentChange < change:
3755 earliestCommit = "^%s" % next
3756 else:
3757 if next == latestCommit:
3758 die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref, change))
3759 latestCommit = "%s^@" % next
3760
3761 return ""
3762
3763 def importNewBranch(self, branch, maxChange):
3764 # make fast-import flush all changes to disk and update the refs using the checkpoint
3765 # command so that we can try to find the branch parent in the git history
3766 self.gitStream.write("checkpoint\n\n")
3767 self.gitStream.flush()
3768 branchPrefix = self.depotPaths[0] + branch + "/"
3769 range = "@1,%s" % maxChange
3770 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3771 if len(changes) <= 0:
3772 return False
3773 firstChange = changes[0]
3774 sourceBranch = self.knownBranches[branch]
3775 sourceDepotPath = self.depotPaths[0] + sourceBranch
3776 sourceRef = self.gitRefForBranch(sourceBranch)
3777
3778 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3779 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3780 if len(gitParent) > 0:
3781 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3782
3783 self.importChanges(changes)
3784 return True
3785
3786 def searchParent(self, parent, branch, target):
3787 targetTree = read_pipe(["git", "rev-parse",
3788 "{}^{{tree}}".format(target)]).strip()
3789 for line in read_pipe_lines(["git", "rev-list", "--format=%H %T",
3790 "--no-merges", parent]):
3791 if line.startswith("commit "):
3792 continue
3793 commit, tree = line.strip().split(" ")
3794 if tree == targetTree:
3795 if self.verbose:
3796 print("Found parent of %s in commit %s" % (branch, commit))
3797 return commit
3798 return None
3799
3800 def importChanges(self, changes, origin_revision=0):
3801 cnt = 1
3802 for change in changes:
3803 description = p4_describe(change)
3804 self.updateOptionDict(description)
3805
3806 if not self.silent:
3807 sys.stdout.write("\rImporting revision %s (%d%%)" % (
3808 change, (cnt * 100) // len(changes)))
3809 sys.stdout.flush()
3810 cnt = cnt + 1
3811
3812 try:
3813 if self.detectBranches:
3814 branches = self.splitFilesIntoBranches(description)
3815 for branch in branches.keys():
3816 # HACK --hwn
3817 branchPrefix = self.depotPaths[0] + branch + "/"
3818 self.branchPrefixes = [branchPrefix]
3819
3820 parent = ""
3821
3822 filesForCommit = branches[branch]
3823
3824 if self.verbose:
3825 print("branch is %s" % branch)
3826
3827 self.updatedBranches.add(branch)
3828
3829 if branch not in self.createdBranches:
3830 self.createdBranches.add(branch)
3831 parent = self.knownBranches[branch]
3832 if parent == branch:
3833 parent = ""
3834 else:
3835 fullBranch = self.projectName + branch
3836 if fullBranch not in self.p4BranchesInGit:
3837 if not self.silent:
3838 print("\n Importing new branch %s" % fullBranch)
3839 if self.importNewBranch(branch, change - 1):
3840 parent = ""
3841 self.p4BranchesInGit.append(fullBranch)
3842 if not self.silent:
3843 print("\n Resuming with change %s" % change)
3844
3845 if self.verbose:
3846 print("parent determined through known branches: %s" % parent)
3847
3848 branch = self.gitRefForBranch(branch)
3849 parent = self.gitRefForBranch(parent)
3850
3851 if self.verbose:
3852 print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3853
3854 if len(parent) == 0 and branch in self.initialParents:
3855 parent = self.initialParents[branch]
3856 del self.initialParents[branch]
3857
3858 blob = None
3859 if len(parent) > 0:
3860 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3861 if self.verbose:
3862 print("Creating temporary branch: " + tempBranch)
3863 self.commit(description, filesForCommit, tempBranch)
3864 self.tempBranches.append(tempBranch)
3865 self.checkpoint()
3866 blob = self.searchParent(parent, branch, tempBranch)
3867 if blob:
3868 self.commit(description, filesForCommit, branch, blob)
3869 else:
3870 if self.verbose:
3871 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3872 self.commit(description, filesForCommit, branch, parent)
3873 else:
3874 files = self.extractFilesFromCommit(description)
3875 self.commit(description, files, self.branch,
3876 self.initialParent)
3877 # only needed once, to connect to the previous commit
3878 self.initialParent = ""
3879 except IOError:
3880 print(self.gitError.read())
3881 sys.exit(1)
3882
3883 def sync_origin_only(self):
3884 if self.syncWithOrigin:
3885 self.hasOrigin = originP4BranchesExist()
3886 if self.hasOrigin:
3887 if not self.silent:
3888 print('Syncing with origin first, using "git fetch origin"')
3889 system(["git", "fetch", "origin"])
3890
3891 def importHeadRevision(self, revision):
3892 print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3893
3894 details = {}
3895 details["user"] = "git perforce import user"
3896 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3897 % (' '.join(self.depotPaths), revision))
3898 details["change"] = revision
3899 newestRevision = 0
3900
3901 fileCnt = 0
3902 fileArgs = ["%s...%s" % (p, revision) for p in self.depotPaths]
3903
3904 for info in p4CmdList(["files"] + fileArgs):
3905
3906 if 'code' in info and info['code'] == 'error':
3907 sys.stderr.write("p4 returned an error: %s\n"
3908 % info['data'])
3909 if info['data'].find("must refer to client") >= 0:
3910 sys.stderr.write("This particular p4 error is misleading.\n")
3911 sys.stderr.write("Perhaps the depot path was misspelled.\n")
3912 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3913 sys.exit(1)
3914 if 'p4ExitCode' in info:
3915 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3916 sys.exit(1)
3917
3918 change = int(info["change"])
3919 if change > newestRevision:
3920 newestRevision = change
3921
3922 if info["action"] in self.delete_actions:
3923 continue
3924
3925 for prop in ["depotFile", "rev", "action", "type"]:
3926 details["%s%s" % (prop, fileCnt)] = info[prop]
3927
3928 fileCnt = fileCnt + 1
3929
3930 details["change"] = newestRevision
3931
3932 # Use time from top-most change so that all git p4 clones of
3933 # the same p4 repo have the same commit SHA1s.
3934 res = p4_describe(newestRevision)
3935 details["time"] = res["time"]
3936
3937 self.updateOptionDict(details)
3938 try:
3939 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3940 except IOError as err:
3941 print("IO error with git fast-import. Is your git version recent enough?")
3942 print("IO error details: {}".format(err))
3943 print(self.gitError.read())
3944
3945 def importRevisions(self, args, branch_arg_given):
3946 changes = []
3947
3948 if len(self.changesFile) > 0:
3949 with open(self.changesFile) as f:
3950 output = f.readlines()
3951 changeSet = set()
3952 for line in output:
3953 changeSet.add(int(line))
3954
3955 for change in changeSet:
3956 changes.append(change)
3957
3958 changes.sort()
3959 else:
3960 # catch "git p4 sync" with no new branches, in a repo that
3961 # does not have any existing p4 branches
3962 if len(args) == 0:
3963 if not self.p4BranchesInGit:
3964 raise P4CommandException("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3965
3966 # The default branch is master, unless --branch is used to
3967 # specify something else. Make sure it exists, or complain
3968 # nicely about how to use --branch.
3969 if not self.detectBranches:
3970 if not branch_exists(self.branch):
3971 if branch_arg_given:
3972 raise P4CommandException("Error: branch %s does not exist." % self.branch)
3973 else:
3974 raise P4CommandException("Error: no branch %s; perhaps specify one with --branch." %
3975 self.branch)
3976
3977 if self.verbose:
3978 print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3979 self.changeRange))
3980 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3981
3982 if len(self.maxChanges) > 0:
3983 changes = changes[:min(int(self.maxChanges), len(changes))]
3984
3985 if len(changes) == 0:
3986 if not self.silent:
3987 print("No changes to import!")
3988 else:
3989 if not self.silent and not self.detectBranches:
3990 print("Import destination: %s" % self.branch)
3991
3992 self.updatedBranches = set()
3993
3994 if not self.detectBranches:
3995 if args:
3996 # start a new branch
3997 self.initialParent = ""
3998 else:
3999 # build on a previous revision
4000 self.initialParent = parseRevision(self.branch)
4001
4002 self.importChanges(changes)
4003
4004 if not self.silent:
4005 print("")
4006 if len(self.updatedBranches) > 0:
4007 sys.stdout.write("Updated branches: ")
4008 for b in self.updatedBranches:
4009 sys.stdout.write("%s " % b)
4010 sys.stdout.write("\n")
4011
4012 def openStreams(self):
4013 self.importProcess = subprocess.Popen(["git", "fast-import"],
4014 stdin=subprocess.PIPE,
4015 stdout=subprocess.PIPE,
4016 stderr=subprocess.PIPE)
4017 self.gitOutput = self.importProcess.stdout
4018 self.gitStream = self.importProcess.stdin
4019 self.gitError = self.importProcess.stderr
4020
4021 if bytes is not str:
4022 # Wrap gitStream.write() so that it can be called using `str` arguments
4023 def make_encoded_write(write):
4024 def encoded_write(s):
4025 return write(s.encode() if isinstance(s, str) else s)
4026 return encoded_write
4027
4028 self.gitStream.write = make_encoded_write(self.gitStream.write)
4029
4030 def closeStreams(self):
4031 if self.gitStream is None:
4032 return
4033 self.gitStream.close()
4034 if self.importProcess.wait() != 0:
4035 die("fast-import failed: %s" % self.gitError.read())
4036 self.gitOutput.close()
4037 self.gitError.close()
4038 self.gitStream = None
4039
4040 def run(self, args):
4041 if self.importIntoRemotes:
4042 self.refPrefix = "refs/remotes/p4/"
4043 else:
4044 self.refPrefix = "refs/heads/p4/"
4045
4046 self.sync_origin_only()
4047
4048 branch_arg_given = bool(self.branch)
4049 if len(self.branch) == 0:
4050 self.branch = self.refPrefix + "master"
4051 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
4052 system(["git", "update-ref", self.branch, "refs/heads/p4"])
4053 system(["git", "branch", "-D", "p4"])
4054
4055 # accept either the command-line option, or the configuration variable
4056 if self.useClientSpec:
4057 # will use this after clone to set the variable
4058 self.useClientSpec_from_options = True
4059 else:
4060 if gitConfigBool("git-p4.useclientspec"):
4061 self.useClientSpec = True
4062 if self.useClientSpec:
4063 self.clientSpecDirs = getClientSpec()
4064
4065 # TODO: should always look at previous commits,
4066 # merge with previous imports, if possible.
4067 if args == []:
4068 if self.hasOrigin:
4069 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
4070
4071 # branches holds mapping from branch name to sha1
4072 branches = p4BranchesInGit(self.importIntoRemotes)
4073
4074 # restrict to just this one, disabling detect-branches
4075 if branch_arg_given:
4076 short = shortP4Ref(self.branch, self.importIntoRemotes)
4077 if short in branches:
4078 self.p4BranchesInGit = [short]
4079 elif self.branch.startswith('refs/') and \
4080 branchExists(self.branch) and \
4081 '[git-p4:' in extractLogMessageFromGitCommit(self.branch):
4082 self.p4BranchesInGit = [self.branch]
4083 else:
4084 self.p4BranchesInGit = branches.keys()
4085
4086 if len(self.p4BranchesInGit) > 1:
4087 if not self.silent:
4088 print("Importing from/into multiple branches")
4089 self.detectBranches = True
4090 for branch in branches.keys():
4091 self.initialParents[self.refPrefix + branch] = \
4092 branches[branch]
4093
4094 if self.verbose:
4095 print("branches: %s" % self.p4BranchesInGit)
4096
4097 p4Change = 0
4098 for branch in self.p4BranchesInGit:
4099 logMsg = extractLogMessageFromGitCommit(fullP4Ref(branch,
4100 self.importIntoRemotes))
4101
4102 settings = extractSettingsGitLog(logMsg)
4103
4104 self.readOptions(settings)
4105 if 'depot-paths' in settings and 'change' in settings:
4106 change = int(settings['change']) + 1
4107 p4Change = max(p4Change, change)
4108
4109 depotPaths = sorted(settings['depot-paths'])
4110 if self.previousDepotPaths == []:
4111 self.previousDepotPaths = depotPaths
4112 else:
4113 paths = []
4114 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
4115 prev_list = prev.split("/")
4116 cur_list = cur.split("/")
4117 for i in range(0, min(len(cur_list), len(prev_list))):
4118 if cur_list[i] != prev_list[i]:
4119 i = i - 1
4120 break
4121
4122 paths.append("/".join(cur_list[:i + 1]))
4123
4124 self.previousDepotPaths = paths
4125
4126 if p4Change > 0:
4127 self.depotPaths = sorted(self.previousDepotPaths)
4128 self.changeRange = "@%s,#head" % p4Change
4129 if not self.silent and not self.detectBranches:
4130 print("Performing incremental import into %s git branch" % self.branch)
4131
4132 self.branch = fullP4Ref(self.branch, self.importIntoRemotes)
4133
4134 if len(args) == 0 and self.depotPaths:
4135 if not self.silent:
4136 print("Depot paths: %s" % ' '.join(self.depotPaths))
4137 else:
4138 if self.depotPaths and self.depotPaths != args:
4139 print("previous import used depot path %s and now %s was specified. "
4140 "This doesn't work!" % (' '.join(self.depotPaths),
4141 ' '.join(args)))
4142 sys.exit(1)
4143
4144 self.depotPaths = sorted(args)
4145
4146 revision = ""
4147 self.users = {}
4148
4149 # Make sure no revision specifiers are used when --changesfile
4150 # is specified.
4151 bad_changesfile = False
4152 if len(self.changesFile) > 0:
4153 for p in self.depotPaths:
4154 if p.find("@") >= 0 or p.find("#") >= 0:
4155 bad_changesfile = True
4156 break
4157 if bad_changesfile:
4158 die("Option --changesfile is incompatible with revision specifiers")
4159
4160 newPaths = []
4161 for p in self.depotPaths:
4162 if p.find("@") != -1:
4163 atIdx = p.index("@")
4164 self.changeRange = p[atIdx:]
4165 if self.changeRange == "@all":
4166 self.changeRange = ""
4167 elif ',' not in self.changeRange:
4168 revision = self.changeRange
4169 self.changeRange = ""
4170 p = p[:atIdx]
4171 elif p.find("#") != -1:
4172 hashIdx = p.index("#")
4173 revision = p[hashIdx:]
4174 p = p[:hashIdx]
4175 elif self.previousDepotPaths == []:
4176 # pay attention to changesfile, if given, else import
4177 # the entire p4 tree at the head revision
4178 if len(self.changesFile) == 0:
4179 revision = "#head"
4180
4181 p = re.sub("\.\.\.$", "", p)
4182 if not p.endswith("/"):
4183 p += "/"
4184
4185 newPaths.append(p)
4186
4187 self.depotPaths = newPaths
4188
4189 # --detect-branches may change this for each branch
4190 self.branchPrefixes = self.depotPaths
4191
4192 self.loadUserMapFromCache()
4193 self.labels = {}
4194 if self.detectLabels:
4195 self.getLabels()
4196
4197 if self.detectBranches:
4198 # FIXME - what's a P4 projectName ?
4199 self.projectName = self.guessProjectName()
4200
4201 if self.hasOrigin:
4202 self.getBranchMappingFromGitBranches()
4203 else:
4204 self.getBranchMapping()
4205 if self.verbose:
4206 print("p4-git branches: %s" % self.p4BranchesInGit)
4207 print("initial parents: %s" % self.initialParents)
4208 for b in self.p4BranchesInGit:
4209 if b != "master":
4210
4211 # FIXME
4212 b = b[len(self.projectName):]
4213 self.createdBranches.add(b)
4214
4215 p4_check_access()
4216
4217 self.openStreams()
4218
4219 err = None
4220
4221 try:
4222 if revision:
4223 self.importHeadRevision(revision)
4224 else:
4225 self.importRevisions(args, branch_arg_given)
4226
4227 if gitConfigBool("git-p4.importLabels"):
4228 self.importLabels = True
4229
4230 if self.importLabels:
4231 p4Labels = getP4Labels(self.depotPaths)
4232 gitTags = getGitTags()
4233
4234 missingP4Labels = p4Labels - gitTags
4235 self.importP4Labels(self.gitStream, missingP4Labels)
4236
4237 except P4CommandException as e:
4238 err = e
4239
4240 finally:
4241 self.closeStreams()
4242
4243 if err:
4244 die(str(err))
4245
4246 # Cleanup temporary branches created during import
4247 if self.tempBranches != []:
4248 for branch in self.tempBranches:
4249 read_pipe(["git", "update-ref", "-d", branch])
4250 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
4251
4252 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
4253 # a convenient shortcut refname "p4".
4254 if self.importIntoRemotes:
4255 head_ref = self.refPrefix + "HEAD"
4256 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
4257 system(["git", "symbolic-ref", head_ref, self.branch])
4258
4259 return True
4260
4261
4262 class P4Rebase(Command):
4263 def __init__(self):
4264 Command.__init__(self)
4265 self.options = [
4266 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
4267 ]
4268 self.importLabels = False
4269 self.description = ("Fetches the latest revision from perforce and "
4270 + "rebases the current work (branch) against it")
4271
4272 def run(self, args):
4273 sync = P4Sync()
4274 sync.importLabels = self.importLabels
4275 sync.run([])
4276
4277 return self.rebase()
4278
4279 def rebase(self):
4280 if os.system("git update-index --refresh") != 0:
4281 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.")
4282 if len(read_pipe(["git", "diff-index", "HEAD", "--"])) > 0:
4283 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.")
4284
4285 upstream, settings = findUpstreamBranchPoint()
4286 if len(upstream) == 0:
4287 die("Cannot find upstream branchpoint for rebase")
4288
4289 # the branchpoint may be p4/foo~3, so strip off the parent
4290 upstream = re.sub("~[0-9]+$", "", upstream)
4291
4292 print("Rebasing the current branch onto %s" % upstream)
4293 oldHead = read_pipe(["git", "rev-parse", "HEAD"]).strip()
4294 system(["git", "rebase", upstream])
4295 system(["git", "diff-tree", "--stat", "--summary", "-M", oldHead,
4296 "HEAD", "--"])
4297 return True
4298
4299
4300 class P4Clone(P4Sync):
4301 def __init__(self):
4302 P4Sync.__init__(self)
4303 self.description = "Creates a new git repository and imports from Perforce into it"
4304 self.usage = "usage: %prog [options] //depot/path[@revRange]"
4305 self.options += [
4306 optparse.make_option("--destination", dest="cloneDestination",
4307 action='store', default=None,
4308 help="where to leave result of the clone"),
4309 optparse.make_option("--bare", dest="cloneBare",
4310 action="store_true", default=False),
4311 ]
4312 self.cloneDestination = None
4313 self.needsGit = False
4314 self.cloneBare = False
4315
4316 def defaultDestination(self, args):
4317 # TODO: use common prefix of args?
4318 depotPath = args[0]
4319 depotDir = re.sub("(@[^@]*)$", "", depotPath)
4320 depotDir = re.sub("(#[^#]*)$", "", depotDir)
4321 depotDir = re.sub(r"\.\.\.$", "", depotDir)
4322 depotDir = re.sub(r"/$", "", depotDir)
4323 return os.path.split(depotDir)[1]
4324
4325 def run(self, args):
4326 if len(args) < 1:
4327 return False
4328
4329 if self.keepRepoPath and not self.cloneDestination:
4330 sys.stderr.write("Must specify destination for --keep-path\n")
4331 sys.exit(1)
4332
4333 depotPaths = args
4334
4335 if not self.cloneDestination and len(depotPaths) > 1:
4336 self.cloneDestination = depotPaths[-1]
4337 depotPaths = depotPaths[:-1]
4338
4339 for p in depotPaths:
4340 if not p.startswith("//"):
4341 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
4342 return False
4343
4344 if not self.cloneDestination:
4345 self.cloneDestination = self.defaultDestination(args)
4346
4347 print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
4348
4349 if not os.path.exists(self.cloneDestination):
4350 os.makedirs(self.cloneDestination)
4351 chdir(self.cloneDestination)
4352
4353 init_cmd = ["git", "init"]
4354 if self.cloneBare:
4355 init_cmd.append("--bare")
4356 retcode = subprocess.call(init_cmd)
4357 if retcode:
4358 raise subprocess.CalledProcessError(retcode, init_cmd)
4359
4360 if not P4Sync.run(self, depotPaths):
4361 return False
4362
4363 # create a master branch and check out a work tree
4364 if gitBranchExists(self.branch):
4365 system(["git", "branch", currentGitBranch(), self.branch])
4366 if not self.cloneBare:
4367 system(["git", "checkout", "-f"])
4368 else:
4369 print('Not checking out any branch, use '
4370 '"git checkout -q -b master <branch>"')
4371
4372 # auto-set this variable if invoked with --use-client-spec
4373 if self.useClientSpec_from_options:
4374 system(["git", "config", "--bool", "git-p4.useclientspec", "true"])
4375
4376 # persist any git-p4 encoding-handling config options passed in for clone:
4377 if gitConfig('git-p4.metadataDecodingStrategy'):
4378 system(["git", "config", "git-p4.metadataDecodingStrategy", gitConfig('git-p4.metadataDecodingStrategy')])
4379 if gitConfig('git-p4.metadataFallbackEncoding'):
4380 system(["git", "config", "git-p4.metadataFallbackEncoding", gitConfig('git-p4.metadataFallbackEncoding')])
4381 if gitConfig('git-p4.pathEncoding'):
4382 system(["git", "config", "git-p4.pathEncoding", gitConfig('git-p4.pathEncoding')])
4383
4384 return True
4385
4386
4387 class P4Unshelve(Command):
4388 def __init__(self):
4389 Command.__init__(self)
4390 self.options = []
4391 self.origin = "HEAD"
4392 self.description = "Unshelve a P4 changelist into a git commit"
4393 self.usage = "usage: %prog [options] changelist"
4394 self.options += [
4395 optparse.make_option("--origin", dest="origin",
4396 help="Use this base revision instead of the default (%s)" % self.origin),
4397 ]
4398 self.verbose = False
4399 self.noCommit = False
4400 self.destbranch = "refs/remotes/p4-unshelved"
4401
4402 def renameBranch(self, branch_name):
4403 """Rename the existing branch to branch_name.N ."""
4404
4405 for i in range(0, 1000):
4406 backup_branch_name = "{0}.{1}".format(branch_name, i)
4407 if not gitBranchExists(backup_branch_name):
4408 # Copy ref to backup
4409 gitUpdateRef(backup_branch_name, branch_name)
4410 gitDeleteRef(branch_name)
4411 print("renamed old unshelve branch to {0}".format(backup_branch_name))
4412 break
4413 else:
4414 sys.exit("gave up trying to rename existing branch {0}".format(branch_name))
4415
4416 def findLastP4Revision(self, starting_point):
4417 """Look back from starting_point for the first commit created by git-p4
4418 to find the P4 commit we are based on, and the depot-paths.
4419 """
4420
4421 for parent in (range(65535)):
4422 log = extractLogMessageFromGitCommit("{0}~{1}".format(starting_point, parent))
4423 settings = extractSettingsGitLog(log)
4424 if 'change' in settings:
4425 return settings
4426
4427 sys.exit("could not find git-p4 commits in {0}".format(self.origin))
4428
4429 def createShelveParent(self, change, branch_name, sync, origin):
4430 """Create a commit matching the parent of the shelved changelist
4431 'change'.
4432 """
4433 parent_description = p4_describe(change, shelved=True)
4434 parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change)
4435 files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)
4436
4437 parent_files = []
4438 for f in files:
4439 # if it was added in the shelved changelist, it won't exist in the parent
4440 if f['action'] in self.add_actions:
4441 continue
4442
4443 # if it was deleted in the shelved changelist it must not be deleted
4444 # in the parent - we might even need to create it if the origin branch
4445 # does not have it
4446 if f['action'] in self.delete_actions:
4447 f['action'] = 'add'
4448
4449 parent_files.append(f)
4450
4451 sync.commit(parent_description, parent_files, branch_name,
4452 parent=origin, allow_empty=True)
4453 print("created parent commit for {0} based on {1} in {2}".format(
4454 change, self.origin, branch_name))
4455
4456 def run(self, args):
4457 if len(args) != 1:
4458 return False
4459
4460 if not gitBranchExists(self.origin):
4461 sys.exit("origin branch {0} does not exist".format(self.origin))
4462
4463 sync = P4Sync()
4464 changes = args
4465
4466 # only one change at a time
4467 change = changes[0]
4468
4469 # if the target branch already exists, rename it
4470 branch_name = "{0}/{1}".format(self.destbranch, change)
4471 if gitBranchExists(branch_name):
4472 self.renameBranch(branch_name)
4473 sync.branch = branch_name
4474
4475 sync.verbose = self.verbose
4476 sync.suppress_meta_comment = True
4477
4478 settings = self.findLastP4Revision(self.origin)
4479 sync.depotPaths = settings['depot-paths']
4480 sync.branchPrefixes = sync.depotPaths
4481
4482 sync.openStreams()
4483 sync.loadUserMapFromCache()
4484 sync.silent = True
4485
4486 # create a commit for the parent of the shelved changelist
4487 self.createShelveParent(change, branch_name, sync, self.origin)
4488
4489 # create the commit for the shelved changelist itself
4490 description = p4_describe(change, True)
4491 files = sync.extractFilesFromCommit(description, True, change)
4492
4493 sync.commit(description, files, branch_name, "")
4494 sync.closeStreams()
4495
4496 print("unshelved changelist {0} into {1}".format(change, branch_name))
4497
4498 return True
4499
4500
4501 class P4Branches(Command):
4502 def __init__(self):
4503 Command.__init__(self)
4504 self.options = []
4505 self.description = ("Shows the git branches that hold imports and their "
4506 + "corresponding perforce depot paths")
4507 self.verbose = False
4508
4509 def run(self, args):
4510 if originP4BranchesExist():
4511 createOrUpdateBranchesFromOrigin()
4512
4513 for line in read_pipe_lines(["git", "rev-parse", "--symbolic", "--remotes"]):
4514 line = line.strip()
4515
4516 if not line.startswith('p4/') or line == "p4/HEAD":
4517 continue
4518 branch = line
4519
4520 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4521 settings = extractSettingsGitLog(log)
4522
4523 print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4524 return True
4525
4526
4527 class HelpFormatter(optparse.IndentedHelpFormatter):
4528 def __init__(self):
4529 optparse.IndentedHelpFormatter.__init__(self)
4530
4531 def format_description(self, description):
4532 if description:
4533 return description + "\n"
4534 else:
4535 return ""
4536
4537
4538 def printUsage(commands):
4539 print("usage: %s <command> [options]" % sys.argv[0])
4540 print("")
4541 print("valid commands: %s" % ", ".join(commands))
4542 print("")
4543 print("Try %s <command> --help for command specific help." % sys.argv[0])
4544 print("")
4545
4546
4547 commands = {
4548 "submit": P4Submit,
4549 "commit": P4Submit,
4550 "sync": P4Sync,
4551 "rebase": P4Rebase,
4552 "clone": P4Clone,
4553 "branches": P4Branches,
4554 "unshelve": P4Unshelve,
4555 }
4556
4557
4558 def main():
4559 if len(sys.argv[1:]) == 0:
4560 printUsage(commands.keys())
4561 sys.exit(2)
4562
4563 cmdName = sys.argv[1]
4564 try:
4565 klass = commands[cmdName]
4566 cmd = klass()
4567 except KeyError:
4568 print("unknown command %s" % cmdName)
4569 print("")
4570 printUsage(commands.keys())
4571 sys.exit(2)
4572
4573 options = cmd.options
4574 cmd.gitdir = os.environ.get("GIT_DIR", None)
4575
4576 args = sys.argv[2:]
4577
4578 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4579 if cmd.needsGit:
4580 options.append(optparse.make_option("--git-dir", dest="gitdir"))
4581
4582 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4583 options,
4584 description=cmd.description,
4585 formatter=HelpFormatter())
4586
4587 try:
4588 cmd, args = parser.parse_args(sys.argv[2:], cmd)
4589 except:
4590 parser.print_help()
4591 raise
4592
4593 global verbose
4594 verbose = cmd.verbose
4595 if cmd.needsGit:
4596 if cmd.gitdir is None:
4597 cmd.gitdir = os.path.abspath(".git")
4598 if not isValidGitDir(cmd.gitdir):
4599 # "rev-parse --git-dir" without arguments will try $PWD/.git
4600 cmd.gitdir = read_pipe(["git", "rev-parse", "--git-dir"]).strip()
4601 if os.path.exists(cmd.gitdir):
4602 cdup = read_pipe(["git", "rev-parse", "--show-cdup"]).strip()
4603 if len(cdup) > 0:
4604 chdir(cdup)
4605
4606 if not isValidGitDir(cmd.gitdir):
4607 if isValidGitDir(cmd.gitdir + "/.git"):
4608 cmd.gitdir += "/.git"
4609 else:
4610 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4611
4612 # so git commands invoked from the P4 workspace will succeed
4613 os.environ["GIT_DIR"] = cmd.gitdir
4614
4615 if not cmd.run(args):
4616 parser.print_help()
4617 sys.exit(2)
4618
4619
4620 if __name__ == '__main__':
4621 main()