]> git.ipfire.org Git - thirdparty/git.git/blame - git-p4.py
Merge branch 'jk/clone-allow-bare-and-o-together'
[thirdparty/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
794bb28d
JH
10# pylint: disable=bad-whitespace
11# pylint: disable=broad-except
12# pylint: disable=consider-iterating-dictionary
13# pylint: disable=disable
14# pylint: disable=fixme
15# pylint: disable=invalid-name
16# pylint: disable=line-too-long
17# pylint: disable=missing-docstring
18# pylint: disable=no-self-use
19# pylint: disable=superfluous-parens
20# pylint: disable=too-few-public-methods
21# pylint: disable=too-many-arguments
22# pylint: disable=too-many-branches
23# pylint: disable=too-many-instance-attributes
24# pylint: disable=too-many-lines
25# pylint: disable=too-many-locals
26# pylint: disable=too-many-nested-blocks
27# pylint: disable=too-many-statements
28# pylint: disable=ungrouped-imports
29# pylint: disable=unused-import
30# pylint: disable=wrong-import-order
31# pylint: disable=wrong-import-position
4c1d5867 32#
4ff0108d 33
f7b5ff60 34import struct
a33faf28 35import sys
0b4396f0
YZ
36if sys.version_info.major < 3 and sys.version_info.minor < 7:
37 sys.stderr.write("git-p4: requires Python 2.7 or later.\n")
a33faf28 38 sys.exit(1)
4ff0108d
JH
39
40import ctypes
41import errno
a6b13067 42import functools
4ff0108d 43import glob
f629fa59 44import marshal
4ff0108d
JH
45import optparse
46import os
f629fa59
PW
47import platform
48import re
49import shutil
d20f0f8e 50import stat
4ff0108d
JH
51import subprocess
52import tempfile
53import time
a5db4b12
LS
54import zipfile
55import zlib
8b41a97f 56
7575f4fd
YZ
57# On python2.7 where raw_input() and input() are both availble,
58# we want raw_input's semantics, but aliased to input for python3
59# compatibility
efdcc992
LD
60# support basestring in python3
61try:
7575f4fd
YZ
62 if raw_input and input:
63 input = raw_input
64except:
65 pass
a235e85c 66
4addad22 67verbose = False
86949eef 68
06804c76 69# Only labels/tags matching this will be imported/exported
c8942a22 70defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 71
3deed5e0 72# The block size is reduced automatically if required
6febb9f8 73defaultBlockSize = 1 << 20
1051ef00 74
f7b5ff60
TK
75defaultMetadataDecodingStrategy = 'passthrough' if sys.version_info.major == 2 else 'fallback'
76defaultFallbackMetadataEncoding = 'cp1252'
77
0ef67acd 78p4_access_checked = False
1051ef00 79
70c0d553
JH
80re_ko_keywords = re.compile(br'\$(Id|Header)(:[^$\n]+)?\$')
81re_k_keywords = re.compile(br'\$(Id|Header|Author|Date|DateTime|Change|File|Revision)(:[^$\n]+)?\$')
e665e98e 82
adf159b4 83
ae9b9509 84def format_size_human_readable(num):
59ef3fc1
JH
85 """Returns a number of units (typically bytes) formatted as a
86 human-readable string.
87 """
ae9b9509
JH
88 if num < 1024:
89 return '{:d} B'.format(num)
90 for unit in ["Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
91 num /= 1024.0
92 if num < 1024.0:
93 return "{:3.1f} {}B".format(num, unit)
94 return "{:.1f} YiB".format(num)
95
adf159b4 96
21a50753
AK
97def p4_build_cmd(cmd):
98 """Build a suitable p4 command line.
99
59ef3fc1
JH
100 This consolidates building and returning a p4 command line into one
101 location. It means that hooking into the environment, or other
102 configuration can be done more easily.
103 """
6de040df 104 real_cmd = ["p4"]
abcaf073
AK
105
106 user = gitConfig("git-p4.user")
107 if len(user) > 0:
12a77f5b 108 real_cmd += ["-u", user]
abcaf073
AK
109
110 password = gitConfig("git-p4.password")
111 if len(password) > 0:
6de040df 112 real_cmd += ["-P", password]
abcaf073
AK
113
114 port = gitConfig("git-p4.port")
115 if len(port) > 0:
6de040df 116 real_cmd += ["-p", port]
abcaf073
AK
117
118 host = gitConfig("git-p4.host")
119 if len(host) > 0:
41799aa2 120 real_cmd += ["-H", host]
abcaf073
AK
121
122 client = gitConfig("git-p4.client")
123 if len(client) > 0:
6de040df 124 real_cmd += ["-c", client]
abcaf073 125
89a6ecc5
LS
126 retries = gitConfigInt("git-p4.retries")
127 if retries is None:
128 # Perform 3 retries by default
129 retries = 3
bc233524
IK
130 if retries > 0:
131 # Provide a way to not pass this option by setting git-p4.retries to 0
132 real_cmd += ["-r", str(retries)]
6de040df 133
8a470599 134 real_cmd += cmd
0ef67acd
LD
135
136 # now check that we can actually talk to the server
137 global p4_access_checked
138 if not p4_access_checked:
139 p4_access_checked = True # suppress access checks in p4_check_access itself
140 p4_check_access()
141
21a50753
AK
142 return real_cmd
143
adf159b4 144
378f7be1 145def git_dir(path):
59ef3fc1
JH
146 """Return TRUE if the given path is a git directory (/path/to/dir/.git).
147 This won't automatically add ".git" to a directory.
148 """
378f7be1
LD
149 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
150 if not d or len(d) == 0:
151 return None
152 else:
153 return d
154
adf159b4 155
bbd84863 156def chdir(path, is_client_path=False):
59ef3fc1
JH
157 """Do chdir to the given path, and set the PWD environment variable for use
158 by P4. It does not look at getcwd() output. Since we're not using the
159 shell, it is necessary to set the PWD environment variable explicitly.
160
161 Normally, expand the path to force it to be absolute. This addresses
162 the use of relative path names inside P4 settings, e.g.
163 P4CONFIG=.p4config. P4 does not simply open the filename as given; it
164 looks for .p4config using PWD.
165
166 If is_client_path, the path was handed to us directly by p4, and may be
167 a symbolic link. Do not call os.getcwd() in this case, because it will
168 cause p4 to think that PWD is not inside the client path.
bbd84863
MF
169 """
170
171 os.chdir(path)
172 if not is_client_path:
173 path = os.getcwd()
174 os.environ['PWD'] = path
053fd0c1 175
adf159b4 176
4d25dc44
LS
177def calcDiskFree():
178 """Return free space in bytes on the disk of the given dirname."""
179 if platform.system() == 'Windows':
180 free_bytes = ctypes.c_ulonglong(0)
181 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
182 return free_bytes.value
183 else:
184 st = os.statvfs(os.getcwd())
185 return st.f_bavail * st.f_frsize
186
adf159b4 187
86dff6b6 188def die(msg):
59ef3fc1
JH
189 """Terminate execution. Make sure that any running child processes have
190 been wait()ed for before calling this.
191 """
86dff6b6
HWN
192 if verbose:
193 raise Exception(msg)
194 else:
195 sys.stderr.write(msg + "\n")
196 sys.exit(1)
197
adf159b4 198
e2aed5fd 199def prompt(prompt_text):
59ef3fc1 200 """Prompt the user to choose one of the choices.
e2aed5fd 201
59ef3fc1
JH
202 Choices are identified in the prompt_text by square brackets around a
203 single letter option.
204 """
e2aed5fd
BK
205 choices = set(m.group(1) for m in re.finditer(r"\[(.)\]", prompt_text))
206 while True:
6b602a2f
BK
207 sys.stderr.flush()
208 sys.stdout.write(prompt_text)
209 sys.stdout.flush()
6febb9f8 210 response = sys.stdin.readline().strip().lower()
e2aed5fd
BK
211 if not response:
212 continue
213 response = response[0]
214 if response in choices:
215 return response
216
adf159b4 217
6cec21a8
YZ
218# We need different encoding/decoding strategies for text data being passed
219# around in pipes depending on python version
220if bytes is not str:
221 # For python3, always encode and decode as appropriate
222 def decode_text_stream(s):
223 return s.decode() if isinstance(s, bytes) else s
adf159b4 224
6cec21a8
YZ
225 def encode_text_stream(s):
226 return s.encode() if isinstance(s, str) else s
227else:
228 # For python2.7, pass read strings as-is, but also allow writing unicode
229 def decode_text_stream(s):
230 return s
adf159b4 231
6cec21a8
YZ
232 def encode_text_stream(s):
233 return s.encode('utf_8') if isinstance(s, unicode) else s
234
adf159b4 235
f7b5ff60
TK
236class MetadataDecodingException(Exception):
237 def __init__(self, input_string):
238 self.input_string = input_string
239
240 def __str__(self):
241 return """Decoding perforce metadata failed!
242The failing string was:
243---
244{}
245---
246Consider setting the git-p4.metadataDecodingStrategy config option to
247'fallback', to allow metadata to be decoded using a fallback encoding,
248defaulting to cp1252.""".format(self.input_string)
249
250
251encoding_fallback_warning_issued = False
252encoding_escape_warning_issued = False
253def metadata_stream_to_writable_bytes(s):
254 encodingStrategy = gitConfig('git-p4.metadataDecodingStrategy') or defaultMetadataDecodingStrategy
255 fallbackEncoding = gitConfig('git-p4.metadataFallbackEncoding') or defaultFallbackMetadataEncoding
256 if not isinstance(s, bytes):
257 return s.encode('utf_8')
258 if encodingStrategy == 'passthrough':
259 return s
260 try:
261 s.decode('utf_8')
262 return s
263 except UnicodeDecodeError:
264 if encodingStrategy == 'fallback' and fallbackEncoding:
265 global encoding_fallback_warning_issued
266 global encoding_escape_warning_issued
267 try:
268 if not encoding_fallback_warning_issued:
269 print("\nCould not decode value as utf-8; using configured fallback encoding %s: %s" % (fallbackEncoding, s))
270 print("\n(this warning is only displayed once during an import)")
271 encoding_fallback_warning_issued = True
272 return s.decode(fallbackEncoding).encode('utf_8')
273 except Exception as exc:
274 if not encoding_escape_warning_issued:
275 print("\nCould not decode value with configured fallback encoding %s; escaping bytes over 127: %s" % (fallbackEncoding, s))
276 print("\n(this warning is only displayed once during an import)")
277 encoding_escape_warning_issued = True
278 escaped_bytes = b''
279 # bytes and strings work very differently in python2 vs python3...
280 if str is bytes:
281 for byte in s:
282 byte_number = struct.unpack('>B', byte)[0]
283 if byte_number > 127:
284 escaped_bytes += b'%'
285 escaped_bytes += hex(byte_number)[2:].upper()
286 else:
287 escaped_bytes += byte
288 else:
289 for byte_number in s:
290 if byte_number > 127:
291 escaped_bytes += b'%'
292 escaped_bytes += hex(byte_number).upper().encode()[2:]
293 else:
294 escaped_bytes += bytes([byte_number])
295 return escaped_bytes
296
297 raise MetadataDecodingException(s)
298
3af1df04 299
d38208a2 300def decode_path(path):
59ef3fc1
JH
301 """Decode a given string (bytes or otherwise) using configured path
302 encoding options.
303 """
304
d38208a2
YZ
305 encoding = gitConfig('git-p4.pathEncoding') or 'utf_8'
306 if bytes is not str:
307 return path.decode(encoding, errors='replace') if isinstance(path, bytes) else path
308 else:
309 try:
310 path.decode('ascii')
311 except:
312 path = path.decode(encoding, errors='replace')
313 if verbose:
314 print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding, path))
315 return path
316
adf159b4 317
9f59ca4d
BK
318def run_git_hook(cmd, param=[]):
319 """Execute a hook if the hook exists."""
0c8ac06b
ES
320 args = ['git', 'hook', 'run', '--ignore-missing', cmd]
321 if param:
322 args.append("--")
323 for p in param:
324 args.append(p)
325 return subprocess.call(args) == 0
9f59ca4d 326
adf159b4 327
3d8a3038 328def write_pipe(c, stdin, *k, **kw):
4addad22 329 if verbose:
727e6ea3 330 sys.stderr.write('Writing pipe: {}\n'.format(' '.join(c)))
b016d397 331
3d8a3038 332 p = subprocess.Popen(c, stdin=subprocess.PIPE, *k, **kw)
6de040df
LD
333 pipe = p.stdin
334 val = pipe.write(stdin)
335 pipe.close()
336 if p.wait():
727e6ea3 337 die('Command failed: {}'.format(' '.join(c)))
b016d397
HWN
338
339 return val
340
adf159b4 341
3d8a3038 342def p4_write_pipe(c, stdin, *k, **kw):
d9429194 343 real_cmd = p4_build_cmd(c)
6cec21a8
YZ
344 if bytes is not str and isinstance(stdin, str):
345 stdin = encode_text_stream(stdin)
3d8a3038 346 return write_pipe(real_cmd, stdin, *k, **kw)
d9429194 347
adf159b4 348
3d8a3038 349def read_pipe_full(c, *k, **kw):
59ef3fc1
JH
350 """Read output from command. Returns a tuple of the return status, stdout
351 text and stderr text.
352 """
4addad22 353 if verbose:
727e6ea3 354 sys.stderr.write('Reading pipe: {}\n'.format(' '.join(c)))
8b41a97f 355
3d8a3038
JH
356 p = subprocess.Popen(
357 c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, *k, **kw)
0874bb01 358 out, err = p.communicate()
6cec21a8 359 return (p.returncode, out, decode_text_stream(err))
78871bf4 360
adf159b4 361
3d8a3038 362def read_pipe(c, ignore_error=False, raw=False, *k, **kw):
59ef3fc1
JH
363 """Read output from command. Returns the output text on success. On
364 failure, terminates execution, unless ignore_error is True, when it
365 returns an empty string.
86dca24b 366
59ef3fc1
JH
367 If raw is True, do not attempt to decode output text.
368 """
0874bb01 369 retcode, out, err = read_pipe_full(c, *k, **kw)
78871bf4
LD
370 if retcode != 0:
371 if ignore_error:
372 out = ""
373 else:
727e6ea3 374 die('Command failed: {}\nError: {}'.format(' '.join(c), err))
86dca24b
YZ
375 if not raw:
376 out = decode_text_stream(out)
1f5f3907 377 return out
b016d397 378
adf159b4 379
3d8a3038 380def read_pipe_text(c, *k, **kw):
59ef3fc1
JH
381 """Read output from a command with trailing whitespace stripped. On error,
382 returns None.
383 """
0874bb01 384 retcode, out, err = read_pipe_full(c, *k, **kw)
78871bf4
LD
385 if retcode != 0:
386 return None
387 else:
6cec21a8 388 return decode_text_stream(out).rstrip()
78871bf4 389
adf159b4 390
3d8a3038 391def p4_read_pipe(c, ignore_error=False, raw=False, *k, **kw):
d9429194 392 real_cmd = p4_build_cmd(c)
3d8a3038 393 return read_pipe(real_cmd, ignore_error, raw=raw, *k, **kw)
b016d397 394
adf159b4 395
3d8a3038 396def read_pipe_lines(c, raw=False, *k, **kw):
4addad22 397 if verbose:
727e6ea3 398 sys.stderr.write('Reading pipe: {}\n'.format(' '.join(c)))
6de040df 399
3d8a3038 400 p = subprocess.Popen(c, stdout=subprocess.PIPE, *k, **kw)
6de040df 401 pipe = p.stdout
9732e222
JH
402 lines = pipe.readlines()
403 if not raw:
404 lines = [decode_text_stream(line) for line in lines]
6de040df 405 if pipe.close() or p.wait():
727e6ea3 406 die('Command failed: {}'.format(' '.join(c)))
9732e222 407 return lines
caace111 408
adf159b4 409
3d8a3038 410def p4_read_pipe_lines(c, *k, **kw):
59ef3fc1 411 """Specifically invoke p4 on the command supplied."""
155af834 412 real_cmd = p4_build_cmd(c)
3d8a3038 413 return read_pipe_lines(real_cmd, *k, **kw)
2318121b 414
adf159b4 415
8e9497c2 416def p4_has_command(cmd):
59ef3fc1
JH
417 """Ask p4 for help on this command. If it returns an error, the command
418 does not exist in this version of p4.
419 """
8e9497c2
GG
420 real_cmd = p4_build_cmd(["help", cmd])
421 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
422 stderr=subprocess.PIPE)
423 p.communicate()
424 return p.returncode == 0
425
adf159b4 426
249da4c0 427def p4_has_move_command():
59ef3fc1
JH
428 """See if the move command exists, that it supports -k, and that it has not
429 been administratively disabled. The arguments must be correct, but the
430 filenames do not have to exist. Use ones with wildcards so even if they
431 exist, it will fail.
432 """
249da4c0
PW
433
434 if not p4_has_command("move"):
435 return False
436 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
437 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
0874bb01 438 out, err = p.communicate()
6cec21a8 439 err = decode_text_stream(err)
249da4c0
PW
440 # return code will be 1 in either case
441 if err.find("Invalid option") >= 0:
442 return False
443 if err.find("disabled") >= 0:
444 return False
445 # assume it failed because @... was invalid changelist
446 return True
447
adf159b4 448
3d8a3038 449def system(cmd, ignore_error=False, *k, **kw):
4addad22 450 if verbose:
727e6ea3
JH
451 sys.stderr.write("executing {}\n".format(
452 ' '.join(cmd) if isinstance(cmd, list) else cmd))
3d8a3038 453 retcode = subprocess.call(cmd, *k, **kw)
cbff4b25 454 if retcode and not ignore_error:
40e7cfdd 455 raise subprocess.CalledProcessError(retcode, cmd)
6754a299 456
cbff4b25
LD
457 return retcode
458
adf159b4 459
3d8a3038 460def p4_system(cmd, *k, **kw):
59ef3fc1 461 """Specifically invoke p4 as the system command."""
155af834 462 real_cmd = p4_build_cmd(cmd)
3d8a3038 463 retcode = subprocess.call(real_cmd, *k, **kw)
a235e85c 464 if retcode:
40e7cfdd 465 raise subprocess.CalledProcessError(retcode, real_cmd)
6de040df 466
adf159b4 467
0ef67acd
LD
468def die_bad_access(s):
469 die("failure accessing depot: {0}".format(s.rstrip()))
470
adf159b4 471
0ef67acd 472def p4_check_access(min_expiration=1):
59ef3fc1
JH
473 """Check if we can access Perforce - account still logged in."""
474
0ef67acd
LD
475 results = p4CmdList(["login", "-s"])
476
477 if len(results) == 0:
478 # should never get here: always get either some results, or a p4ExitCode
479 assert("could not parse response from perforce")
480
481 result = results[0]
482
483 if 'p4ExitCode' in result:
484 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
485 die_bad_access("could not run p4")
486
487 code = result.get("code")
488 if not code:
489 # we get here if we couldn't connect and there was nothing to unmarshal
490 die_bad_access("could not connect")
491
492 elif code == "stat":
493 expiry = result.get("TicketExpiration")
494 if expiry:
495 expiry = int(expiry)
496 if expiry > min_expiration:
497 # ok to carry on
498 return
499 else:
500 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
501
502 else:
503 # account without a timeout - all ok
504 return
505
506 elif code == "error":
507 data = result.get("data")
508 if data:
509 die_bad_access("p4 error: {0}".format(data))
510 else:
511 die_bad_access("unknown error")
d4990d56
PO
512 elif code == "info":
513 return
0ef67acd
LD
514 else:
515 die_bad_access("unknown error code {0}".format(code))
516
adf159b4 517
7f0e5962 518_p4_version_string = None
adf159b4
JH
519
520
7f0e5962 521def p4_version_string():
59ef3fc1
JH
522 """Read the version string, showing just the last line, which hopefully is
523 the interesting version bit.
7f0e5962
PW
524
525 $ p4 -V
526 Perforce - The Fast Software Configuration Management System.
527 Copyright 1995-2011 Perforce Software. All rights reserved.
528 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
59ef3fc1 529 """
7f0e5962
PW
530 global _p4_version_string
531 if not _p4_version_string:
532 a = p4_read_pipe_lines(["-V"])
533 _p4_version_string = a[-1].rstrip()
534 return _p4_version_string
535
adf159b4 536
6de040df 537def p4_integrate(src, dest):
9d7d446a 538 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 539
adf159b4 540
8d7ec362 541def p4_sync(f, *options):
9d7d446a 542 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df 543
adf159b4 544
6de040df 545def p4_add(f):
522e914f 546 """Forcibly add file names with wildcards."""
9d7d446a
PW
547 if wildcard_present(f):
548 p4_system(["add", "-f", f])
549 else:
550 p4_system(["add", f])
6de040df 551
adf159b4 552
6de040df 553def p4_delete(f):
9d7d446a 554 p4_system(["delete", wildcard_encode(f)])
6de040df 555
adf159b4 556
a02b8bc4
RP
557def p4_edit(f, *options):
558 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
6de040df 559
adf159b4 560
6de040df 561def p4_revert(f):
9d7d446a 562 p4_system(["revert", wildcard_encode(f)])
6de040df 563
adf159b4 564
9d7d446a
PW
565def p4_reopen(type, f):
566 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 567
adf159b4 568
46c609e9
LD
569def p4_reopen_in_change(changelist, files):
570 cmd = ["reopen", "-c", str(changelist)] + files
571 p4_system(cmd)
572
adf159b4 573
8e9497c2
GG
574def p4_move(src, dest):
575 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
576
adf159b4 577
1051ef00 578def p4_last_change():
1997e91f 579 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
1051ef00
LD
580 return int(results[0]['change'])
581
adf159b4 582
123f6317 583def p4_describe(change, shelved=False):
59ef3fc1
JH
584 """Make sure it returns a valid result by checking for the presence of
585 field "time".
586
587 Return a dict of the results.
588 """
18fa13d0 589
123f6317
LD
590 cmd = ["describe", "-s"]
591 if shelved:
592 cmd += ["-S"]
593 cmd += [str(change)]
594
595 ds = p4CmdList(cmd, skip_info=True)
18fa13d0
PW
596 if len(ds) != 1:
597 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
598
599 d = ds[0]
600
601 if "p4ExitCode" in d:
602 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
603 str(d)))
604 if "code" in d:
605 if d["code"] == "error":
606 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
607
608 if "time" not in d:
609 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
610
611 return d
612
adf159b4 613
9cffb8c8 614def split_p4_type(p4type):
522e914f
JH
615 """Canonicalize the p4 type and return a tuple of the base type, plus any
616 modifiers. See "p4 help filetypes" for a list and explanation.
617 """
9cffb8c8
PW
618
619 p4_filetypes_historical = {
620 "ctempobj": "binary+Sw",
621 "ctext": "text+C",
622 "cxtext": "text+Cx",
623 "ktext": "text+k",
624 "kxtext": "text+kx",
625 "ltext": "text+F",
626 "tempobj": "binary+FSw",
627 "ubinary": "binary+F",
628 "uresource": "resource+F",
629 "uxbinary": "binary+Fx",
630 "xbinary": "binary+x",
631 "xltext": "text+Fx",
632 "xtempobj": "binary+Swx",
633 "xtext": "text+x",
634 "xunicode": "unicode+x",
635 "xutf16": "utf16+x",
636 }
637 if p4type in p4_filetypes_historical:
638 p4type = p4_filetypes_historical[p4type]
639 mods = ""
640 s = p4type.split("+")
641 base = s[0]
642 mods = ""
643 if len(s) > 1:
644 mods = s[1]
645 return (base, mods)
b9fc6ea9 646
adf159b4 647
79467e61 648def p4_type(f):
522e914f
JH
649 """Return the raw p4 type of a file (text, text+ko, etc)."""
650
79467e61 651 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
652 return results[0]['headType']
653
adf159b4 654
60df071c 655def p4_keywords_regexp_for_type(base, type_mods):
522e914f
JH
656 """Given a type base and modifier, return a regexp matching the keywords
657 that can be expanded in the file.
658 """
659
60df071c 660 if base in ("text", "unicode", "binary"):
60df071c 661 if "ko" in type_mods:
e665e98e 662 return re_ko_keywords
60df071c 663 elif "k" in type_mods:
e665e98e 664 return re_k_keywords
60df071c
LD
665 else:
666 return None
60df071c
LD
667 else:
668 return None
669
adf159b4 670
60df071c 671def p4_keywords_regexp_for_file(file):
522e914f
JH
672 """Given a file, return a regexp matching the possible RCS keywords that
673 will be expanded, or None for files with kw expansion turned off.
674 """
675
60df071c
LD
676 if not os.path.exists(file):
677 return None
678 else:
0874bb01 679 type_base, type_mods = split_p4_type(p4_type(file))
60df071c 680 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 681
adf159b4 682
c65b670e 683def setP4ExecBit(file, mode):
522e914f
JH
684 """Reopens an already open file and changes the execute bit to match the
685 execute bit setting in the passed in mode.
686 """
c65b670e
CP
687
688 p4Type = "+x"
689
690 if not isModeExec(mode):
691 p4Type = getP4OpenedType(file)
692 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
693 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
694 if p4Type[-1] == "+":
695 p4Type = p4Type[0:-1]
696
6de040df 697 p4_reopen(p4Type, file)
c65b670e 698
adf159b4 699
c65b670e 700def getP4OpenedType(file):
522e914f 701 """Returns the perforce file type for the given file."""
c65b670e 702
9d7d446a 703 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 704 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
705 if match:
706 return match.group(1)
707 else:
f3e5ae4f 708 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 709
adf159b4 710
06804c76 711def getP4Labels(depotPaths):
522e914f
JH
712 """Return the set of all p4 labels."""
713
06804c76 714 labels = set()
484d09c3 715 if not isinstance(depotPaths, list):
06804c76
LD
716 depotPaths = [depotPaths]
717
718 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
719 label = l['label']
720 labels.add(label)
721
722 return labels
723
adf159b4 724
06804c76 725def getGitTags():
522e914f
JH
726 """Return the set of all git tags."""
727
06804c76
LD
728 gitTags = set()
729 for line in read_pipe_lines(["git", "tag"]):
730 tag = line.strip()
731 gitTags.add(tag)
732 return gitTags
733
adf159b4 734
ce425eb4 735_diff_tree_pattern = None
b43b0a3c 736
adf159b4 737
b43b0a3c
CP
738def parseDiffTreeEntry(entry):
739 """Parses a single diff tree entry into its component elements.
740
59ef3fc1
JH
741 See git-diff-tree(1) manpage for details about the format of the diff
742 output. This method returns a dictionary with the following elements:
743
744 src_mode - The mode of the source file
745 dst_mode - The mode of the destination file
746 src_sha1 - The sha1 for the source file
747 dst_sha1 - The sha1 fr the destination file
748 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
749 status_score - The score for the status (applicable for 'C' and 'R'
750 statuses). This is None if there is no score.
751 src - The path for the source file.
752 dst - The path for the destination file. This is only present for
753 copy or renames. If it is not present, this is None.
754
755 If the pattern is not matched, None is returned.
756 """
b43b0a3c 757
ce425eb4
YZ
758 global _diff_tree_pattern
759 if not _diff_tree_pattern:
760 _diff_tree_pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
761
762 match = _diff_tree_pattern.match(entry)
b43b0a3c
CP
763 if match:
764 return {
765 'src_mode': match.group(1),
766 'dst_mode': match.group(2),
767 'src_sha1': match.group(3),
768 'dst_sha1': match.group(4),
769 'status': match.group(5),
770 'status_score': match.group(6),
771 'src': match.group(7),
772 'dst': match.group(10)
773 }
774 return None
775
adf159b4 776
c65b670e 777def isModeExec(mode):
522e914f
JH
778 """Returns True if the given git mode represents an executable file,
779 otherwise False.
780 """
c65b670e
CP
781 return mode[-3:] == "755"
782
adf159b4 783
55bb3e36 784class P4Exception(Exception):
59ef3fc1 785 """Base class for exceptions from the p4 client."""
adf159b4 786
55bb3e36
LD
787 def __init__(self, exit_code):
788 self.p4ExitCode = exit_code
789
adf159b4 790
55bb3e36 791class P4ServerException(P4Exception):
59ef3fc1
JH
792 """Base class for exceptions where we get some kind of marshalled up result
793 from the server.
794 """
adf159b4 795
55bb3e36
LD
796 def __init__(self, exit_code, p4_result):
797 super(P4ServerException, self).__init__(exit_code)
798 self.p4_result = p4_result
799 self.code = p4_result[0]['code']
800 self.data = p4_result[0]['data']
801
adf159b4 802
55bb3e36 803class P4RequestSizeException(P4ServerException):
59ef3fc1 804 """One of the maxresults or maxscanrows errors."""
adf159b4 805
55bb3e36
LD
806 def __init__(self, exit_code, p4_result, limit):
807 super(P4RequestSizeException, self).__init__(exit_code, p4_result)
808 self.limit = limit
809
adf159b4 810
5c3d5020 811class P4CommandException(P4Exception):
59ef3fc1 812 """Something went wrong calling p4 which means we have to give up."""
adf159b4 813
5c3d5020
LD
814 def __init__(self, msg):
815 self.msg = msg
816
817 def __str__(self):
818 return self.msg
819
adf159b4 820
c65b670e
CP
821def isModeExecChanged(src_mode, dst_mode):
822 return isModeExec(src_mode) != isModeExec(dst_mode)
823
adf159b4 824
d2054836
KK
825def p4KeysContainingNonUtf8Chars():
826 """Returns all keys which may contain non UTF-8 encoded strings
827 for which a fallback strategy has to be applied.
828 """
829 return ['desc', 'client', 'FullName']
830
831
832def p4KeysContainingBinaryData():
833 """Returns all keys which may contain arbitrary binary data
834 """
835 return ['data']
836
837
838def p4KeyContainsFilePaths(key):
839 """Returns True if the key contains file paths. These are handled by decode_path().
840 Otherwise False.
841 """
842 return key.startswith('depotFile') or key in ['path', 'clientFile']
843
844
845def p4KeyWhichCanBeDirectlyDecoded(key):
846 """Returns True if the key can be directly decoded as UTF-8 string
847 Otherwise False.
848
849 Keys which can not be encoded directly:
850 - `data` which may contain arbitrary binary data
851 - `desc` or `client` or `FullName` which may contain non-UTF8 encoded text
852 - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text, handled by decode_path()
853 """
854 if key in p4KeysContainingNonUtf8Chars() or \
855 key in p4KeysContainingBinaryData() or \
856 p4KeyContainsFilePaths(key):
857 return False
858 return True
859
860
55bb3e36 861def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
3d8a3038 862 errors_as_exceptions=False, *k, **kw):
6de040df 863
8a470599 864 cmd = p4_build_cmd(["-G"] + cmd)
6a49f8e2 865 if verbose:
727e6ea3 866 sys.stderr.write("Opening pipe: {}\n".format(' '.join(cmd)))
9f90c733
SL
867
868 # Use a temporary file to avoid deadlocks without
869 # subprocess.communicate(), which would put another copy
870 # of stdout into memory.
871 stdin_file = None
872 if stdin is not None:
873 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
484d09c3 874 if not isinstance(stdin, list):
6de040df
LD
875 stdin_file.write(stdin)
876 else:
877 for i in stdin:
86dca24b
YZ
878 stdin_file.write(encode_text_stream(i))
879 stdin_file.write(b'\n')
9f90c733
SL
880 stdin_file.flush()
881 stdin_file.seek(0)
882
3d8a3038
JH
883 p4 = subprocess.Popen(
884 cmd, stdin=stdin_file, stdout=subprocess.PIPE, *k, **kw)
86949eef
SH
885
886 result = []
887 try:
888 while True:
9f90c733 889 entry = marshal.load(p4.stdout)
d2054836 890
6cec21a8 891 if bytes is not str:
d2054836 892 # Decode unmarshalled dict to use str keys and values. Special cases are handled below.
6cec21a8
YZ
893 decoded_entry = {}
894 for key, value in entry.items():
895 key = key.decode()
d2054836 896 if isinstance(value, bytes) and p4KeyWhichCanBeDirectlyDecoded(key):
6cec21a8
YZ
897 value = value.decode()
898 decoded_entry[key] = value
899 # Parse out data if it's an error response
900 if decoded_entry.get('code') == 'error' and 'data' in decoded_entry:
901 decoded_entry['data'] = decoded_entry['data'].decode()
902 entry = decoded_entry
1997e91f
MT
903 if skip_info:
904 if 'code' in entry and entry['code'] == 'info':
905 continue
d2054836
KK
906 for key in p4KeysContainingNonUtf8Chars():
907 if key in entry:
908 entry[key] = metadata_stream_to_writable_bytes(entry[key])
c3f6163b
AG
909 if cb is not None:
910 cb(entry)
911 else:
912 result.append(entry)
86949eef
SH
913 except EOFError:
914 pass
9f90c733
SL
915 exitCode = p4.wait()
916 if exitCode != 0:
55bb3e36
LD
917 if errors_as_exceptions:
918 if len(result) > 0:
919 data = result[0].get('data')
920 if data:
921 m = re.search('Too many rows scanned \(over (\d+)\)', data)
922 if not m:
923 m = re.search('Request too large \(over (\d+)\)', data)
924
925 if m:
926 limit = int(m.group(1))
927 raise P4RequestSizeException(exitCode, result, limit)
928
929 raise P4ServerException(exitCode, result)
930 else:
931 raise P4Exception(exitCode)
932 else:
933 entry = {}
934 entry["p4ExitCode"] = exitCode
935 result.append(entry)
86949eef
SH
936
937 return result
938
adf159b4 939
3d8a3038
JH
940def p4Cmd(cmd, *k, **kw):
941 list = p4CmdList(cmd, *k, **kw)
86949eef
SH
942 result = {}
943 for entry in list:
944 result.update(entry)
990547aa 945 return result
86949eef 946
adf159b4 947
cb2c9db5
SH
948def p4Where(depotPath):
949 if not depotPath.endswith("/"):
950 depotPath += "/"
cd884106
VA
951 depotPathLong = depotPath + "..."
952 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
953 output = None
954 for entry in outputList:
75bc9573 955 if "depotFile" in entry:
cd884106
VA
956 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
957 # The base path always ends with "/...".
d38208a2
YZ
958 entry_path = decode_path(entry['depotFile'])
959 if entry_path.find(depotPath) == 0 and entry_path[-4:] == "/...":
75bc9573
TAL
960 output = entry
961 break
962 elif "data" in entry:
963 data = entry.get("data")
964 space = data.find(" ")
965 if data[:space] == depotPath:
966 output = entry
967 break
da0134f6 968 if output is None:
7f705dc3 969 return ""
dc524036
SH
970 if output["code"] == "error":
971 return ""
cb2c9db5
SH
972 clientPath = ""
973 if "path" in output:
d38208a2 974 clientPath = decode_path(output['path'])
cb2c9db5
SH
975 elif "data" in output:
976 data = output.get("data")
d38208a2
YZ
977 lastSpace = data.rfind(b" ")
978 clientPath = decode_path(data[lastSpace + 1:])
cb2c9db5
SH
979
980 if clientPath.endswith("..."):
981 clientPath = clientPath[:-3]
982 return clientPath
983
adf159b4 984
86949eef 985def currentGitBranch():
eff45110 986 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
86949eef 987
adf159b4 988
4f5cf76a 989def isValidGitDir(path):
da0134f6 990 return git_dir(path) is not None
4f5cf76a 991
adf159b4 992
463e8af6 993def parseRevision(ref):
8a470599 994 return read_pipe(["git", "rev-parse", ref]).strip()
463e8af6 995
adf159b4 996
28755dba
PW
997def branchExists(ref):
998 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
999 ignore_error=True)
1000 return len(rev) > 0
1001
adf159b4 1002
6ae8de88
SH
1003def extractLogMessageFromGitCommit(commit):
1004 logMessage = ""
b016d397 1005
c785e202 1006 # fixme: title is first line of commit, not 1st paragraph.
6ae8de88 1007 foundTitle = False
c3f2358d 1008 for log in read_pipe_lines(["git", "cat-file", "commit", commit]):
812ee74e
JH
1009 if not foundTitle:
1010 if len(log) == 1:
1011 foundTitle = True
1012 continue
6ae8de88 1013
812ee74e 1014 logMessage += log
6ae8de88
SH
1015 return logMessage
1016
adf159b4 1017
bb6e09b2 1018def extractSettingsGitLog(log):
6ae8de88
SH
1019 values = {}
1020 for line in log.split("\n"):
1021 line = line.strip()
843d847f 1022 m = re.search(r"^ *\[git-p4: (.*)\]$", line)
6326aa58
HWN
1023 if not m:
1024 continue
1025
843d847f 1026 assignments = m.group(1).split(':')
6326aa58 1027 for a in assignments:
843d847f 1028 vals = a.split('=')
6326aa58 1029 key = vals[0].strip()
843d847f
JH
1030 val = ('='.join(vals[1:])).strip()
1031 if val.endswith('\"') and val.startswith('"'):
6326aa58
HWN
1032 val = val[1:-1]
1033
1034 values[key] = val
1035
845b42cb
SH
1036 paths = values.get("depot-paths")
1037 if not paths:
1038 paths = values.get("depot-path")
a3fdd579
SH
1039 if paths:
1040 values['depot-paths'] = paths.split(',')
bb6e09b2 1041 return values
6ae8de88 1042
adf159b4 1043
8136a639 1044def gitBranchExists(branch):
bb6e09b2 1045 proc = subprocess.Popen(["git", "rev-parse", branch],
990547aa
JH
1046 stderr=subprocess.PIPE, stdout=subprocess.PIPE)
1047 return proc.wait() == 0
8136a639 1048
adf159b4 1049
123f6317
LD
1050def gitUpdateRef(ref, newvalue):
1051 subprocess.check_call(["git", "update-ref", ref, newvalue])
1052
adf159b4 1053
123f6317
LD
1054def gitDeleteRef(ref):
1055 subprocess.check_call(["git", "update-ref", "-d", ref])
1056
adf159b4 1057
36bd8446 1058_gitConfig = {}
b345d6c3 1059
adf159b4 1060
692e1796 1061def gitConfig(key, typeSpecifier=None):
dba1c9d9 1062 if key not in _gitConfig:
84af8b85 1063 cmd = ["git", "config"]
692e1796 1064 if typeSpecifier:
84af8b85
JH
1065 cmd += [typeSpecifier]
1066 cmd += [key]
b345d6c3
PW
1067 s = read_pipe(cmd, ignore_error=True)
1068 _gitConfig[key] = s.strip()
36bd8446 1069 return _gitConfig[key]
01265103 1070
adf159b4 1071
0d609032
PW
1072def gitConfigBool(key):
1073 """Return a bool, using git config --bool. It is True only if the
1074 variable is set to true, and False if set to false or not present
59ef3fc1
JH
1075 in the config.
1076 """
0d609032 1077
dba1c9d9 1078 if key not in _gitConfig:
692e1796 1079 _gitConfig[key] = gitConfig(key, '--bool') == "true"
36bd8446 1080 return _gitConfig[key]
01265103 1081
adf159b4 1082
cb1dafdf 1083def gitConfigInt(key):
dba1c9d9 1084 if key not in _gitConfig:
84af8b85 1085 cmd = ["git", "config", "--int", key]
0d609032
PW
1086 s = read_pipe(cmd, ignore_error=True)
1087 v = s.strip()
cb1dafdf
LS
1088 try:
1089 _gitConfig[key] = int(gitConfig(key, '--int'))
1090 except ValueError:
1091 _gitConfig[key] = None
36bd8446 1092 return _gitConfig[key]
01265103 1093
adf159b4 1094
7199cf13 1095def gitConfigList(key):
dba1c9d9 1096 if key not in _gitConfig:
2abba301 1097 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
c3c2b057 1098 _gitConfig[key] = s.strip().splitlines()
7960e707
LS
1099 if _gitConfig[key] == ['']:
1100 _gitConfig[key] = []
7199cf13
VA
1101 return _gitConfig[key]
1102
17f273ff
TK
1103def fullP4Ref(incomingRef, importIntoRemotes=True):
1104 """Standardize a given provided p4 ref value to a full git ref:
1105 refs/foo/bar/branch -> use it exactly
1106 p4/branch -> prepend refs/remotes/ or refs/heads/
1107 branch -> prepend refs/remotes/p4/ or refs/heads/p4/"""
1108 if incomingRef.startswith("refs/"):
1109 return incomingRef
1110 if importIntoRemotes:
1111 prepend = "refs/remotes/"
1112 else:
1113 prepend = "refs/heads/"
1114 if not incomingRef.startswith("p4/"):
1115 prepend += "p4/"
1116 return prepend + incomingRef
1117
1118def shortP4Ref(incomingRef, importIntoRemotes=True):
1119 """Standardize to a "short ref" if possible:
1120 refs/foo/bar/branch -> ignore
1121 refs/remotes/p4/branch or refs/heads/p4/branch -> shorten
1122 p4/branch -> shorten"""
1123 if importIntoRemotes:
1124 longprefix = "refs/remotes/p4/"
1125 else:
1126 longprefix = "refs/heads/p4/"
1127 if incomingRef.startswith(longprefix):
1128 return incomingRef[len(longprefix):]
1129 if incomingRef.startswith("p4/"):
1130 return incomingRef[3:]
1131 return incomingRef
adf159b4 1132
2c8037ed
PW
1133def p4BranchesInGit(branchesAreInRemotes=True):
1134 """Find all the branches whose names start with "p4/", looking
1135 in remotes or heads as specified by the argument. Return
1136 a dictionary of { branch: revision } for each one found.
1137 The branch names are the short names, without any
59ef3fc1
JH
1138 "p4/" prefix.
1139 """
2c8037ed 1140
062410bb
SH
1141 branches = {}
1142
8a470599 1143 cmdline = ["git", "rev-parse", "--symbolic"]
062410bb 1144 if branchesAreInRemotes:
8a470599 1145 cmdline.append("--remotes")
062410bb 1146 else:
8a470599 1147 cmdline.append("--branches")
062410bb
SH
1148
1149 for line in read_pipe_lines(cmdline):
1150 line = line.strip()
1151
2c8037ed
PW
1152 # only import to p4/
1153 if not line.startswith('p4/'):
1154 continue
1155 # special symbolic ref to p4/master
1156 if line == "p4/HEAD":
062410bb 1157 continue
062410bb 1158
2c8037ed
PW
1159 # strip off p4/ prefix
1160 branch = line[len("p4/"):]
062410bb
SH
1161
1162 branches[branch] = parseRevision(line)
2c8037ed 1163
062410bb
SH
1164 return branches
1165
adf159b4 1166
5a8e84cd
PW
1167def branch_exists(branch):
1168 """Make sure that the given ref name really exists."""
1169
84af8b85 1170 cmd = ["git", "rev-parse", "--symbolic", "--verify", branch]
5a8e84cd
PW
1171 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
1172 out, _ = p.communicate()
6cec21a8 1173 out = decode_text_stream(out)
5a8e84cd
PW
1174 if p.returncode:
1175 return False
1176 # expect exactly one line of output: the branch name
1177 return out.rstrip() == branch
1178
adf159b4 1179
57fe2ce0 1180def findUpstreamBranchPoint(head="HEAD"):
86506fe5
SH
1181 branches = p4BranchesInGit()
1182 # map from depot-path to branch name
1183 branchByDepotPath = {}
1184 for branch in branches.keys():
1185 tip = branches[branch]
1186 log = extractLogMessageFromGitCommit(tip)
1187 settings = extractSettingsGitLog(log)
dba1c9d9 1188 if "depot-paths" in settings:
944db25c 1189 git_branch = "remotes/p4/" + branch
86506fe5 1190 paths = ",".join(settings["depot-paths"])
944db25c
KF
1191 branchByDepotPath[paths] = git_branch
1192 if "change" in settings:
1193 paths = paths + ";" + settings["change"]
1194 branchByDepotPath[paths] = git_branch
86506fe5 1195
27d2d811 1196 settings = None
27d2d811
SH
1197 parent = 0
1198 while parent < 65535:
9ceab363 1199 commit = head + "~%s" % parent
27d2d811
SH
1200 log = extractLogMessageFromGitCommit(commit)
1201 settings = extractSettingsGitLog(log)
dba1c9d9 1202 if "depot-paths" in settings:
86506fe5 1203 paths = ",".join(settings["depot-paths"])
944db25c
KF
1204 if "change" in settings:
1205 expaths = paths + ";" + settings["change"]
1206 if expaths in branchByDepotPath:
1207 return [branchByDepotPath[expaths], settings]
dba1c9d9 1208 if paths in branchByDepotPath:
86506fe5 1209 return [branchByDepotPath[paths], settings]
27d2d811 1210
86506fe5 1211 parent = parent + 1
27d2d811 1212
86506fe5 1213 return ["", settings]
27d2d811 1214
adf159b4 1215
57fe2ce0 1216def createOrUpdateBranchesFromOrigin(localRefPrefix="refs/remotes/p4/", silent=True):
5ca44617 1217 if not silent:
f2606b17 1218 print("Creating/updating branch(es) in %s based on origin branch(es)"
5ca44617
SH
1219 % localRefPrefix)
1220
1221 originPrefix = "origin/p4/"
1222
8a470599 1223 for line in read_pipe_lines(["git", "rev-parse", "--symbolic", "--remotes"]):
5ca44617
SH
1224 line = line.strip()
1225 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
1226 continue
1227
1228 headName = line[len(originPrefix):]
1229 remoteHead = localRefPrefix + headName
1230 originHead = line
1231
1232 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
7a3e83d0 1233 if 'depot-paths' not in original or 'change' not in original:
5ca44617
SH
1234 continue
1235
1236 update = False
1237 if not gitBranchExists(remoteHead):
1238 if verbose:
f2606b17 1239 print("creating %s" % remoteHead)
5ca44617
SH
1240 update = True
1241 else:
1242 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
dba1c9d9 1243 if 'change' in settings:
5ca44617
SH
1244 if settings['depot-paths'] == original['depot-paths']:
1245 originP4Change = int(original['change'])
1246 p4Change = int(settings['change'])
1247 if originP4Change > p4Change:
f2606b17 1248 print("%s (%s) is newer than %s (%s). "
5ca44617
SH
1249 "Updating p4 branch from origin."
1250 % (originHead, originP4Change,
1251 remoteHead, p4Change))
1252 update = True
1253 else:
f2606b17 1254 print("Ignoring: %s was imported from %s while "
5ca44617
SH
1255 "%s was imported from %s"
1256 % (originHead, ','.join(original['depot-paths']),
1257 remoteHead, ','.join(settings['depot-paths'])))
1258
1259 if update:
8a470599 1260 system(["git", "update-ref", remoteHead, originHead])
5ca44617 1261
adf159b4 1262
5ca44617 1263def originP4BranchesExist():
812ee74e 1264 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
5ca44617 1265
1051ef00
LD
1266
1267def p4ParseNumericChangeRange(parts):
1268 changeStart = int(parts[0][1:])
1269 if parts[1] == '#head':
1270 changeEnd = p4_last_change()
1271 else:
1272 changeEnd = int(parts[1])
1273
1274 return (changeStart, changeEnd)
1275
adf159b4 1276
1051ef00
LD
1277def chooseBlockSize(blockSize):
1278 if blockSize:
1279 return blockSize
1280 else:
1281 return defaultBlockSize
1282
adf159b4 1283
1051ef00 1284def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 1285 assert depotPaths
96b2d54a 1286
1051ef00
LD
1287 # Parse the change range into start and end. Try to find integer
1288 # revision ranges as these can be broken up into blocks to avoid
1289 # hitting server-side limits (maxrows, maxscanresults). But if
1290 # that doesn't work, fall back to using the raw revision specifier
1291 # strings, without using block mode.
1292
96b2d54a 1293 if changeRange is None or changeRange == '':
1051ef00
LD
1294 changeStart = 1
1295 changeEnd = p4_last_change()
1296 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
1297 else:
1298 parts = changeRange.split(',')
1299 assert len(parts) == 2
1051ef00 1300 try:
0874bb01 1301 changeStart, changeEnd = p4ParseNumericChangeRange(parts)
1051ef00 1302 block_size = chooseBlockSize(requestedBlockSize)
8fa0abf8 1303 except ValueError:
1051ef00
LD
1304 changeStart = parts[0][1:]
1305 changeEnd = parts[1]
1306 if requestedBlockSize:
1307 die("cannot use --changes-block-size with non-numeric revisions")
1308 block_size = None
4f6432d8 1309
9943e5b9 1310 changes = set()
96b2d54a 1311
1f90a648 1312 # Retrieve changes a block at a time, to prevent running
3deed5e0
LD
1313 # into a MaxResults/MaxScanRows error from the server. If
1314 # we _do_ hit one of those errors, turn down the block size
1051ef00 1315
1f90a648
SH
1316 while True:
1317 cmd = ['changes']
1051ef00 1318
1f90a648
SH
1319 if block_size:
1320 end = min(changeEnd, changeStart + block_size)
1321 revisionRange = "%d,%d" % (changeStart, end)
1322 else:
1323 revisionRange = "%s,%s" % (changeStart, changeEnd)
1051ef00 1324
1f90a648 1325 for p in depotPaths:
1051ef00
LD
1326 cmd += ["%s...@%s" % (p, revisionRange)]
1327
3deed5e0
LD
1328 # fetch the changes
1329 try:
1330 result = p4CmdList(cmd, errors_as_exceptions=True)
1331 except P4RequestSizeException as e:
1332 if not block_size:
1333 block_size = e.limit
1334 elif block_size > e.limit:
1335 block_size = e.limit
1336 else:
1337 block_size = max(2, block_size // 2)
1338
e8f8b3b2
JH
1339 if verbose:
1340 print("block size error, retrying with block size {0}".format(block_size))
3deed5e0
LD
1341 continue
1342 except P4Exception as e:
1343 die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1344
1f90a648 1345 # Insert changes in chronological order
3deed5e0 1346 for entry in reversed(result):
dba1c9d9 1347 if 'change' not in entry:
b596b3b9
MT
1348 continue
1349 changes.add(int(entry['change']))
1051ef00 1350
1f90a648
SH
1351 if not block_size:
1352 break
1051ef00 1353
1f90a648
SH
1354 if end >= changeEnd:
1355 break
1051ef00 1356
1f90a648 1357 changeStart = end + 1
4f6432d8 1358
1f90a648
SH
1359 changes = sorted(changes)
1360 return changes
4f6432d8 1361
adf159b4 1362
d53de8b9 1363def p4PathStartsWith(path, prefix):
522e914f
JH
1364 """This method tries to remedy a potential mixed-case issue:
1365
1366 If UserA adds //depot/DirA/file1
1367 and UserB adds //depot/dira/file2
1368
1369 we may or may not have a problem. If you have core.ignorecase=true,
1370 we treat DirA and dira as the same directory.
1371 """
0d609032 1372 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
1373 return path.lower().startswith(prefix.lower())
1374 return path.startswith(prefix)
1375
adf159b4 1376
543987bd
PW
1377def getClientSpec():
1378 """Look at the p4 client spec, create a View() object that contains
59ef3fc1
JH
1379 all the mappings, and return it.
1380 """
543987bd 1381
8a470599 1382 specList = p4CmdList(["client", "-o"])
543987bd
PW
1383 if len(specList) != 1:
1384 die('Output from "client -o" is %d lines, expecting 1' %
1385 len(specList))
1386
1387 # dictionary of all client parameters
1388 entry = specList[0]
1389
9d57c4a6
KS
1390 # the //client/ name
1391 client_name = entry["Client"]
1392
543987bd 1393 # just the keys that start with "View"
84af8b85 1394 view_keys = [k for k in entry.keys() if k.startswith("View")]
543987bd
PW
1395
1396 # hold this new View
9d57c4a6 1397 view = View(client_name)
543987bd
PW
1398
1399 # append the lines, in order, to the view
1400 for view_num in range(len(view_keys)):
1401 k = "View%d" % view_num
1402 if k not in view_keys:
1403 die("Expected view key %s missing" % k)
1404 view.append(entry[k])
1405
1406 return view
1407
adf159b4 1408
543987bd
PW
1409def getClientRoot():
1410 """Grab the client directory."""
1411
8a470599 1412 output = p4CmdList(["client", "-o"])
543987bd
PW
1413 if len(output) != 1:
1414 die('Output from "client -o" is %d lines, expecting 1' % len(output))
1415
1416 entry = output[0]
1417 if "Root" not in entry:
1418 die('Client has no "Root"')
1419
1420 return entry["Root"]
1421
adf159b4 1422
9d7d446a 1423def wildcard_decode(path):
522e914f
JH
1424 """Decode P4 wildcards into %xx encoding
1425
1426 P4 wildcards are not allowed in filenames. P4 complains if you simply
1427 add them, but you can force it with "-f", in which case it translates
1428 them into %xx encoding internally.
1429 """
1430
9d7d446a
PW
1431 # Search for and fix just these four characters. Do % last so
1432 # that fixing it does not inadvertently create new %-escapes.
1433 # Cannot have * in a filename in windows; untested as to
1434 # what p4 would do in such a case.
1435 if not platform.system() == "Windows":
1436 path = path.replace("%2A", "*")
1437 path = path.replace("%23", "#") \
1438 .replace("%40", "@") \
1439 .replace("%25", "%")
1440 return path
1441
adf159b4 1442
9d7d446a 1443def wildcard_encode(path):
522e914f
JH
1444 """Encode %xx coded wildcards into P4 coding."""
1445
9d7d446a
PW
1446 # do % first to avoid double-encoding the %s introduced here
1447 path = path.replace("%", "%25") \
1448 .replace("*", "%2A") \
1449 .replace("#", "%23") \
1450 .replace("@", "%40")
1451 return path
1452
adf159b4 1453
9d7d446a 1454def wildcard_present(path):
598354c0
BC
1455 m = re.search("[*#@%]", path)
1456 return m is not None
9d7d446a 1457
adf159b4 1458
a5db4b12
LS
1459class LargeFileSystem(object):
1460 """Base class for large file system support."""
1461
1462 def __init__(self, writeToGitStream):
1463 self.largeFiles = set()
1464 self.writeToGitStream = writeToGitStream
1465
1466 def generatePointer(self, cloneDestination, contentFile):
59ef3fc1
JH
1467 """Return the content of a pointer file that is stored in Git instead
1468 of the actual content.
1469 """
a5db4b12
LS
1470 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1471
1472 def pushFile(self, localLargeFile):
1473 """Push the actual content which is not stored in the Git repository to
59ef3fc1
JH
1474 a server.
1475 """
a5db4b12
LS
1476 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1477
1478 def hasLargeFileExtension(self, relPath):
a6b13067 1479 return functools.reduce(
a5db4b12
LS
1480 lambda a, b: a or b,
1481 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1482 False
1483 )
1484
1485 def generateTempFile(self, contents):
1486 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1487 for d in contents:
1488 contentFile.write(d)
1489 contentFile.close()
1490 return contentFile.name
1491
1492 def exceedsLargeFileThreshold(self, relPath, contents):
1493 if gitConfigInt('git-p4.largeFileThreshold'):
1494 contentsSize = sum(len(d) for d in contents)
1495 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1496 return True
1497 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1498 contentsSize = sum(len(d) for d in contents)
1499 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1500 return False
1501 contentTempFile = self.generateTempFile(contents)
de5abb5f
PM
1502 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=True)
1503 with zipfile.ZipFile(compressedContentFile, mode='w') as zf:
1504 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1505 compressedContentsSize = zf.infolist()[0].compress_size
a5db4b12 1506 os.remove(contentTempFile)
a5db4b12
LS
1507 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1508 return True
1509 return False
1510
1511 def addLargeFile(self, relPath):
1512 self.largeFiles.add(relPath)
1513
1514 def removeLargeFile(self, relPath):
1515 self.largeFiles.remove(relPath)
1516
1517 def isLargeFile(self, relPath):
1518 return relPath in self.largeFiles
1519
1520 def processContent(self, git_mode, relPath, contents):
1521 """Processes the content of git fast import. This method decides if a
1522 file is stored in the large file system and handles all necessary
59ef3fc1
JH
1523 steps.
1524 """
a5db4b12
LS
1525 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1526 contentTempFile = self.generateTempFile(contents)
0874bb01 1527 pointer_git_mode, contents, localLargeFile = self.generatePointer(contentTempFile)
d5eb3cf5
LS
1528 if pointer_git_mode:
1529 git_mode = pointer_git_mode
1530 if localLargeFile:
1531 # Move temp file to final location in large file system
1532 largeFileDir = os.path.dirname(localLargeFile)
1533 if not os.path.isdir(largeFileDir):
1534 os.makedirs(largeFileDir)
1535 shutil.move(contentTempFile, localLargeFile)
1536 self.addLargeFile(relPath)
1537 if gitConfigBool('git-p4.largeFilePush'):
1538 self.pushFile(localLargeFile)
1539 if verbose:
1540 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
a5db4b12
LS
1541 return (git_mode, contents)
1542
adf159b4 1543
a5db4b12
LS
1544class MockLFS(LargeFileSystem):
1545 """Mock large file system for testing."""
1546
1547 def generatePointer(self, contentFile):
1548 """The pointer content is the original content prefixed with "pointer-".
59ef3fc1
JH
1549 The local filename of the large file storage is derived from the
1550 file content.
a5db4b12
LS
1551 """
1552 with open(contentFile, 'r') as f:
1553 content = next(f)
1554 gitMode = '100644'
1555 pointerContents = 'pointer-' + content
1556 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1557 return (gitMode, pointerContents, localLargeFile)
1558
1559 def pushFile(self, localLargeFile):
59ef3fc1
JH
1560 """The remote filename of the large file storage is the same as the
1561 local one but in a different directory.
a5db4b12
LS
1562 """
1563 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1564 if not os.path.exists(remotePath):
1565 os.makedirs(remotePath)
1566 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1567
adf159b4 1568
b47d807d
LS
1569class GitLFS(LargeFileSystem):
1570 """Git LFS as backend for the git-p4 large file system.
59ef3fc1
JH
1571 See https://git-lfs.github.com/ for details.
1572 """
b47d807d
LS
1573
1574 def __init__(self, *args):
1575 LargeFileSystem.__init__(self, *args)
1576 self.baseGitAttributes = []
1577
1578 def generatePointer(self, contentFile):
1579 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1580 mode and content which is stored in the Git repository instead of
1581 the actual content. Return also the new location of the actual
1582 content.
1583 """
d5eb3cf5
LS
1584 if os.path.getsize(contentFile) == 0:
1585 return (None, '', None)
1586
b47d807d
LS
1587 pointerProcess = subprocess.Popen(
1588 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1589 stdout=subprocess.PIPE
1590 )
86dca24b 1591 pointerFile = decode_text_stream(pointerProcess.stdout.read())
b47d807d
LS
1592 if pointerProcess.wait():
1593 os.remove(contentFile)
1594 die('git-lfs pointer command failed. Did you install the extension?')
82f2567e
LS
1595
1596 # Git LFS removed the preamble in the output of the 'pointer' command
1597 # starting from version 1.2.0. Check for the preamble here to support
1598 # earlier versions.
1599 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1600 if pointerFile.startswith('Git LFS pointer for'):
1601 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1602
1603 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
ea94b16f 1604 # if someone use external lfs.storage ( not in local repo git )
1605 lfs_path = gitConfig('lfs.storage')
1606 if not lfs_path:
1607 lfs_path = 'lfs'
1608 if not os.path.isabs(lfs_path):
1609 lfs_path = os.path.join(os.getcwd(), '.git', lfs_path)
b47d807d 1610 localLargeFile = os.path.join(
ea94b16f 1611 lfs_path,
1612 'objects', oid[:2], oid[2:4],
b47d807d
LS
1613 oid,
1614 )
1615 # LFS Spec states that pointer files should not have the executable bit set.
1616 gitMode = '100644'
82f2567e 1617 return (gitMode, pointerFile, localLargeFile)
b47d807d
LS
1618
1619 def pushFile(self, localLargeFile):
1620 uploadProcess = subprocess.Popen(
1621 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1622 )
1623 if uploadProcess.wait():
1624 die('git-lfs push command failed. Did you define a remote?')
1625
1626 def generateGitAttributes(self):
1627 return (
1628 self.baseGitAttributes +
1629 [
1630 '\n',
1631 '#\n',
1632 '# Git LFS (see https://git-lfs.github.com/)\n',
1633 '#\n',
1634 ] +
862f9312 1635 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1636 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1637 ] +
862f9312 1638 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1639 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1640 ]
1641 )
1642
1643 def addLargeFile(self, relPath):
1644 LargeFileSystem.addLargeFile(self, relPath)
1645 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1646
1647 def removeLargeFile(self, relPath):
1648 LargeFileSystem.removeLargeFile(self, relPath)
1649 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1650
1651 def processContent(self, git_mode, relPath, contents):
1652 if relPath == '.gitattributes':
1653 self.baseGitAttributes = contents
1654 return (git_mode, self.generateGitAttributes())
1655 else:
1656 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1657
adf159b4 1658
b984733c 1659class Command:
84af8b85
JH
1660 delete_actions = ("delete", "move/delete", "purge")
1661 add_actions = ("add", "branch", "move/add")
89143ac2 1662
b984733c
SH
1663 def __init__(self):
1664 self.usage = "usage: %prog [options]"
8910ac0e 1665 self.needsGit = True
6a10b6aa 1666 self.verbose = False
b984733c 1667
ff8c50ed 1668 # This is required for the "append" update_shelve action
8cf422db
LD
1669 def ensure_value(self, attr, value):
1670 if not hasattr(self, attr) or getattr(self, attr) is None:
1671 setattr(self, attr, value)
1672 return getattr(self, attr)
1673
adf159b4 1674
3ea2cfd4
LD
1675class P4UserMap:
1676 def __init__(self):
1677 self.userMapFromPerforceServer = False
affb474f
LD
1678 self.myP4UserId = None
1679
1680 def p4UserId(self):
1681 if self.myP4UserId:
1682 return self.myP4UserId
1683
8a470599 1684 results = p4CmdList(["user", "-o"])
affb474f 1685 for r in results:
dba1c9d9 1686 if 'User' in r:
affb474f
LD
1687 self.myP4UserId = r['User']
1688 return r['User']
1689 die("Could not find your p4 user id")
1690
1691 def p4UserIsMe(self, p4User):
522e914f 1692 """Return True if the given p4 user is actually me."""
affb474f
LD
1693 me = self.p4UserId()
1694 if not p4User or p4User != me:
1695 return False
1696 else:
1697 return True
3ea2cfd4
LD
1698
1699 def getUserCacheFilename(self):
1700 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1701 return home + "/.gitp4-usercache.txt"
1702
1703 def getUserMapFromPerforceServer(self):
1704 if self.userMapFromPerforceServer:
1705 return
1706 self.users = {}
1707 self.emails = {}
1708
8a470599 1709 for output in p4CmdList(["users"]):
dba1c9d9 1710 if "User" not in output:
3ea2cfd4 1711 continue
f7b5ff60
TK
1712 # "FullName" is bytes. "Email" on the other hand might be bytes
1713 # or unicode string depending on whether we are running under
1714 # python2 or python3. To support
1715 # git-p4.metadataDecodingStrategy=fallback, self.users dict values
1716 # are always bytes, ready to be written to git.
1717 emailbytes = metadata_stream_to_writable_bytes(output["Email"])
1718 self.users[output["User"]] = output["FullName"] + b" <" + emailbytes + b">"
3ea2cfd4
LD
1719 self.emails[output["Email"]] = output["User"]
1720
10d08a14
LS
1721 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1722 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1723 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1724 if mapUser and len(mapUser[0]) == 3:
1725 user = mapUser[0][0]
1726 fullname = mapUser[0][1]
1727 email = mapUser[0][2]
f7b5ff60
TK
1728 fulluser = fullname + " <" + email + ">"
1729 self.users[user] = metadata_stream_to_writable_bytes(fulluser)
10d08a14 1730 self.emails[email] = user
3ea2cfd4 1731
f7b5ff60 1732 s = b''
3ea2cfd4 1733 for (key, val) in self.users.items():
f7b5ff60
TK
1734 keybytes = metadata_stream_to_writable_bytes(key)
1735 s += b"%s\t%s\n" % (keybytes.expandtabs(1), val.expandtabs(1))
3ea2cfd4 1736
f7b5ff60 1737 open(self.getUserCacheFilename(), 'wb').write(s)
3ea2cfd4
LD
1738 self.userMapFromPerforceServer = True
1739
1740 def loadUserMapFromCache(self):
1741 self.users = {}
1742 self.userMapFromPerforceServer = False
1743 try:
f7b5ff60 1744 cache = open(self.getUserCacheFilename(), 'rb')
3ea2cfd4
LD
1745 lines = cache.readlines()
1746 cache.close()
1747 for line in lines:
f7b5ff60
TK
1748 entry = line.strip().split(b"\t")
1749 self.users[entry[0].decode('utf_8')] = entry[1]
3ea2cfd4
LD
1750 except IOError:
1751 self.getUserMapFromPerforceServer()
1752
adf159b4 1753
3ea2cfd4 1754class P4Submit(Command, P4UserMap):
6bbfd137
PW
1755
1756 conflict_behavior_choices = ("ask", "skip", "quit")
1757
4f5cf76a 1758 def __init__(self):
b984733c 1759 Command.__init__(self)
3ea2cfd4 1760 P4UserMap.__init__(self)
4f5cf76a 1761 self.options = [
4f5cf76a 1762 optparse.make_option("--origin", dest="origin"),
ae901090 1763 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1764 # preserve the user, requires relevant p4 permissions
1765 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1766 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1767 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1768 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1769 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1770 choices=self.conflict_behavior_choices),
1771 optparse.make_option("--branch", dest="branch"),
b34fa577
VK
1772 optparse.make_option("--shelve", dest="shelve", action="store_true",
1773 help="Shelve instead of submit. Shelved files are reverted, "
1774 "restoring the workspace to the state before the shelve"),
8cf422db 1775 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
46c609e9 1776 metavar="CHANGELIST",
8cf422db 1777 help="update an existing shelved changelist, implies --shelve, "
f55b87c1
RM
1778 "repeat in-order for multiple shelved changelists"),
1779 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1780 help="submit only the specified commit(s), one commit or xxx..xxx"),
1781 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1782 help="Disable rebase after submit is completed. Can be useful if you "
b9d34db9
LD
1783 "work from a local git branch that is not master"),
1784 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1785 help="Skip Perforce sync of p4/master after submit or shelve"),
4935c458 1786 optparse.make_option("--no-verify", dest="no_verify", action="store_true",
38ecf752 1787 help="Bypass p4-pre-submit and p4-changelist hooks"),
4f5cf76a 1788 ]
251c8c50 1789 self.description = """Submit changes from git to the perforce depot.\n
4935c458
BK
1790 The `p4-pre-submit` hook is executed if it exists and is executable. It
1791 can be bypassed with the `--no-verify` command line option. The hook takes
1792 no parameters and nothing from standard input. Exiting with a non-zero status
1793 from this script prevents `git-p4 submit` from launching.
251c8c50 1794
4935c458 1795 One usage scenario is to run unit tests in the hook.
38ecf752
BK
1796
1797 The `p4-prepare-changelist` hook is executed right after preparing the default
1798 changelist message and before the editor is started. It takes one parameter,
1799 the name of the file that contains the changelist text. Exiting with a non-zero
1800 status from the script will abort the process.
1801
1802 The purpose of the hook is to edit the message file in place, and it is not
1803 supressed by the `--no-verify` option. This hook is called even if
1804 `--prepare-p4-only` is set.
1805
1806 The `p4-changelist` hook is executed after the changelist message has been
1807 edited by the user. It can be bypassed with the `--no-verify` option. It
1808 takes a single parameter, the name of the file that holds the proposed
1809 changelist text. Exiting with a non-zero status causes the command to abort.
1810
1811 The hook is allowed to edit the changelist file and can be used to normalize
1812 the text into some project standard format. It can also be used to refuse the
1813 Submit after inspect the message file.
1814
1815 The `p4-post-changelist` hook is invoked after the submit has successfully
b7e20b43 1816 occurred in P4. It takes no parameters and is meant primarily for notification
38ecf752 1817 and cannot affect the outcome of the git p4 submit action.
4935c458 1818 """
251c8c50 1819
c9b50e63 1820 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1821 self.origin = ""
ae901090 1822 self.detectRenames = False
0d609032 1823 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1824 self.dry_run = False
b34fa577 1825 self.shelve = False
8cf422db 1826 self.update_shelve = list()
f55b87c1 1827 self.commit = ""
3b3477ea 1828 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
b9d34db9 1829 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
728b7ad8 1830 self.prepare_p4_only = False
6bbfd137 1831 self.conflict_behavior = None
f7baba8b 1832 self.isWindows = (platform.system() == "Windows")
06804c76 1833 self.exportLabels = False
249da4c0 1834 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1835 self.branch = None
4935c458 1836 self.no_verify = False
4f5cf76a 1837
a5db4b12
LS
1838 if gitConfig('git-p4.largeFileSystem'):
1839 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1840
4f5cf76a 1841 def check(self):
8a470599 1842 if len(p4CmdList(["opened", "..."])) > 0:
4f5cf76a
SH
1843 die("You have files opened with perforce! Close them before starting the sync.")
1844
f19cb0a0 1845 def separate_jobs_from_description(self, message):
59ef3fc1
JH
1846 """Extract and return a possible Jobs field in the commit message. It
1847 goes into a separate section in the p4 change specification.
f19cb0a0 1848
59ef3fc1
JH
1849 A jobs line starts with "Jobs:" and looks like a new field in a
1850 form. Values are white-space separated on the same line or on
1851 following lines that start with a tab.
f19cb0a0 1852
59ef3fc1
JH
1853 This does not parse and extract the full git commit message like a
1854 p4 form. It just sees the Jobs: line as a marker to pass everything
1855 from then on directly into the p4 form, but outside the description
1856 section.
f19cb0a0 1857
59ef3fc1
JH
1858 Return a tuple (stripped log message, jobs string).
1859 """
f19cb0a0
PW
1860
1861 m = re.search(r'^Jobs:', message, re.MULTILINE)
1862 if m is None:
1863 return (message, None)
1864
1865 jobtext = message[m.start():]
1866 stripped_message = message[:m.start()].rstrip()
1867 return (stripped_message, jobtext)
1868
1869 def prepareLogMessage(self, template, message, jobs):
59ef3fc1
JH
1870 """Edits the template returned from "p4 change -o" to insert the
1871 message in the Description field, and the jobs text in the Jobs
1872 field.
1873 """
4f5cf76a
SH
1874 result = ""
1875
edae1e2f
SH
1876 inDescriptionSection = False
1877
4f5cf76a
SH
1878 for line in template.split("\n"):
1879 if line.startswith("#"):
1880 result += line + "\n"
1881 continue
1882
edae1e2f 1883 if inDescriptionSection:
c9dbab04 1884 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1885 inDescriptionSection = False
f19cb0a0
PW
1886 # insert Jobs section
1887 if jobs:
1888 result += jobs + "\n"
edae1e2f
SH
1889 else:
1890 continue
1891 else:
1892 if line.startswith("Description:"):
1893 inDescriptionSection = True
1894 line += "\n"
1895 for messageLine in message.split("\n"):
1896 line += "\t" + messageLine + "\n"
1897
1898 result += line + "\n"
4f5cf76a
SH
1899
1900 return result
1901
e665e98e 1902 def patchRCSKeywords(self, file, regexp):
522e914f
JH
1903 """Attempt to zap the RCS keywords in a p4 controlled file matching the
1904 given regex.
1905 """
0874bb01 1906 handle, outFileName = tempfile.mkstemp(dir='.')
60df071c 1907 try:
70c0d553 1908 with os.fdopen(handle, "wb") as outFile, open(file, "rb") as inFile:
8618d322 1909 for line in inFile.readlines():
70c0d553 1910 outFile.write(regexp.sub(br'$\1$', line))
60df071c
LD
1911 # Forcibly overwrite the original file
1912 os.unlink(file)
1913 shutil.move(outFileName, file)
1914 except:
1915 # cleanup our temporary file
1916 os.unlink(outFileName)
f2606b17 1917 print("Failed to strip RCS keywords in %s" % file)
60df071c
LD
1918 raise
1919
f2606b17 1920 print("Patched up RCS keywords in %s" % file)
60df071c 1921
12a77f5b 1922 def p4UserForCommit(self, id):
522e914f
JH
1923 """Return the tuple (perforce user,git email) for a given git commit
1924 id.
1925 """
3ea2cfd4 1926 self.getUserMapFromPerforceServer()
9bf28855
PW
1927 gitEmail = read_pipe(["git", "log", "--max-count=1",
1928 "--format=%ae", id])
3ea2cfd4 1929 gitEmail = gitEmail.strip()
dba1c9d9 1930 if gitEmail not in self.emails:
12a77f5b 1931 return (None, gitEmail)
3ea2cfd4 1932 else:
12a77f5b 1933 return (self.emails[gitEmail], gitEmail)
3ea2cfd4 1934
12a77f5b 1935 def checkValidP4Users(self, commits):
522e914f 1936 """Check if any git authors cannot be mapped to p4 users."""
3ea2cfd4 1937 for id in commits:
0874bb01 1938 user, email = self.p4UserForCommit(id)
3ea2cfd4
LD
1939 if not user:
1940 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1941 if gitConfigBool("git-p4.allowMissingP4Users"):
f2606b17 1942 print("%s" % msg)
3ea2cfd4
LD
1943 else:
1944 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1945
1946 def lastP4Changelist(self):
522e914f
JH
1947 """Get back the last changelist number submitted in this client spec.
1948
1949 This then gets used to patch up the username in the change. If the
1950 same client spec is being used by multiple processes then this might
1951 go wrong.
1952 """
8a470599 1953 results = p4CmdList(["client", "-o"]) # find the current client
3ea2cfd4
LD
1954 client = None
1955 for r in results:
dba1c9d9 1956 if 'Client' in r:
3ea2cfd4
LD
1957 client = r['Client']
1958 break
1959 if not client:
1960 die("could not get client spec")
6de040df 1961 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4 1962 for r in results:
dba1c9d9 1963 if 'change' in r:
3ea2cfd4
LD
1964 return r['change']
1965 die("Could not get changelist number for last submit - cannot patch up user details")
1966
1967 def modifyChangelistUser(self, changelist, newUser):
522e914f 1968 """Fixup the user field of a changelist after it has been submitted."""
8a470599 1969 changes = p4CmdList(["change", "-o", changelist])
ecdba36d
LD
1970 if len(changes) != 1:
1971 die("Bad output from p4 change modifying %s to user %s" %
1972 (changelist, newUser))
1973
1974 c = changes[0]
e8f8b3b2
JH
1975 if c['User'] == newUser:
1976 # Nothing to do
1977 return
ecdba36d 1978 c['User'] = newUser
50da1e73
YZ
1979 # p4 does not understand format version 3 and above
1980 input = marshal.dumps(c, 2)
ecdba36d 1981
8a470599 1982 result = p4CmdList(["change", "-f", "-i"], stdin=input)
3ea2cfd4 1983 for r in result:
dba1c9d9 1984 if 'code' in r:
3ea2cfd4
LD
1985 if r['code'] == 'error':
1986 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
dba1c9d9 1987 if 'data' in r:
3ea2cfd4
LD
1988 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1989 return
1990 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1991
1992 def canChangeChangelists(self):
522e914f
JH
1993 """Check to see if we have p4 admin or super-user permissions, either
1994 of which are required to modify changelists.
1995 """
52a4880b 1996 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4 1997 for r in results:
dba1c9d9 1998 if 'perm' in r:
3ea2cfd4
LD
1999 if r['perm'] == 'admin':
2000 return 1
2001 if r['perm'] == 'super':
2002 return 1
2003 return 0
2004
46c609e9 2005 def prepareSubmitTemplate(self, changelist=None):
f19cb0a0 2006 """Run "p4 change -o" to grab a change specification template.
59ef3fc1 2007
f19cb0a0
PW
2008 This does not use "p4 -G", as it is nice to keep the submission
2009 template in original order, since a human might edit it.
2010
2011 Remove lines in the Files section that show changes to files
59ef3fc1
JH
2012 outside the depot path we're committing into.
2013 """
f19cb0a0 2014
0874bb01 2015 upstream, settings = findUpstreamBranchPoint()
cbc69242 2016
b596b3b9
MT
2017 template = """\
2018# A Perforce Change Specification.
2019#
2020# Change: The change number. 'new' on a new changelist.
2021# Date: The date this specification was last modified.
2022# Client: The client on which the changelist was created. Read-only.
2023# User: The user who created the changelist.
2024# Status: Either 'pending' or 'submitted'. Read-only.
2025# Type: Either 'public' or 'restricted'. Default is 'public'.
2026# Description: Comments about the changelist. Required.
2027# Jobs: What opened jobs are to be closed by this changelist.
2028# You may delete jobs from this list. (New changelists only.)
2029# Files: What opened files from the default changelist are to be added
2030# to this changelist. You may delete files from this list.
2031# (New changelists only.)
2032"""
2033 files_list = []
ea99c3ae 2034 inFilesSection = False
b596b3b9 2035 change_entry = None
46c609e9
LD
2036 args = ['change', '-o']
2037 if changelist:
2038 args.append(str(changelist))
b596b3b9 2039 for entry in p4CmdList(args):
dba1c9d9 2040 if 'code' not in entry:
b596b3b9
MT
2041 continue
2042 if entry['code'] == 'stat':
2043 change_entry = entry
2044 break
2045 if not change_entry:
2046 die('Failed to decode output of p4 change -o')
2e2aa8d9 2047 for key, value in change_entry.items():
b596b3b9 2048 if key.startswith('File'):
dba1c9d9 2049 if 'depot-paths' in settings:
b596b3b9
MT
2050 if not [p for p in settings['depot-paths']
2051 if p4PathStartsWith(value, p)]:
2052 continue
ea99c3ae 2053 else:
b596b3b9
MT
2054 if not p4PathStartsWith(value, self.depotPath):
2055 continue
2056 files_list.append(value)
2057 continue
2058 # Output in the order expected by prepareLogMessage
2059 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
dba1c9d9 2060 if key not in change_entry:
b596b3b9
MT
2061 continue
2062 template += '\n'
2063 template += key + ':'
2064 if key == 'Description':
2065 template += '\n'
2066 for field_line in change_entry[key].splitlines():
2067 template += '\t'+field_line+'\n'
2068 if len(files_list) > 0:
2069 template += '\n'
2070 template += 'Files:\n'
2071 for path in files_list:
2072 template += '\t'+path+'\n'
ea99c3ae
SH
2073 return template
2074
7c766e57 2075 def edit_template(self, template_file):
59ef3fc1
JH
2076 """Invoke the editor to let the user change the submission message.
2077
2078 Return true if okay to continue with the submit.
2079 """
7c766e57
PW
2080
2081 # if configured to skip the editing part, just submit
0d609032 2082 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
2083 return True
2084
2085 # look at the modification time, to check later if the user saved
2086 # the file
2087 mtime = os.stat(template_file).st_mtime
2088
2089 # invoke the editor
dba1c9d9 2090 if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
2091 editor = os.environ.get("P4EDITOR")
2092 else:
8a470599 2093 editor = read_pipe(["git", "var", "GIT_EDITOR"]).strip()
2dade7a7 2094 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
2095
2096 # If the file was not saved, prompt to see if this patch should
2097 # be skipped. But skip this verification step if configured so.
0d609032 2098 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
2099 return True
2100
d1652049
PW
2101 # modification time updated means user saved the file
2102 if os.stat(template_file).st_mtime > mtime:
2103 return True
2104
e2aed5fd
BK
2105 response = prompt("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
2106 if response == 'y':
2107 return True
2108 if response == 'n':
2109 return False
7c766e57 2110
df8a9e86 2111 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
b4073bb3 2112 # diff
dba1c9d9 2113 if "P4DIFF" in os.environ:
b4073bb3
MC
2114 del(os.environ["P4DIFF"])
2115 diff = ""
2116 for editedFile in editedFiles:
2117 diff += p4_read_pipe(['diff', '-du',
2118 wildcard_encode(editedFile)])
2119
2120 # new file diff
2121 newdiff = ""
2122 for newFile in filesToAdd:
2123 newdiff += "==== new file ====\n"
2124 newdiff += "--- /dev/null\n"
2125 newdiff += "+++ %s\n" % newFile
df8a9e86
LD
2126
2127 is_link = os.path.islink(newFile)
2128 expect_link = newFile in symlinks
2129
2130 if is_link and expect_link:
2131 newdiff += "+%s\n" % os.readlink(newFile)
2132 else:
2133 f = open(newFile, "r")
54662d59 2134 try:
2135 for line in f.readlines():
2136 newdiff += "+" + line
2137 except UnicodeDecodeError:
4768af20
JH
2138 # Found non-text data and skip, since diff description
2139 # should only include text
2140 pass
df8a9e86 2141 f.close()
b4073bb3 2142
e2a892ee 2143 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 2144
7cb5cbef 2145 def applyCommit(self, id):
67b0fe2e
PW
2146 """Apply one commit, return True if it succeeded."""
2147
f2606b17
LD
2148 print("Applying", read_pipe(["git", "show", "-s",
2149 "--format=format:%h %s", id]))
ae901090 2150
0874bb01 2151 p4User, gitEmail = self.p4UserForCommit(id)
3ea2cfd4 2152
3d8a3038 2153 diff = read_pipe_lines(
8a470599 2154 ["git", "diff-tree", "-r"] + self.diffOpts + ["{}^".format(id), id])
4f5cf76a 2155 filesToAdd = set()
a02b8bc4 2156 filesToChangeType = set()
4f5cf76a 2157 filesToDelete = set()
d336c158 2158 editedFiles = set()
b6ad6dcc 2159 pureRenameCopy = set()
df8a9e86 2160 symlinks = set()
c65b670e 2161 filesToChangeExecBit = {}
46c609e9 2162 all_files = list()
60df071c 2163
4f5cf76a 2164 for line in diff:
b43b0a3c
CP
2165 diff = parseDiffTreeEntry(line)
2166 modifier = diff['status']
2167 path = diff['src']
46c609e9
LD
2168 all_files.append(path)
2169
4f5cf76a 2170 if modifier == "M":
6de040df 2171 p4_edit(path)
c65b670e
CP
2172 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
2173 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 2174 editedFiles.add(path)
4f5cf76a
SH
2175 elif modifier == "A":
2176 filesToAdd.add(path)
c65b670e 2177 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
2178 if path in filesToDelete:
2179 filesToDelete.remove(path)
df8a9e86
LD
2180
2181 dst_mode = int(diff['dst_mode'], 8)
db2d997e 2182 if dst_mode == 0o120000:
df8a9e86
LD
2183 symlinks.add(path)
2184
4f5cf76a
SH
2185 elif modifier == "D":
2186 filesToDelete.add(path)
2187 if path in filesToAdd:
2188 filesToAdd.remove(path)
4fddb41b
VA
2189 elif modifier == "C":
2190 src, dest = diff['src'], diff['dst']
7a10946a 2191 all_files.append(dest)
6de040df 2192 p4_integrate(src, dest)
b6ad6dcc 2193 pureRenameCopy.add(dest)
4fddb41b 2194 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 2195 p4_edit(dest)
b6ad6dcc 2196 pureRenameCopy.discard(dest)
4fddb41b 2197 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 2198 p4_edit(dest)
b6ad6dcc 2199 pureRenameCopy.discard(dest)
4fddb41b 2200 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
2201 if self.isWindows:
2202 # turn off read-only attribute
2203 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
2204 os.unlink(dest)
2205 editedFiles.add(dest)
d9a5f25b 2206 elif modifier == "R":
b43b0a3c 2207 src, dest = diff['src'], diff['dst']
7a10946a 2208 all_files.append(dest)
8e9497c2
GG
2209 if self.p4HasMoveCommand:
2210 p4_edit(src) # src must be open before move
2211 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 2212 else:
8e9497c2
GG
2213 p4_integrate(src, dest)
2214 if diff['src_sha1'] != diff['dst_sha1']:
2215 p4_edit(dest)
2216 else:
2217 pureRenameCopy.add(dest)
c65b670e 2218 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
2219 if not self.p4HasMoveCommand:
2220 p4_edit(dest) # with move: already open, writable
c65b670e 2221 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 2222 if not self.p4HasMoveCommand:
d20f0f8e
PW
2223 if self.isWindows:
2224 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
2225 os.unlink(dest)
2226 filesToDelete.add(src)
d9a5f25b 2227 editedFiles.add(dest)
a02b8bc4
RP
2228 elif modifier == "T":
2229 filesToChangeType.add(path)
4f5cf76a
SH
2230 else:
2231 die("unknown modifier %s for %s" % (modifier, path))
2232
749b668c 2233 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 2234 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
2235 tryPatchCmd = patchcmd + "--check -"
2236 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 2237 patch_succeeded = True
51a2640a 2238
1ec4a0a3
BK
2239 if verbose:
2240 print("TryPatch: %s" % tryPatchCmd)
2241
47a130b7 2242 if os.system(tryPatchCmd) != 0:
60df071c
LD
2243 fixed_rcs_keywords = False
2244 patch_succeeded = False
f2606b17 2245 print("Unfortunately applying the change failed!")
60df071c
LD
2246
2247 # Patch failed, maybe it's just RCS keyword woes. Look through
2248 # the patch to see if that's possible.
0d609032 2249 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c 2250 file = None
60df071c
LD
2251 kwfiles = {}
2252 for file in editedFiles | filesToDelete:
2253 # did this file's delta contain RCS keywords?
e665e98e
JH
2254 regexp = p4_keywords_regexp_for_file(file)
2255 if regexp:
60df071c 2256 # this file is a possibility...look for RCS keywords.
70c0d553 2257 for line in read_pipe_lines(
7a3e83d0
JH
2258 ["git", "diff", "%s^..%s" % (id, id), file],
2259 raw=True):
60df071c
LD
2260 if regexp.search(line):
2261 if verbose:
c0d2b074 2262 print("got keyword match on %s in %s in %s" % (regexp.pattern, line, file))
e665e98e 2263 kwfiles[file] = regexp
60df071c
LD
2264 break
2265
e665e98e 2266 for file, regexp in kwfiles.items():
60df071c 2267 if verbose:
e665e98e 2268 print("zapping %s with %s" % (line, regexp.pattern))
d20f0f8e
PW
2269 # File is being deleted, so not open in p4. Must
2270 # disable the read-only bit on windows.
2271 if self.isWindows and file not in editedFiles:
2272 os.chmod(file, stat.S_IWRITE)
60df071c
LD
2273 self.patchRCSKeywords(file, kwfiles[file])
2274 fixed_rcs_keywords = True
2275
2276 if fixed_rcs_keywords:
f2606b17 2277 print("Retrying the patch with RCS keywords cleaned up")
60df071c
LD
2278 if os.system(tryPatchCmd) == 0:
2279 patch_succeeded = True
1ec4a0a3 2280 print("Patch succeesed this time with RCS keywords cleaned")
60df071c
LD
2281
2282 if not patch_succeeded:
7e5dd9f2
PW
2283 for f in editedFiles:
2284 p4_revert(f)
7e5dd9f2 2285 return False
51a2640a 2286
55ac2ed6
PW
2287 #
2288 # Apply the patch for real, and do add/delete/+x handling.
2289 #
3d8a3038 2290 system(applyPatchCmd, shell=True)
4f5cf76a 2291
a02b8bc4
RP
2292 for f in filesToChangeType:
2293 p4_edit(f, "-t", "auto")
4f5cf76a 2294 for f in filesToAdd:
6de040df 2295 p4_add(f)
4f5cf76a 2296 for f in filesToDelete:
6de040df
LD
2297 p4_revert(f)
2298 p4_delete(f)
4f5cf76a 2299
c65b670e
CP
2300 # Set/clear executable bits
2301 for f in filesToChangeExecBit.keys():
2302 mode = filesToChangeExecBit[f]
2303 setP4ExecBit(f, mode)
2304
8cf422db
LD
2305 update_shelve = 0
2306 if len(self.update_shelve) > 0:
2307 update_shelve = self.update_shelve.pop(0)
2308 p4_reopen_in_change(update_shelve, all_files)
46c609e9 2309
55ac2ed6
PW
2310 #
2311 # Build p4 change description, starting with the contents
2312 # of the git commit message.
2313 #
0e36f2d7 2314 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 2315 logMessage = logMessage.strip()
0874bb01 2316 logMessage, jobs = self.separate_jobs_from_description(logMessage)
4f5cf76a 2317
8cf422db 2318 template = self.prepareSubmitTemplate(update_shelve)
f19cb0a0 2319 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 2320
c47178d4 2321 if self.preserveUser:
812ee74e 2322 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 2323
55ac2ed6
PW
2324 if self.checkAuthorship and not self.p4UserIsMe(p4User):
2325 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
2326 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
2327 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 2328
55ac2ed6 2329 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
2330 if not self.prepare_p4_only:
2331 submitTemplate += separatorLine
df8a9e86 2332 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
55ac2ed6 2333
0874bb01 2334 handle, fileName = tempfile.mkstemp()
e2a892ee 2335 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
2336 if self.isWindows:
2337 submitTemplate = submitTemplate.replace("\n", "\r\n")
6cec21a8 2338 tmpFile.write(encode_text_stream(submitTemplate))
c47178d4
PW
2339 tmpFile.close()
2340
b7638fed 2341 submitted = False
cdc7e388 2342
b7638fed 2343 try:
38ecf752
BK
2344 # Allow the hook to edit the changelist text before presenting it
2345 # to the user.
2346 if not run_git_hook("p4-prepare-changelist", [fileName]):
2347 return False
cd1e0dc4
BK
2348
2349 if self.prepare_p4_only:
2350 #
2351 # Leave the p4 tree prepared, and the submit template around
2352 # and let the user decide what to do next
2353 #
2354 submitted = True
2355 print("")
2356 print("P4 workspace prepared for submission.")
2357 print("To submit or revert, go to client workspace")
2358 print(" " + self.clientPath)
2359 print("")
2360 print("To submit, use \"p4 submit\" to write a new description,")
968e29e1 2361 print("or \"p4 submit -i <%s\" to use the one prepared by"
cd1e0dc4
BK
2362 " \"git p4\"." % fileName)
2363 print("You can delete the file \"%s\" when finished." % fileName)
2364
2365 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
968e29e1
JH
2366 print("To preserve change ownership by user %s, you must\n"
2367 "do \"p4 change -f <change>\" after submitting and\n"
cd1e0dc4
BK
2368 "edit the User field.")
2369 if pureRenameCopy:
2370 print("After submitting, renamed files must be re-synced.")
2371 print("Invoke \"p4 sync -f\" on each of these files:")
2372 for f in pureRenameCopy:
2373 print(" " + f)
2374
2375 print("")
2376 print("To revert the changes, use \"p4 revert ...\", and delete")
2377 print("the submit template file \"%s\"" % fileName)
2378 if filesToAdd:
2379 print("Since the commit adds new files, they must be deleted:")
2380 for f in filesToAdd:
2381 print(" " + f)
2382 print("")
2383 sys.stdout.flush()
2384 return True
2385
b7638fed 2386 if self.edit_template(fileName):
38ecf752
BK
2387 if not self.no_verify:
2388 if not run_git_hook("p4-changelist", [fileName]):
2389 print("The p4-changelist hook failed.")
2390 sys.stdout.flush()
2391 return False
2392
b7638fed
GE
2393 # read the edited message and submit
2394 tmpFile = open(fileName, "rb")
6cec21a8 2395 message = decode_text_stream(tmpFile.read())
b7638fed
GE
2396 tmpFile.close()
2397 if self.isWindows:
2398 message = message.replace("\r\n", "\n")
cd1e0dc4
BK
2399 if message.find(separatorLine) != -1:
2400 submitTemplate = message[:message.index(separatorLine)]
2401 else:
2402 submitTemplate = message
2403
2404 if len(submitTemplate.strip()) == 0:
2405 print("Changelist is empty, aborting this changelist.")
2406 sys.stdout.flush()
2407 return False
46c609e9 2408
8cf422db 2409 if update_shelve:
46c609e9
LD
2410 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2411 elif self.shelve:
b34fa577
VK
2412 p4_write_pipe(['shelve', '-i'], submitTemplate)
2413 else:
2414 p4_write_pipe(['submit', '-i'], submitTemplate)
2415 # The rename/copy happened by applying a patch that created a
2416 # new file. This leaves it writable, which confuses p4.
2417 for f in pureRenameCopy:
2418 p4_sync(f, "-f")
b7638fed
GE
2419
2420 if self.preserveUser:
2421 if p4User:
2422 # Get last changelist number. Cannot easily get it from
2423 # the submit command output as the output is
2424 # unmarshalled.
2425 changelist = self.lastP4Changelist()
2426 self.modifyChangelistUser(changelist, p4User)
2427
b7638fed
GE
2428 submitted = True
2429
38ecf752 2430 run_git_hook("p4-post-changelist")
b7638fed 2431 finally:
cd1e0dc4 2432 # Revert changes if we skip this patch
b34fa577
VK
2433 if not submitted or self.shelve:
2434 if self.shelve:
843d847f 2435 print("Reverting shelved files.")
b34fa577 2436 else:
843d847f 2437 print("Submission cancelled, undoing p4 changes.")
cd1e0dc4 2438 sys.stdout.flush()
b34fa577 2439 for f in editedFiles | filesToDelete:
b7638fed
GE
2440 p4_revert(f)
2441 for f in filesToAdd:
2442 p4_revert(f)
2443 os.remove(f)
c47178d4 2444
cd1e0dc4
BK
2445 if not self.prepare_p4_only:
2446 os.remove(fileName)
b7638fed 2447 return submitted
4f5cf76a 2448
06804c76 2449 def exportGitTags(self, gitTags):
522e914f
JH
2450 """Export git tags as p4 labels. Create a p4 label and then tag with
2451 that.
2452 """
2453
c8942a22
LD
2454 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2455 if len(validLabelRegexp) == 0:
2456 validLabelRegexp = defaultLabelRegexp
2457 m = re.compile(validLabelRegexp)
06804c76
LD
2458
2459 for name in gitTags:
2460
2461 if not m.match(name):
2462 if verbose:
f2606b17 2463 print("tag %s does not match regexp %s" % (name, validLabelRegexp))
06804c76
LD
2464 continue
2465
2466 # Get the p4 commit this corresponds to
c8942a22
LD
2467 logMessage = extractLogMessageFromGitCommit(name)
2468 values = extractSettingsGitLog(logMessage)
06804c76 2469
dba1c9d9 2470 if 'change' not in values:
06804c76
LD
2471 # a tag pointing to something not sent to p4; ignore
2472 if verbose:
f2606b17 2473 print("git tag %s does not give a p4 commit" % name)
06804c76 2474 continue
c8942a22
LD
2475 else:
2476 changelist = values['change']
06804c76
LD
2477
2478 # Get the tag details.
2479 inHeader = True
2480 isAnnotated = False
2481 body = []
2482 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2483 l = l.strip()
2484 if inHeader:
2485 if re.match(r'tag\s+', l):
2486 isAnnotated = True
2487 elif re.match(r'\s*$', l):
2488 inHeader = False
2489 continue
2490 else:
2491 body.append(l)
2492
2493 if not isAnnotated:
2494 body = ["lightweight tag imported by git p4\n"]
2495
2496 # Create the label - use the same view as the client spec we are using
2497 clientSpec = getClientSpec()
2498
6febb9f8 2499 labelTemplate = "Label: %s\n" % name
06804c76
LD
2500 labelTemplate += "Description:\n"
2501 for b in body:
2502 labelTemplate += "\t" + b + "\n"
2503 labelTemplate += "View:\n"
9d57c4a6
KS
2504 for depot_side in clientSpec.mappings:
2505 labelTemplate += "\t%s\n" % depot_side
06804c76 2506
ef739f08 2507 if self.dry_run:
f2606b17 2508 print("Would create p4 label %s for tag" % name)
728b7ad8 2509 elif self.prepare_p4_only:
968e29e1 2510 print("Not creating p4 label %s for tag due to option"
f2606b17 2511 " --prepare-p4-only" % name)
ef739f08
PW
2512 else:
2513 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 2514
ef739f08
PW
2515 # Use the label
2516 p4_system(["tag", "-l", name] +
9d57c4a6 2517 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 2518
ef739f08 2519 if verbose:
f2606b17 2520 print("created p4 label for tag %s" % name)
06804c76 2521
4f5cf76a 2522 def run(self, args):
c9b50e63
SH
2523 if len(args) == 0:
2524 self.master = currentGitBranch()
c9b50e63
SH
2525 elif len(args) == 1:
2526 self.master = args[0]
28755dba
PW
2527 if not branchExists(self.master):
2528 die("Branch %s does not exist" % self.master)
c9b50e63
SH
2529 else:
2530 return False
2531
8cf422db
LD
2532 for i in self.update_shelve:
2533 if i <= 0:
2534 sys.exit("invalid changelist %d" % i)
2535
00ad6e31
LD
2536 if self.master:
2537 allowSubmit = gitConfig("git-p4.allowSubmit")
2538 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2539 die("%s is not in git-p4.allowSubmit" % self.master)
4c2d5d72 2540
0874bb01 2541 upstream, settings = findUpstreamBranchPoint()
ea99c3ae 2542 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
2543 if len(self.origin) == 0:
2544 self.origin = upstream
a3fdd579 2545
8cf422db 2546 if len(self.update_shelve) > 0:
46c609e9
LD
2547 self.shelve = True
2548
3ea2cfd4
LD
2549 if self.preserveUser:
2550 if not self.canChangeChangelists():
2551 die("Cannot preserve user names without p4 super-user or admin permissions")
2552
6bbfd137
PW
2553 # if not set from the command line, try the config file
2554 if self.conflict_behavior is None:
2555 val = gitConfig("git-p4.conflict")
2556 if val:
2557 if val not in self.conflict_behavior_choices:
2558 die("Invalid value '%s' for config git-p4.conflict" % val)
2559 else:
2560 val = "ask"
2561 self.conflict_behavior = val
2562
a3fdd579 2563 if self.verbose:
f2606b17 2564 print("Origin branch is " + self.origin)
9512497b 2565
ea99c3ae 2566 if len(self.depotPath) == 0:
f2606b17 2567 print("Internal error: cannot locate perforce depot path from existing branches")
9512497b
SH
2568 sys.exit(128)
2569
543987bd 2570 self.useClientSpec = False
0d609032 2571 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
2572 self.useClientSpec = True
2573 if self.useClientSpec:
2574 self.clientSpecDirs = getClientSpec()
9512497b 2575
2e3a16b2 2576 # Check for the existence of P4 branches
cd884106
VA
2577 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2578
2579 if self.useClientSpec and not branchesDetected:
543987bd
PW
2580 # all files are relative to the client spec
2581 self.clientPath = getClientRoot()
2582 else:
2583 self.clientPath = p4Where(self.depotPath)
9512497b 2584
543987bd
PW
2585 if self.clientPath == "":
2586 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 2587
f2606b17 2588 print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
7944f142 2589 self.oldWorkingDirectory = os.getcwd()
c1b296b9 2590
0591cfa8 2591 # ensure the clientPath exists
8d7ec362 2592 new_client_dir = False
0591cfa8 2593 if not os.path.exists(self.clientPath):
8d7ec362 2594 new_client_dir = True
0591cfa8
GG
2595 os.makedirs(self.clientPath)
2596
bbd84863 2597 chdir(self.clientPath, is_client_path=True)
ef739f08 2598 if self.dry_run:
f2606b17 2599 print("Would synchronize p4 checkout in %s" % self.clientPath)
8d7ec362 2600 else:
f2606b17 2601 print("Synchronizing p4 checkout...")
ef739f08
PW
2602 if new_client_dir:
2603 # old one was destroyed, and maybe nobody told p4
2604 p4_sync("...", "-f")
2605 else:
2606 p4_sync("...")
4f5cf76a 2607 self.check()
4f5cf76a 2608
4c750c0d 2609 commits = []
00ad6e31 2610 if self.master:
89f32a92 2611 committish = self.master
00ad6e31 2612 else:
89f32a92 2613 committish = 'HEAD'
00ad6e31 2614
f55b87c1
RM
2615 if self.commit != "":
2616 if self.commit.find("..") != -1:
2617 limits_ish = self.commit.split("..")
2618 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2619 commits.append(line.strip())
2620 commits.reverse()
2621 else:
2622 commits.append(self.commit)
2623 else:
e6388994 2624 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
f55b87c1
RM
2625 commits.append(line.strip())
2626 commits.reverse()
4f5cf76a 2627
0d609032 2628 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
2629 self.checkAuthorship = False
2630 else:
2631 self.checkAuthorship = True
2632
3ea2cfd4
LD
2633 if self.preserveUser:
2634 self.checkValidP4Users(commits)
2635
84cb0003
GG
2636 #
2637 # Build up a set of options to be passed to diff when
2638 # submitting each commit to p4.
2639 #
2640 if self.detectRenames:
2641 # command-line -M arg
8a470599 2642 self.diffOpts = ["-M"]
84cb0003
GG
2643 else:
2644 # If not explicitly set check the config variable
2645 detectRenames = gitConfig("git-p4.detectRenames")
2646
2647 if detectRenames.lower() == "false" or detectRenames == "":
8a470599 2648 self.diffOpts = []
84cb0003 2649 elif detectRenames.lower() == "true":
8a470599 2650 self.diffOpts = ["-M"]
84cb0003 2651 else:
8a470599 2652 self.diffOpts = ["-M{}".format(detectRenames)]
84cb0003
GG
2653
2654 # no command-line arg for -C or --find-copies-harder, just
2655 # config variables
2656 detectCopies = gitConfig("git-p4.detectCopies")
2657 if detectCopies.lower() == "false" or detectCopies == "":
2658 pass
2659 elif detectCopies.lower() == "true":
8a470599 2660 self.diffOpts.append("-C")
84cb0003 2661 else:
8a470599 2662 self.diffOpts.append("-C{}".format(detectCopies))
84cb0003 2663
0d609032 2664 if gitConfigBool("git-p4.detectCopiesHarder"):
8a470599 2665 self.diffOpts.append("--find-copies-harder")
84cb0003 2666
8cf422db
LD
2667 num_shelves = len(self.update_shelve)
2668 if num_shelves > 0 and num_shelves != len(commits):
2669 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2670 (len(commits), num_shelves))
2671
4935c458
BK
2672 if not self.no_verify:
2673 try:
2674 if not run_git_hook("p4-pre-submit"):
968e29e1
JH
2675 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nYou can skip "
2676 "this pre-submission check by adding\nthe command line option '--no-verify', "
4935c458
BK
2677 "however,\nthis will also skip the p4-changelist hook as well.")
2678 sys.exit(1)
2679 except Exception as e:
968e29e1 2680 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nThe hook failed "
84af8b85 2681 "with the error '{0}'".format(e.message))
aa8b766a 2682 sys.exit(1)
251c8c50 2683
7e5dd9f2
PW
2684 #
2685 # Apply the commits, one at a time. On failure, ask if should
2686 # continue to try the rest of the patches, or quit.
2687 #
ef739f08 2688 if self.dry_run:
f2606b17 2689 print("Would apply")
67b0fe2e 2690 applied = []
7e5dd9f2
PW
2691 last = len(commits) - 1
2692 for i, commit in enumerate(commits):
ef739f08 2693 if self.dry_run:
f2606b17
LD
2694 print(" ", read_pipe(["git", "show", "-s",
2695 "--format=format:%h %s", commit]))
ef739f08
PW
2696 ok = True
2697 else:
2698 ok = self.applyCommit(commit)
67b0fe2e
PW
2699 if ok:
2700 applied.append(commit)
2dfdd705
BK
2701 if self.prepare_p4_only:
2702 if i < last:
968e29e1 2703 print("Processing only the first commit due to option"
2dfdd705 2704 " --prepare-p4-only")
728b7ad8 2705 break
2dfdd705 2706 else:
7e5dd9f2 2707 if i < last:
e2aed5fd
BK
2708 # prompt for what to do, or use the option/variable
2709 if self.conflict_behavior == "ask":
2710 print("What do you want to do?")
2711 response = prompt("[s]kip this commit but apply the rest, or [q]uit? ")
2712 elif self.conflict_behavior == "skip":
2713 response = "s"
2714 elif self.conflict_behavior == "quit":
2715 response = "q"
2716 else:
2717 die("Unknown conflict_behavior '%s'" %
2718 self.conflict_behavior)
2719
2720 if response == "s":
2721 print("Skipping this commit, but applying the rest")
2722 if response == "q":
2723 print("Quitting")
7e5dd9f2 2724 break
4f5cf76a 2725
67b0fe2e 2726 chdir(self.oldWorkingDirectory)
b34fa577 2727 shelved_applied = "shelved" if self.shelve else "applied"
ef739f08
PW
2728 if self.dry_run:
2729 pass
728b7ad8
PW
2730 elif self.prepare_p4_only:
2731 pass
ef739f08 2732 elif len(commits) == len(applied):
f2606b17 2733 print("All commits {0}!".format(shelved_applied))
14594f4b 2734
4c750c0d 2735 sync = P4Sync()
44e8d26c
PW
2736 if self.branch:
2737 sync.branch = self.branch
b9d34db9
LD
2738 if self.disable_p4sync:
2739 sync.sync_origin_only()
2740 else:
2741 sync.run([])
14594f4b 2742
b9d34db9
LD
2743 if not self.disable_rebase:
2744 rebase = P4Rebase()
2745 rebase.rebase()
4f5cf76a 2746
67b0fe2e
PW
2747 else:
2748 if len(applied) == 0:
f2606b17 2749 print("No commits {0}.".format(shelved_applied))
67b0fe2e 2750 else:
f2606b17 2751 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
67b0fe2e
PW
2752 for c in commits:
2753 if c in applied:
2754 star = "*"
2755 else:
2756 star = " "
f2606b17
LD
2757 print(star, read_pipe(["git", "show", "-s",
2758 "--format=format:%h %s", c]))
2759 print("You will have to do 'git p4 sync' and rebase.")
67b0fe2e 2760
0d609032 2761 if gitConfigBool("git-p4.exportLabels"):
06dcd152 2762 self.exportLabels = True
06804c76
LD
2763
2764 if self.exportLabels:
2765 p4Labels = getP4Labels(self.depotPath)
2766 gitTags = getGitTags()
2767
2768 missingGitTags = gitTags - p4Labels
2769 self.exportGitTags(missingGitTags)
2770
98e023de 2771 # exit with error unless everything applied perfectly
67b0fe2e 2772 if len(commits) != len(applied):
812ee74e 2773 sys.exit(1)
67b0fe2e 2774
b984733c
SH
2775 return True
2776
adf159b4 2777
ecb7cf98 2778class View(object):
59ef3fc1
JH
2779 """Represent a p4 view ("p4 help views"), and map files in a repo according
2780 to the view.
2781 """
ecb7cf98 2782
9d57c4a6 2783 def __init__(self, client_name):
ecb7cf98 2784 self.mappings = []
9d57c4a6
KS
2785 self.client_prefix = "//%s/" % client_name
2786 # cache results of "p4 where" to lookup client file locations
2787 self.client_spec_path_cache = {}
ecb7cf98
PW
2788
2789 def append(self, view_line):
59ef3fc1
JH
2790 """Parse a view line, splitting it into depot and client sides. Append
2791 to self.mappings, preserving order. This is only needed for tag
2792 creation.
2793 """
ecb7cf98
PW
2794
2795 # Split the view line into exactly two words. P4 enforces
2796 # structure on these lines that simplifies this quite a bit.
2797 #
2798 # Either or both words may be double-quoted.
2799 # Single quotes do not matter.
2800 # Double-quote marks cannot occur inside the words.
2801 # A + or - prefix is also inside the quotes.
2802 # There are no quotes unless they contain a space.
2803 # The line is already white-space stripped.
2804 # The two words are separated by a single space.
2805 #
2806 if view_line[0] == '"':
2807 # First word is double quoted. Find its end.
2808 close_quote_index = view_line.find('"', 1)
2809 if close_quote_index <= 0:
2810 die("No first-word closing quote found: %s" % view_line)
2811 depot_side = view_line[1:close_quote_index]
2812 # skip closing quote and space
2813 rhs_index = close_quote_index + 1 + 1
2814 else:
2815 space_index = view_line.find(" ")
2816 if space_index <= 0:
2817 die("No word-splitting space found: %s" % view_line)
2818 depot_side = view_line[0:space_index]
2819 rhs_index = space_index + 1
2820
ecb7cf98 2821 # prefix + means overlay on previous mapping
ecb7cf98 2822 if depot_side.startswith("+"):
ecb7cf98
PW
2823 depot_side = depot_side[1:]
2824
9d57c4a6 2825 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
2826 exclude = False
2827 if depot_side.startswith("-"):
2828 exclude = True
2829 depot_side = depot_side[1:]
2830
9d57c4a6
KS
2831 if not exclude:
2832 self.mappings.append(depot_side)
ecb7cf98 2833
9d57c4a6
KS
2834 def convert_client_path(self, clientFile):
2835 # chop off //client/ part to make it relative
d38208a2 2836 if not decode_path(clientFile).startswith(self.client_prefix):
9d57c4a6
KS
2837 die("No prefix '%s' on clientFile '%s'" %
2838 (self.client_prefix, clientFile))
2839 return clientFile[len(self.client_prefix):]
ecb7cf98 2840
9d57c4a6 2841 def update_client_spec_path_cache(self, files):
59ef3fc1 2842 """Caching file paths by "p4 where" batch query."""
ecb7cf98 2843
9d57c4a6 2844 # List depot file paths exclude that already cached
d38208a2 2845 fileArgs = [f['path'] for f in files if decode_path(f['path']) not in self.client_spec_path_cache]
ecb7cf98 2846
9d57c4a6
KS
2847 if len(fileArgs) == 0:
2848 return # All files in cache
ecb7cf98 2849
9d57c4a6
KS
2850 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2851 for res in where_result:
2852 if "code" in res and res["code"] == "error":
2853 # assume error is "... file(s) not in client view"
2854 continue
2855 if "clientFile" not in res:
20005443 2856 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
2857 if "unmap" in res:
2858 # it will list all of them, but only one not unmap-ped
2859 continue
d38208a2 2860 depot_path = decode_path(res['depotFile'])
a0a50d87 2861 if gitConfigBool("core.ignorecase"):
d38208a2
YZ
2862 depot_path = depot_path.lower()
2863 self.client_spec_path_cache[depot_path] = self.convert_client_path(res["clientFile"])
ecb7cf98 2864
9d57c4a6
KS
2865 # not found files or unmap files set to ""
2866 for depotFile in fileArgs:
d38208a2 2867 depotFile = decode_path(depotFile)
a0a50d87
LS
2868 if gitConfigBool("core.ignorecase"):
2869 depotFile = depotFile.lower()
9d57c4a6 2870 if depotFile not in self.client_spec_path_cache:
d38208a2 2871 self.client_spec_path_cache[depotFile] = b''
ecb7cf98 2872
9d57c4a6 2873 def map_in_client(self, depot_path):
59ef3fc1
JH
2874 """Return the relative location in the client where this depot file
2875 should live.
2876
2877 Returns "" if the file should not be mapped in the client.
2878 """
ecb7cf98 2879
a0a50d87
LS
2880 if gitConfigBool("core.ignorecase"):
2881 depot_path = depot_path.lower()
2882
9d57c4a6
KS
2883 if depot_path in self.client_spec_path_cache:
2884 return self.client_spec_path_cache[depot_path]
2885
84af8b85 2886 die("Error: %s is not found in client spec path" % depot_path)
9d57c4a6 2887 return ""
ecb7cf98 2888
adf159b4 2889
ff8c50ed
AM
2890def cloneExcludeCallback(option, opt_str, value, parser):
2891 # prepend "/" because the first "/" was consumed as part of the option itself.
2892 # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2893 parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)]
2894
adf159b4 2895
3ea2cfd4 2896class P4Sync(Command, P4UserMap):
56c09345 2897
b984733c
SH
2898 def __init__(self):
2899 Command.__init__(self)
3ea2cfd4 2900 P4UserMap.__init__(self)
b984733c
SH
2901 self.options = [
2902 optparse.make_option("--branch", dest="branch"),
2903 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2904 optparse.make_option("--changesfile", dest="changesFile"),
2905 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2906 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2907 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2908 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2909 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
2910 optparse.make_option("--max-changes", dest="maxChanges",
2911 help="Maximum number of changes to import"),
2912 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2913 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 2914 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2915 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2916 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
2917 help="Only sync files that are included in the Perforce Client Spec"),
2918 optparse.make_option("-/", dest="cloneExclude",
ff8c50ed 2919 action="callback", callback=cloneExcludeCallback, type="string",
51334bb0 2920 help="exclude depot path"),
b984733c
SH
2921 ]
2922 self.description = """Imports from Perforce into a git repository.\n
2923 example:
2924 //depot/my/project/ -- to import the current head
2925 //depot/my/project/@all -- to import everything
2926 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2927
2928 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2929
2930 self.usage += " //depot/path[@revRange]"
b984733c 2931 self.silent = False
1d7367dc
RG
2932 self.createdBranches = set()
2933 self.committedChanges = set()
569d1bd4 2934 self.branch = ""
b984733c 2935 self.detectBranches = False
cb53e1f8 2936 self.detectLabels = False
06804c76 2937 self.importLabels = False
b984733c 2938 self.changesFile = ""
01265103 2939 self.syncWithOrigin = True
a028a98e 2940 self.importIntoRemotes = True
01a9c9c5 2941 self.maxChanges = ""
1051ef00 2942 self.changes_block_size = None
8b41a97f 2943 self.keepRepoPath = False
6326aa58 2944 self.depotPaths = None
3c699645 2945 self.p4BranchesInGit = []
354081d5 2946 self.cloneExclude = []
3a70cdfa 2947 self.useClientSpec = False
a93d33ee 2948 self.useClientSpec_from_options = False
ecb7cf98 2949 self.clientSpecDirs = None
fed23693 2950 self.tempBranches = []
d604176d 2951 self.tempBranchLocation = "refs/git-p4-tmp"
a5db4b12 2952 self.largeFileSystem = None
123f6317 2953 self.suppress_meta_comment = False
a5db4b12
LS
2954
2955 if gitConfig('git-p4.largeFileSystem'):
2956 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2957 self.largeFileSystem = largeFileSystemConstructor(
2958 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2959 )
b984733c 2960
01265103
SH
2961 if gitConfig("git-p4.syncFromOrigin") == "false":
2962 self.syncWithOrigin = False
2963
123f6317
LD
2964 self.depotPaths = []
2965 self.changeRange = ""
2966 self.previousDepotPaths = []
2967 self.hasOrigin = False
2968
2969 # map from branch depot path to parent branch
2970 self.knownBranches = {}
2971 self.initialParents = {}
2972
2973 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2974 self.labels = {}
2975
fed23693 2976 def checkpoint(self):
522e914f 2977 """Force a checkpoint in fast-import and wait for it to finish."""
fed23693
VA
2978 self.gitStream.write("checkpoint\n\n")
2979 self.gitStream.write("progress checkpoint\n\n")
4294d741 2980 self.gitStream.flush()
fed23693
VA
2981 out = self.gitOutput.readline()
2982 if self.verbose:
f2606b17 2983 print("checkpoint finished: " + out)
fed23693 2984
a2bee10a
AM
2985 def isPathWanted(self, path):
2986 for p in self.cloneExclude:
2987 if p.endswith("/"):
2988 if p4PathStartsWith(path, p):
2989 return False
2990 # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2991 elif path.lower() == p.lower():
2992 return False
2993 for p in self.depotPaths:
d38208a2 2994 if p4PathStartsWith(path, decode_path(p)):
a2bee10a
AM
2995 return True
2996 return False
2997
57fe2ce0 2998 def extractFilesFromCommit(self, commit, shelved=False, shelved_cl=0):
b984733c
SH
2999 files = []
3000 fnum = 0
dba1c9d9 3001 while "depotFile%s" % fnum in commit:
6febb9f8 3002 path = commit["depotFile%s" % fnum]
d38208a2 3003 found = self.isPathWanted(decode_path(path))
6326aa58 3004 if not found:
b984733c
SH
3005 fnum = fnum + 1
3006 continue
3007
3008 file = {}
3009 file["path"] = path
3010 file["rev"] = commit["rev%s" % fnum]
3011 file["action"] = commit["action%s" % fnum]
3012 file["type"] = commit["type%s" % fnum]
123f6317
LD
3013 if shelved:
3014 file["shelved_cl"] = int(shelved_cl)
b984733c
SH
3015 files.append(file)
3016 fnum = fnum + 1
3017 return files
3018
26e6a27d
JD
3019 def extractJobsFromCommit(self, commit):
3020 jobs = []
3021 jnum = 0
dba1c9d9 3022 while "job%s" % jnum in commit:
26e6a27d
JD
3023 job = commit["job%s" % jnum]
3024 jobs.append(job)
3025 jnum = jnum + 1
3026 return jobs
3027
6326aa58 3028 def stripRepoPath(self, path, prefixes):
59ef3fc1
JH
3029 """When streaming files, this is called to map a p4 depot path to where
3030 it should go in git. The prefixes are either self.depotPaths, or
3031 self.branchPrefixes in the case of branch detection.
3032 """
21ef5df4 3033
3952710b 3034 if self.useClientSpec:
21ef5df4
PW
3035 # branch detection moves files up a level (the branch name)
3036 # from what client spec interpretation gives
d38208a2 3037 path = decode_path(self.clientSpecDirs.map_in_client(path))
21ef5df4
PW
3038 if self.detectBranches:
3039 for b in self.knownBranches:
f2768cb3 3040 if p4PathStartsWith(path, b + "/"):
21ef5df4
PW
3041 path = path[len(b)+1:]
3042
3043 elif self.keepRepoPath:
3044 # Preserve everything in relative path name except leading
3045 # //depot/; just look at first prefix as they all should
3046 # be in the same depot.
3047 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
3048 if p4PathStartsWith(path, depot):
3049 path = path[len(depot):]
3952710b 3050
0d1696ef 3051 else:
0d1696ef
PW
3052 for p in prefixes:
3053 if p4PathStartsWith(path, p):
3054 path = path[len(p):]
21ef5df4 3055 break
8b41a97f 3056
0d1696ef 3057 path = wildcard_decode(path)
6326aa58 3058 return path
6754a299 3059
71b112d4 3060 def splitFilesIntoBranches(self, commit):
59ef3fc1
JH
3061 """Look at each depotFile in the commit to figure out to what branch it
3062 belongs.
3063 """
21ef5df4 3064
9d57c4a6
KS
3065 if self.clientSpecDirs:
3066 files = self.extractFilesFromCommit(commit)
3067 self.clientSpecDirs.update_client_spec_path_cache(files)
3068
d5904674 3069 branches = {}
71b112d4 3070 fnum = 0
dba1c9d9 3071 while "depotFile%s" % fnum in commit:
d38208a2
YZ
3072 raw_path = commit["depotFile%s" % fnum]
3073 path = decode_path(raw_path)
d15068a6 3074 found = self.isPathWanted(path)
6326aa58 3075 if not found:
71b112d4
SH
3076 fnum = fnum + 1
3077 continue
3078
3079 file = {}
d38208a2 3080 file["path"] = raw_path
71b112d4
SH
3081 file["rev"] = commit["rev%s" % fnum]
3082 file["action"] = commit["action%s" % fnum]
3083 file["type"] = commit["type%s" % fnum]
3084 fnum = fnum + 1
3085
21ef5df4
PW
3086 # start with the full relative path where this file would
3087 # go in a p4 client
3088 if self.useClientSpec:
d38208a2 3089 relPath = decode_path(self.clientSpecDirs.map_in_client(path))
21ef5df4
PW
3090 else:
3091 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 3092
4b97ffb1 3093 for branch in self.knownBranches.keys():
21ef5df4
PW
3094 # add a trailing slash so that a commit into qt/4.2foo
3095 # doesn't end up in qt/4.2, e.g.
f2768cb3 3096 if p4PathStartsWith(relPath, branch + "/"):
d5904674
SH
3097 if branch not in branches:
3098 branches[branch] = []
71b112d4 3099 branches[branch].append(file)
6555b2cc 3100 break
b984733c
SH
3101
3102 return branches
3103
a5db4b12 3104 def writeToGitStream(self, gitMode, relPath, contents):
6cec21a8 3105 self.gitStream.write(encode_text_stream(u'M {} inline {}\n'.format(gitMode, relPath)))
a5db4b12
LS
3106 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
3107 for d in contents:
3108 self.gitStream.write(d)
3109 self.gitStream.write('\n')
3110
a8b05162
LS
3111 def encodeWithUTF8(self, path):
3112 try:
3113 path.decode('ascii')
3114 except:
3115 encoding = 'utf8'
3116 if gitConfig('git-p4.pathEncoding'):
3117 encoding = gitConfig('git-p4.pathEncoding')
3118 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
3119 if self.verbose:
f2606b17 3120 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
a8b05162
LS
3121 return path
3122
b932705b 3123 def streamOneP4File(self, file, contents):
522e914f
JH
3124 """Output one file from the P4 stream.
3125
3126 This is a helper for streamP4Files().
3127 """
3128
d38208a2
YZ
3129 file_path = file['depotFile']
3130 relPath = self.stripRepoPath(decode_path(file_path), self.branchPrefixes)
3131
b932705b 3132 if verbose:
0742b7c8
LD
3133 if 'fileSize' in self.stream_file:
3134 size = int(self.stream_file['fileSize'])
3135 else:
4768af20
JH
3136 # Deleted files don't get a fileSize apparently
3137 size = 0
ae9b9509
JH
3138 sys.stdout.write('\r%s --> %s (%s)\n' % (
3139 file_path, relPath, format_size_human_readable(size)))
d2176a50 3140 sys.stdout.flush()
b932705b 3141
0874bb01 3142 type_base, type_mods = split_p4_type(file["type"])
9cffb8c8
PW
3143
3144 git_mode = "100644"
3145 if "x" in type_mods:
3146 git_mode = "100755"
3147 if type_base == "symlink":
3148 git_mode = "120000"
1292df11
AJ
3149 # p4 print on a symlink sometimes contains "target\n";
3150 # if it does, remove the newline
6cec21a8 3151 data = ''.join(decode_text_stream(c) for c in contents)
40f846c3
PW
3152 if not data:
3153 # Some version of p4 allowed creating a symlink that pointed
3154 # to nothing. This causes p4 errors when checking out such
3155 # a change, and errors here too. Work around it by ignoring
3156 # the bad symlink; hopefully a future change fixes it.
d38208a2 3157 print("\nIgnoring empty symlink in %s" % file_path)
40f846c3
PW
3158 return
3159 elif data[-1] == '\n':
1292df11
AJ
3160 contents = [data[:-1]]
3161 else:
3162 contents = [data]
b932705b 3163
9cffb8c8 3164 if type_base == "utf16":
55aa5714
PW
3165 # p4 delivers different text in the python output to -G
3166 # than it does when using "print -o", or normal p4 client
3167 # operations. utf16 is converted to ascii or utf8, perhaps.
3168 # But ascii text saved as -t utf16 is completely mangled.
3169 # Invoke print -o to get the real contents.
7f0e5962
PW
3170 #
3171 # On windows, the newlines will always be mangled by print, so put
3172 # them back too. This is not needed to the cygwin windows version,
3173 # just the native "NT" type.
3174 #
1f5f3907 3175 try:
d38208a2 3176 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (decode_path(file['depotFile']), file['change'])], raw=True)
1f5f3907
LS
3177 except Exception as e:
3178 if 'Translation of file content failed' in str(e):
3179 type_base = 'binary'
3180 else:
3181 raise e
3182 else:
3183 if p4_version_string().find('/NT') >= 0:
4d35f744 3184 text = text.replace(b'\x0d\x00\x0a\x00', b'\x0a\x00')
84af8b85 3185 contents = [text]
55aa5714 3186
9f7ef0ea
PW
3187 if type_base == "apple":
3188 # Apple filetype files will be streamed as a concatenation of
3189 # its appledouble header and the contents. This is useless
3190 # on both macs and non-macs. If using "print -q -o xx", it
3191 # will create "xx" with the data, and "%xx" with the header.
3192 # This is also not very useful.
3193 #
3194 # Ideally, someday, this script can learn how to generate
3195 # appledouble files directly and import those to git, but
3196 # non-mac machines can never find a use for apple filetype.
f2606b17 3197 print("\nIgnoring apple filetype file %s" % file['depotFile'])
9f7ef0ea
PW
3198 return
3199
fbe5f6b8
TK
3200 if type_base == "utf8":
3201 # The type utf8 explicitly means utf8 *with BOM*. These are
3202 # streamed just like regular text files, however, without
3203 # the BOM in the stream.
3204 # Therefore, to accurately import these files into git, we
3205 # need to explicitly re-add the BOM before writing.
3206 # 'contents' is a set of bytes in this case, so create the
3207 # BOM prefix as a b'' literal.
3208 contents = [b'\xef\xbb\xbf' + contents[0]] + contents[1:]
3209
55aa5714
PW
3210 # Note that we do not try to de-mangle keywords on utf16 files,
3211 # even though in theory somebody may want that.
e665e98e
JH
3212 regexp = p4_keywords_regexp_for_type(type_base, type_mods)
3213 if regexp:
70c0d553 3214 contents = [regexp.sub(br'$\1$', c) for c in contents]
b932705b 3215
a5db4b12 3216 if self.largeFileSystem:
0874bb01 3217 git_mode, contents = self.largeFileSystem.processContent(git_mode, relPath, contents)
b932705b 3218
a5db4b12 3219 self.writeToGitStream(git_mode, relPath, contents)
b932705b
LD
3220
3221 def streamOneP4Deletion(self, file):
d38208a2 3222 relPath = self.stripRepoPath(decode_path(file['path']), self.branchPrefixes)
b932705b 3223 if verbose:
d2176a50
LS
3224 sys.stdout.write("delete %s\n" % relPath)
3225 sys.stdout.flush()
6cec21a8 3226 self.gitStream.write(encode_text_stream(u'D {}\n'.format(relPath)))
b932705b 3227
a5db4b12
LS
3228 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
3229 self.largeFileSystem.removeLargeFile(relPath)
3230
b932705b 3231 def streamP4FilesCb(self, marshalled):
522e914f 3232 """Handle another chunk of streaming data."""
b932705b 3233
78189bea
PW
3234 # catch p4 errors and complain
3235 err = None
3236 if "code" in marshalled:
3237 if marshalled["code"] == "error":
3238 if "data" in marshalled:
3239 err = marshalled["data"].rstrip()
4d25dc44
LS
3240
3241 if not err and 'fileSize' in self.stream_file:
3242 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
3243 if required_bytes > 0:
ae9b9509
JH
3244 err = 'Not enough space left on %s! Free at least %s.' % (
3245 os.getcwd(), format_size_human_readable(required_bytes))
4d25dc44 3246
78189bea
PW
3247 if err:
3248 f = None
3249 if self.stream_have_file_info:
3250 if "depotFile" in self.stream_file:
3251 f = self.stream_file["depotFile"]
3252 # force a failure in fast-import, else an empty
3253 # commit will be made
3254 self.gitStream.write("\n")
3255 self.gitStream.write("die-now\n")
3256 self.gitStream.close()
3257 # ignore errors, but make sure it exits first
3258 self.importProcess.wait()
3259 if f:
3260 die("Error from p4 print for %s: %s" % (f, err))
3261 else:
3262 die("Error from p4 print: %s" % err)
3263
dba1c9d9 3264 if 'depotFile' in marshalled and self.stream_have_file_info:
c3f6163b
AG
3265 # start of a new file - output the old one first
3266 self.streamOneP4File(self.stream_file, self.stream_contents)
3267 self.stream_file = {}
3268 self.stream_contents = []
3269 self.stream_have_file_info = False
b932705b 3270
c3f6163b
AG
3271 # pick up the new file information... for the
3272 # 'data' field we need to append to our array
3273 for k in marshalled.keys():
3274 if k == 'data':
d2176a50
LS
3275 if 'streamContentSize' not in self.stream_file:
3276 self.stream_file['streamContentSize'] = 0
3277 self.stream_file['streamContentSize'] += len(marshalled['data'])
c3f6163b
AG
3278 self.stream_contents.append(marshalled['data'])
3279 else:
3280 self.stream_file[k] = marshalled[k]
b932705b 3281
d2176a50 3282 if (verbose and
7a3e83d0
JH
3283 'streamContentSize' in self.stream_file and
3284 'fileSize' in self.stream_file and
3285 'depotFile' in self.stream_file):
d2176a50
LS
3286 size = int(self.stream_file["fileSize"])
3287 if size > 0:
3288 progress = 100*self.stream_file['streamContentSize']/size
ae9b9509
JH
3289 sys.stdout.write('\r%s %d%% (%s)' % (
3290 self.stream_file['depotFile'], progress,
3291 format_size_human_readable(size)))
d2176a50
LS
3292 sys.stdout.flush()
3293
c3f6163b 3294 self.stream_have_file_info = True
b932705b 3295
b932705b 3296 def streamP4Files(self, files):
522e914f
JH
3297 """Stream directly from "p4 files" into "git fast-import."""
3298
30b5940b
SH
3299 filesForCommit = []
3300 filesToRead = []
b932705b 3301 filesToDelete = []
30b5940b 3302
3a70cdfa 3303 for f in files:
ecb7cf98
PW
3304 filesForCommit.append(f)
3305 if f['action'] in self.delete_actions:
3306 filesToDelete.append(f)
3307 else:
3308 filesToRead.append(f)
6a49f8e2 3309
b932705b
LD
3310 # deleted files...
3311 for f in filesToDelete:
3312 self.streamOneP4Deletion(f)
1b9a4684 3313
b932705b
LD
3314 if len(filesToRead) > 0:
3315 self.stream_file = {}
3316 self.stream_contents = []
3317 self.stream_have_file_info = False
8ff45f2a 3318
c3f6163b
AG
3319 # curry self argument
3320 def streamP4FilesCbSelf(entry):
3321 self.streamP4FilesCb(entry)
6a49f8e2 3322
123f6317
LD
3323 fileArgs = []
3324 for f in filesToRead:
3325 if 'shelved_cl' in f:
3326 # Handle shelved CLs using the "p4 print file@=N" syntax to print
3327 # the contents
6cec21a8 3328 fileArg = f['path'] + encode_text_stream('@={}'.format(f['shelved_cl']))
123f6317 3329 else:
6cec21a8 3330 fileArg = f['path'] + encode_text_stream('#{}'.format(f['rev']))
123f6317
LD
3331
3332 fileArgs.append(fileArg)
6de040df
LD
3333
3334 p4CmdList(["-x", "-", "print"],
3335 stdin=fileArgs,
3336 cb=streamP4FilesCbSelf)
30b5940b 3337
b932705b 3338 # do the last chunk
dba1c9d9 3339 if 'depotFile' in self.stream_file:
b932705b 3340 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 3341
affb474f
LD
3342 def make_email(self, userid):
3343 if userid in self.users:
3344 return self.users[userid]
3345 else:
f7b5ff60
TK
3346 userid_bytes = metadata_stream_to_writable_bytes(userid)
3347 return b"%s <a@b>" % userid_bytes
affb474f 3348
06804c76 3349 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
59ef3fc1
JH
3350 """Stream a p4 tag.
3351
3352 Commit is either a git commit, or a fast-import mark, ":<p4commit>".
3353 """
b43702ac 3354
06804c76 3355 if verbose:
f2606b17 3356 print("writing tag %s for commit %s" % (labelName, commit))
06804c76
LD
3357 gitStream.write("tag %s\n" % labelName)
3358 gitStream.write("from %s\n" % commit)
3359
dba1c9d9 3360 if 'Owner' in labelDetails:
06804c76
LD
3361 owner = labelDetails["Owner"]
3362 else:
3363 owner = None
3364
3365 # Try to use the owner of the p4 label, or failing that,
3366 # the current p4 user id.
3367 if owner:
3368 email = self.make_email(owner)
3369 else:
3370 email = self.make_email(self.p4UserId())
06804c76 3371
f7b5ff60
TK
3372 gitStream.write("tagger ")
3373 gitStream.write(email)
3374 gitStream.write(" %s %s\n" % (epoch, self.tz))
06804c76 3375
12a77f5b 3376 print("labelDetails=", labelDetails)
dba1c9d9 3377 if 'Description' in labelDetails:
06804c76
LD
3378 description = labelDetails['Description']
3379 else:
3380 description = 'Label from git p4'
3381
3382 gitStream.write("data %d\n" % len(description))
3383 gitStream.write(description)
3384 gitStream.write("\n")
3385
4ae048e6
LS
3386 def inClientSpec(self, path):
3387 if not self.clientSpecDirs:
3388 return True
3389 inClientSpec = self.clientSpecDirs.map_in_client(path)
3390 if not inClientSpec and self.verbose:
3391 print('Ignoring file outside of client spec: {0}'.format(path))
3392 return inClientSpec
3393
3394 def hasBranchPrefix(self, path):
3395 if not self.branchPrefixes:
3396 return True
3397 hasPrefix = [p for p in self.branchPrefixes
3398 if p4PathStartsWith(path, p)]
09667d01 3399 if not hasPrefix and self.verbose:
4ae048e6
LS
3400 print('Ignoring file outside of prefix: {0}'.format(path))
3401 return hasPrefix
3402
82e46d6b 3403 def findShadowedFiles(self, files, change):
522e914f
JH
3404 """Perforce allows you commit files and directories with the same name,
3405 so you could have files //depot/foo and //depot/foo/bar both checked
3406 in. A p4 sync of a repository in this state fails. Deleting one of
3407 the files recovers the repository.
3408
3409 Git will not allow the broken state to exist and only the most
3410 recent of the conflicting names is left in the repository. When one
3411 of the conflicting files is deleted we need to re-add the other one
3412 to make sure the git repository recovers in the same way as
3413 perforce.
3414 """
3415
82e46d6b
AO
3416 deleted = [f for f in files if f['action'] in self.delete_actions]
3417 to_check = set()
3418 for f in deleted:
3419 path = decode_path(f['path'])
3420 to_check.add(path + '/...')
3421 while True:
3422 path = path.rsplit("/", 1)[0]
3423 if path == "/" or path in to_check:
3424 break
3425 to_check.add(path)
3426 to_check = ['%s@%s' % (wildcard_encode(p), change) for p in to_check
3427 if self.hasBranchPrefix(p)]
3428 if to_check:
3429 stat_result = p4CmdList(["-x", "-", "fstat", "-T",
3430 "depotFile,headAction,headRev,headType"], stdin=to_check)
3431 for record in stat_result:
3432 if record['code'] != 'stat':
3433 continue
3434 if record['headAction'] in self.delete_actions:
3435 continue
3436 files.append({
3437 'action': 'add',
3438 'path': record['depotFile'],
3439 'rev': record['headRev'],
3440 'type': record['headType']})
3441
57fe2ce0 3442 def commit(self, details, files, branch, parent="", allow_empty=False):
b984733c
SH
3443 epoch = details["time"]
3444 author = details["user"]
26e6a27d 3445 jobs = self.extractJobsFromCommit(details)
b984733c 3446
4b97ffb1 3447 if self.verbose:
4ae048e6 3448 print('commit into {0}'.format(branch))
96e07dd2 3449
82e46d6b
AO
3450 files = [f for f in files
3451 if self.hasBranchPrefix(decode_path(f['path']))]
3452 self.findShadowedFiles(files, details['change'])
3453
9d57c4a6
KS
3454 if self.clientSpecDirs:
3455 self.clientSpecDirs.update_client_spec_path_cache(files)
3456
82e46d6b 3457 files = [f for f in files if self.inClientSpec(decode_path(f['path']))]
4ae048e6 3458
89143ac2
LD
3459 if gitConfigBool('git-p4.keepEmptyCommits'):
3460 allow_empty = True
3461
3462 if not files and not allow_empty:
4ae048e6
LS
3463 print('Ignoring revision {0} as it would produce an empty commit.'
3464 .format(details['change']))
3465 return
3466
b984733c 3467 self.gitStream.write("commit %s\n" % branch)
b43702ac 3468 self.gitStream.write("mark :%s\n" % details["change"])
b984733c 3469 self.committedChanges.add(int(details["change"]))
b607e71e
SH
3470 if author not in self.users:
3471 self.getUserMapFromPerforceServer()
b984733c 3472
f7b5ff60
TK
3473 self.gitStream.write("committer ")
3474 self.gitStream.write(self.make_email(author))
3475 self.gitStream.write(" %s %s\n" % (epoch, self.tz))
b984733c
SH
3476
3477 self.gitStream.write("data <<EOT\n")
3478 self.gitStream.write(details["desc"])
26e6a27d
JD
3479 if len(jobs) > 0:
3480 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
123f6317
LD
3481
3482 if not self.suppress_meta_comment:
3483 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3484 (','.join(self.branchPrefixes), details["change"]))
3485 if len(details['options']) > 0:
3486 self.gitStream.write(": options = %s" % details['options'])
3487 self.gitStream.write("]\n")
3488
3489 self.gitStream.write("EOT\n\n")
b984733c
SH
3490
3491 if len(parent) > 0:
4b97ffb1 3492 if self.verbose:
f2606b17 3493 print("parent %s" % parent)
b984733c
SH
3494 self.gitStream.write("from %s\n" % parent)
3495
4ae048e6 3496 self.streamP4Files(files)
b984733c
SH
3497 self.gitStream.write("\n")
3498
1f4ba1cb
SH
3499 change = int(details["change"])
3500
dba1c9d9 3501 if change in self.labels:
1f4ba1cb
SH
3502 label = self.labels[change]
3503 labelDetails = label[0]
3504 labelRevisions = label[1]
71b112d4 3505 if self.verbose:
f2606b17 3506 print("Change %s is labelled %s" % (change, labelDetails))
1f4ba1cb 3507
6de040df 3508 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 3509 for p in self.branchPrefixes])
1f4ba1cb
SH
3510
3511 if len(files) == len(labelRevisions):
3512
3513 cleanedFiles = {}
3514 for info in files:
56c09345 3515 if info["action"] in self.delete_actions:
1f4ba1cb
SH
3516 continue
3517 cleanedFiles[info["depotFile"]] = info["rev"]
3518
3519 if cleanedFiles == labelRevisions:
06804c76 3520 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
3521
3522 else:
a46668fa 3523 if not self.silent:
f2606b17 3524 print("Tag %s does not match with change %s: files do not match."
cebdf5af 3525 % (labelDetails["label"], change))
1f4ba1cb
SH
3526
3527 else:
a46668fa 3528 if not self.silent:
f2606b17 3529 print("Tag %s does not match with change %s: file count is different."
cebdf5af 3530 % (labelDetails["label"], change))
b984733c 3531
1f4ba1cb 3532 def getLabels(self):
522e914f
JH
3533 """Build a dictionary of changelists and labels, for "detect-labels"
3534 option.
3535 """
3536
1f4ba1cb
SH
3537 self.labels = {}
3538
52a4880b 3539 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 3540 if len(l) > 0 and not self.silent:
4d88519f 3541 print("Finding files belonging to labels in %s" % self.depotPaths)
01ce1fe9
SH
3542
3543 for output in l:
1f4ba1cb
SH
3544 label = output["label"]
3545 revisions = {}
3546 newestChange = 0
71b112d4 3547 if self.verbose:
f2606b17 3548 print("Querying files for label %s" % label)
6de040df
LD
3549 for file in p4CmdList(["files"] +
3550 ["%s...@%s" % (p, label)
3551 for p in self.depotPaths]):
1f4ba1cb
SH
3552 revisions[file["depotFile"]] = file["rev"]
3553 change = int(file["change"])
3554 if change > newestChange:
3555 newestChange = change
3556
9bda3a85
SH
3557 self.labels[newestChange] = [output, revisions]
3558
3559 if self.verbose:
f2606b17 3560 print("Label changes: %s" % self.labels.keys())
1f4ba1cb 3561
06804c76 3562 def importP4Labels(self, stream, p4Labels):
522e914f
JH
3563 """Import p4 labels as git tags. A direct mapping does not exist, so
3564 assume that if all the files are at the same revision then we can
3565 use that, or it's something more complicated we should just ignore.
3566 """
3567
06804c76 3568 if verbose:
f2606b17 3569 print("import p4 labels: " + ' '.join(p4Labels))
06804c76
LD
3570
3571 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 3572 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
3573 if len(validLabelRegexp) == 0:
3574 validLabelRegexp = defaultLabelRegexp
3575 m = re.compile(validLabelRegexp)
3576
3577 for name in p4Labels:
3578 commitFound = False
3579
3580 if not m.match(name):
3581 if verbose:
12a77f5b 3582 print("label %s does not match regexp %s" % (name, validLabelRegexp))
06804c76
LD
3583 continue
3584
3585 if name in ignoredP4Labels:
3586 continue
3587
3588 labelDetails = p4CmdList(['label', "-o", name])[0]
3589
3590 # get the most recent changelist for each file in this label
3591 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3592 for p in self.depotPaths])
3593
dba1c9d9 3594 if 'change' in change:
06804c76
LD
3595 # find the corresponding git commit; take the oldest commit
3596 changelist = int(change['change'])
b43702ac
LD
3597 if changelist in self.committedChanges:
3598 gitCommit = ":%d" % changelist # use a fast-import mark
06804c76 3599 commitFound = True
b43702ac
LD
3600 else:
3601 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3602 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3603 if len(gitCommit) == 0:
f2606b17 3604 print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
b43702ac
LD
3605 else:
3606 commitFound = True
3607 gitCommit = gitCommit.strip()
3608
3609 if commitFound:
06804c76
LD
3610 # Convert from p4 time format
3611 try:
3612 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3613 except ValueError:
f2606b17 3614 print("Could not convert label time %s" % labelDetails['Update'])
06804c76
LD
3615 tmwhen = 1
3616
3617 when = int(time.mktime(tmwhen))
3618 self.streamTag(stream, name, labelDetails, gitCommit, when)
3619 if verbose:
f2606b17 3620 print("p4 label %s mapped to git commit %s" % (name, gitCommit))
06804c76
LD
3621 else:
3622 if verbose:
f2606b17 3623 print("Label %s has no changelists - possibly deleted?" % name)
06804c76
LD
3624
3625 if not commitFound:
3626 # We can't import this label; don't try again as it will get very
3627 # expensive repeatedly fetching all the files for labels that will
3628 # never be imported. If the label is moved in the future, the
3629 # ignore will need to be removed manually.
3630 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3631
86dff6b6
HWN
3632 def guessProjectName(self):
3633 for p in self.depotPaths:
6e5295c4
SH
3634 if p.endswith("/"):
3635 p = p[:-1]
3636 p = p[p.strip().rfind("/") + 1:]
3637 if not p.endswith("/"):
812ee74e 3638 p += "/"
6e5295c4 3639 return p
86dff6b6 3640
4b97ffb1 3641 def getBranchMapping(self):
6555b2cc
SH
3642 lostAndFoundBranches = set()
3643
8ace74c0 3644 user = gitConfig("git-p4.branchUser")
8ace74c0 3645
8a470599
JH
3646 for info in p4CmdList(
3647 ["branches"] + (["-u", user] if len(user) > 0 else [])):
52a4880b 3648 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1 3649 viewIdx = 0
dba1c9d9 3650 while "View%s" % viewIdx in details:
4b97ffb1
SH
3651 paths = details["View%s" % viewIdx].split(" ")
3652 viewIdx = viewIdx + 1
3653 # require standard //depot/foo/... //depot/bar/... mapping
3654 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3655 continue
3656 source = paths[0]
3657 destination = paths[1]
c785e202 3658 # HACK
d53de8b9 3659 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
3660 source = source[len(self.depotPaths[0]):-4]
3661 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 3662
1a2edf4e
SH
3663 if destination in self.knownBranches:
3664 if not self.silent:
f2606b17
LD
3665 print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3666 print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
1a2edf4e
SH
3667 continue
3668
6555b2cc
SH
3669 self.knownBranches[destination] = source
3670
3671 lostAndFoundBranches.discard(destination)
3672
29bdbac1 3673 if source not in self.knownBranches:
6555b2cc
SH
3674 lostAndFoundBranches.add(source)
3675
7199cf13
VA
3676 # Perforce does not strictly require branches to be defined, so we also
3677 # check git config for a branch list.
3678 #
3679 # Example of branch definition in git config file:
3680 # [git-p4]
3681 # branchList=main:branchA
3682 # branchList=main:branchB
3683 # branchList=branchA:branchC
3684 configBranches = gitConfigList("git-p4.branchList")
3685 for branch in configBranches:
3686 if branch:
0874bb01 3687 source, destination = branch.split(":")
7199cf13
VA
3688 self.knownBranches[destination] = source
3689
3690 lostAndFoundBranches.discard(destination)
3691
3692 if source not in self.knownBranches:
3693 lostAndFoundBranches.add(source)
3694
6555b2cc
SH
3695 for branch in lostAndFoundBranches:
3696 self.knownBranches[branch] = branch
29bdbac1 3697
38f9f5ec
SH
3698 def getBranchMappingFromGitBranches(self):
3699 branches = p4BranchesInGit(self.importIntoRemotes)
3700 for branch in branches.keys():
3701 if branch == "master":
3702 branch = "main"
3703 else:
3704 branch = branch[len(self.projectName):]
3705 self.knownBranches[branch] = branch
3706
bb6e09b2
HWN
3707 def updateOptionDict(self, d):
3708 option_keys = {}
3709 if self.keepRepoPath:
3710 option_keys['keepRepoPath'] = 1
3711
3712 d["options"] = ' '.join(sorted(option_keys.keys()))
3713
3714 def readOptions(self, d):
dba1c9d9 3715 self.keepRepoPath = ('options' in d
bb6e09b2 3716 and ('keepRepoPath' in d['options']))
6326aa58 3717
8134f69c
SH
3718 def gitRefForBranch(self, branch):
3719 if branch == "main":
3720 return self.refPrefix + "master"
3721
3722 if len(branch) <= 0:
3723 return branch
3724
3725 return self.refPrefix + self.projectName + branch
3726
1ca3d710
SH
3727 def gitCommitByP4Change(self, ref, change):
3728 if self.verbose:
f2606b17 3729 print("looking in ref " + ref + " for change %s using bisect..." % change)
1ca3d710
SH
3730
3731 earliestCommit = ""
3732 latestCommit = parseRevision(ref)
3733
3734 while True:
3735 if self.verbose:
f2606b17 3736 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
8a470599
JH
3737 next = read_pipe(["git", "rev-list", "--bisect",
3738 latestCommit, earliestCommit]).strip()
1ca3d710
SH
3739 if len(next) == 0:
3740 if self.verbose:
f2606b17 3741 print("argh")
1ca3d710
SH
3742 return ""
3743 log = extractLogMessageFromGitCommit(next)
3744 settings = extractSettingsGitLog(log)
3745 currentChange = int(settings['change'])
3746 if self.verbose:
f2606b17 3747 print("current change %s" % currentChange)
1ca3d710
SH
3748
3749 if currentChange == change:
3750 if self.verbose:
f2606b17 3751 print("found %s" % next)
1ca3d710
SH
3752 return next
3753
3754 if currentChange < change:
3755 earliestCommit = "^%s" % next
3756 else:
2dda7412
AM
3757 if next == latestCommit:
3758 die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref, change))
3759 latestCommit = "%s^@" % next
1ca3d710
SH
3760
3761 return ""
3762
3763 def importNewBranch(self, branch, maxChange):
3764 # make fast-import flush all changes to disk and update the refs using the checkpoint
3765 # command so that we can try to find the branch parent in the git history
990547aa
JH
3766 self.gitStream.write("checkpoint\n\n")
3767 self.gitStream.flush()
1ca3d710
SH
3768 branchPrefix = self.depotPaths[0] + branch + "/"
3769 range = "@1,%s" % maxChange
96b2d54a 3770 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
3771 if len(changes) <= 0:
3772 return False
3773 firstChange = changes[0]
1ca3d710
SH
3774 sourceBranch = self.knownBranches[branch]
3775 sourceDepotPath = self.depotPaths[0] + sourceBranch
3776 sourceRef = self.gitRefForBranch(sourceBranch)
1ca3d710 3777
52a4880b 3778 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
3779 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3780 if len(gitParent) > 0:
3781 self.initialParents[self.gitRefForBranch(branch)] = gitParent
1ca3d710
SH
3782
3783 self.importChanges(changes)
3784 return True
3785
fed23693 3786 def searchParent(self, parent, branch, target):
6b79818b
JK
3787 targetTree = read_pipe(["git", "rev-parse",
3788 "{}^{{tree}}".format(target)]).strip()
3789 for line in read_pipe_lines(["git", "rev-list", "--format=%H %T",
c7d34884 3790 "--no-merges", parent]):
6b79818b
JK
3791 if line.startswith("commit "):
3792 continue
3793 commit, tree = line.strip().split(" ")
3794 if tree == targetTree:
fed23693 3795 if self.verbose:
6b79818b
JK
3796 print("Found parent of %s in commit %s" % (branch, commit))
3797 return commit
3798 return None
fed23693 3799
89143ac2 3800 def importChanges(self, changes, origin_revision=0):
e87f37ae
SH
3801 cnt = 1
3802 for change in changes:
89143ac2 3803 description = p4_describe(change)
e87f37ae
SH
3804 self.updateOptionDict(description)
3805
3806 if not self.silent:
0f829620
JH
3807 sys.stdout.write("\rImporting revision %s (%d%%)" % (
3808 change, (cnt * 100) // len(changes)))
e87f37ae
SH
3809 sys.stdout.flush()
3810 cnt = cnt + 1
3811
3812 try:
3813 if self.detectBranches:
3814 branches = self.splitFilesIntoBranches(description)
3815 for branch in branches.keys():
c785e202 3816 # HACK --hwn
e87f37ae 3817 branchPrefix = self.depotPaths[0] + branch + "/"
84af8b85 3818 self.branchPrefixes = [branchPrefix]
e87f37ae
SH
3819
3820 parent = ""
3821
3822 filesForCommit = branches[branch]
3823
3824 if self.verbose:
f2606b17 3825 print("branch is %s" % branch)
e87f37ae
SH
3826
3827 self.updatedBranches.add(branch)
3828
3829 if branch not in self.createdBranches:
3830 self.createdBranches.add(branch)
3831 parent = self.knownBranches[branch]
3832 if parent == branch:
3833 parent = ""
1ca3d710
SH
3834 else:
3835 fullBranch = self.projectName + branch
3836 if fullBranch not in self.p4BranchesInGit:
3837 if not self.silent:
990547aa 3838 print("\n Importing new branch %s" % fullBranch)
1ca3d710
SH
3839 if self.importNewBranch(branch, change - 1):
3840 parent = ""
3841 self.p4BranchesInGit.append(fullBranch)
3842 if not self.silent:
990547aa 3843 print("\n Resuming with change %s" % change)
1ca3d710
SH
3844
3845 if self.verbose:
f2606b17 3846 print("parent determined through known branches: %s" % parent)
e87f37ae 3847
8134f69c
SH
3848 branch = self.gitRefForBranch(branch)
3849 parent = self.gitRefForBranch(parent)
e87f37ae
SH
3850
3851 if self.verbose:
f2606b17 3852 print("looking for initial parent for %s; current parent is %s" % (branch, parent))
e87f37ae
SH
3853
3854 if len(parent) == 0 and branch in self.initialParents:
3855 parent = self.initialParents[branch]
3856 del self.initialParents[branch]
3857
fed23693
VA
3858 blob = None
3859 if len(parent) > 0:
4f9273d2 3860 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693 3861 if self.verbose:
f2606b17 3862 print("Creating temporary branch: " + tempBranch)
e63231e5 3863 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
3864 self.tempBranches.append(tempBranch)
3865 self.checkpoint()
3866 blob = self.searchParent(parent, branch, tempBranch)
3867 if blob:
e63231e5 3868 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
3869 else:
3870 if self.verbose:
f2606b17 3871 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
e63231e5 3872 self.commit(description, filesForCommit, branch, parent)
e87f37ae 3873 else:
89143ac2 3874 files = self.extractFilesFromCommit(description)
e63231e5 3875 self.commit(description, files, self.branch,
e87f37ae 3876 self.initialParent)
47497844 3877 # only needed once, to connect to the previous commit
e87f37ae
SH
3878 self.initialParent = ""
3879 except IOError:
f2606b17 3880 print(self.gitError.read())
e87f37ae
SH
3881 sys.exit(1)
3882
b9d34db9
LD
3883 def sync_origin_only(self):
3884 if self.syncWithOrigin:
3885 self.hasOrigin = originP4BranchesExist()
3886 if self.hasOrigin:
3887 if not self.silent:
f2606b17 3888 print('Syncing with origin first, using "git fetch origin"')
8a470599 3889 system(["git", "fetch", "origin"])
b9d34db9 3890
c208a243 3891 def importHeadRevision(self, revision):
f2606b17 3892 print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
c208a243 3893
4e2e6ce4
PW
3894 details = {}
3895 details["user"] = "git perforce import user"
1494fcbb 3896 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
3897 % (' '.join(self.depotPaths), revision))
3898 details["change"] = revision
3899 newestRevision = 0
3900
3901 fileCnt = 0
12a77f5b 3902 fileArgs = ["%s...%s" % (p, revision) for p in self.depotPaths]
6de040df
LD
3903
3904 for info in p4CmdList(["files"] + fileArgs):
c208a243 3905
68b28593 3906 if 'code' in info and info['code'] == 'error':
c208a243
SH
3907 sys.stderr.write("p4 returned an error: %s\n"
3908 % info['data'])
d88e707f
PW
3909 if info['data'].find("must refer to client") >= 0:
3910 sys.stderr.write("This particular p4 error is misleading.\n")
990547aa 3911 sys.stderr.write("Perhaps the depot path was misspelled.\n")
d88e707f 3912 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 3913 sys.exit(1)
68b28593
PW
3914 if 'p4ExitCode' in info:
3915 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
3916 sys.exit(1)
3917
c208a243
SH
3918 change = int(info["change"])
3919 if change > newestRevision:
3920 newestRevision = change
3921
56c09345 3922 if info["action"] in self.delete_actions:
c208a243
SH
3923 continue
3924
84af8b85 3925 for prop in ["depotFile", "rev", "action", "type"]:
c208a243
SH
3926 details["%s%s" % (prop, fileCnt)] = info[prop]
3927
3928 fileCnt = fileCnt + 1
3929
3930 details["change"] = newestRevision
4e2e6ce4 3931
9dcb9f24 3932 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 3933 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
3934 res = p4_describe(newestRevision)
3935 details["time"] = res["time"]
4e2e6ce4 3936
c208a243
SH
3937 self.updateOptionDict(details)
3938 try:
e63231e5 3939 self.commit(details, self.extractFilesFromCommit(details), self.branch)
de5abb5f 3940 except IOError as err:
f2606b17 3941 print("IO error with git fast-import. Is your git version recent enough?")
de5abb5f 3942 print("IO error details: {}".format(err))
f2606b17 3943 print(self.gitError.read())
c208a243 3944
ca5b5cce
LD
3945 def importRevisions(self, args, branch_arg_given):
3946 changes = []
3947
3948 if len(self.changesFile) > 0:
43f33e49
LD
3949 with open(self.changesFile) as f:
3950 output = f.readlines()
ca5b5cce
LD
3951 changeSet = set()
3952 for line in output:
3953 changeSet.add(int(line))
3954
3955 for change in changeSet:
3956 changes.append(change)
3957
3958 changes.sort()
3959 else:
3960 # catch "git p4 sync" with no new branches, in a repo that
3961 # does not have any existing p4 branches
3962 if len(args) == 0:
3963 if not self.p4BranchesInGit:
6026aff5 3964 raise P4CommandException("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
ca5b5cce
LD
3965
3966 # The default branch is master, unless --branch is used to
3967 # specify something else. Make sure it exists, or complain
3968 # nicely about how to use --branch.
3969 if not self.detectBranches:
3970 if not branch_exists(self.branch):
3971 if branch_arg_given:
6026aff5 3972 raise P4CommandException("Error: branch %s does not exist." % self.branch)
ca5b5cce 3973 else:
6026aff5 3974 raise P4CommandException("Error: no branch %s; perhaps specify one with --branch." %
ca5b5cce
LD
3975 self.branch)
3976
3977 if self.verbose:
3978 print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3979 self.changeRange))
3980 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3981
3982 if len(self.maxChanges) > 0:
3983 changes = changes[:min(int(self.maxChanges), len(changes))]
3984
3985 if len(changes) == 0:
3986 if not self.silent:
3987 print("No changes to import!")
3988 else:
3989 if not self.silent and not self.detectBranches:
3990 print("Import destination: %s" % self.branch)
3991
3992 self.updatedBranches = set()
3993
3994 if not self.detectBranches:
3995 if args:
3996 # start a new branch
3997 self.initialParent = ""
3998 else:
3999 # build on a previous revision
4000 self.initialParent = parseRevision(self.branch)
4001
4002 self.importChanges(changes)
4003
4004 if not self.silent:
4005 print("")
4006 if len(self.updatedBranches) > 0:
4007 sys.stdout.write("Updated branches: ")
4008 for b in self.updatedBranches:
4009 sys.stdout.write("%s " % b)
4010 sys.stdout.write("\n")
4011
123f6317
LD
4012 def openStreams(self):
4013 self.importProcess = subprocess.Popen(["git", "fast-import"],
4014 stdin=subprocess.PIPE,
4015 stdout=subprocess.PIPE,
990547aa 4016 stderr=subprocess.PIPE)
123f6317
LD
4017 self.gitOutput = self.importProcess.stdout
4018 self.gitStream = self.importProcess.stdin
4019 self.gitError = self.importProcess.stderr
c208a243 4020
86dca24b
YZ
4021 if bytes is not str:
4022 # Wrap gitStream.write() so that it can be called using `str` arguments
4023 def make_encoded_write(write):
4024 def encoded_write(s):
4025 return write(s.encode() if isinstance(s, str) else s)
4026 return encoded_write
4027
4028 self.gitStream.write = make_encoded_write(self.gitStream.write)
4029
123f6317 4030 def closeStreams(self):
837b3a63
LD
4031 if self.gitStream is None:
4032 return
123f6317
LD
4033 self.gitStream.close()
4034 if self.importProcess.wait() != 0:
4035 die("fast-import failed: %s" % self.gitError.read())
4036 self.gitOutput.close()
4037 self.gitError.close()
837b3a63 4038 self.gitStream = None
29bdbac1 4039
123f6317 4040 def run(self, args):
a028a98e
SH
4041 if self.importIntoRemotes:
4042 self.refPrefix = "refs/remotes/p4/"
4043 else:
db775559 4044 self.refPrefix = "refs/heads/p4/"
a028a98e 4045
b9d34db9 4046 self.sync_origin_only()
10f880f8 4047
5a8e84cd 4048 branch_arg_given = bool(self.branch)
569d1bd4 4049 if len(self.branch) == 0:
db775559 4050 self.branch = self.refPrefix + "master"
a028a98e 4051 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
8a470599
JH
4052 system(["git", "update-ref", self.branch, "refs/heads/p4"])
4053 system(["git", "branch", "-D", "p4"])
967f72e2 4054
a93d33ee
PW
4055 # accept either the command-line option, or the configuration variable
4056 if self.useClientSpec:
4057 # will use this after clone to set the variable
4058 self.useClientSpec_from_options = True
4059 else:
0d609032 4060 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
4061 self.useClientSpec = True
4062 if self.useClientSpec:
543987bd 4063 self.clientSpecDirs = getClientSpec()
3a70cdfa 4064
6a49f8e2
HWN
4065 # TODO: should always look at previous commits,
4066 # merge with previous imports, if possible.
4067 if args == []:
d414c74a 4068 if self.hasOrigin:
5ca44617 4069 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
4070
4071 # branches holds mapping from branch name to sha1
4072 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
4073
4074 # restrict to just this one, disabling detect-branches
4075 if branch_arg_given:
17f273ff 4076 short = shortP4Ref(self.branch, self.importIntoRemotes)
8c9e8b6e 4077 if short in branches:
84af8b85 4078 self.p4BranchesInGit = [short]
17f273ff
TK
4079 elif self.branch.startswith('refs/') and \
4080 branchExists(self.branch) and \
4081 '[git-p4:' in extractLogMessageFromGitCommit(self.branch):
af3a3205 4082 self.p4BranchesInGit = [self.branch]
8c9e8b6e
PW
4083 else:
4084 self.p4BranchesInGit = branches.keys()
abcd790f
SH
4085
4086 if len(self.p4BranchesInGit) > 1:
4087 if not self.silent:
f2606b17 4088 print("Importing from/into multiple branches")
abcd790f 4089 self.detectBranches = True
8c9e8b6e
PW
4090 for branch in branches.keys():
4091 self.initialParents[self.refPrefix + branch] = \
4092 branches[branch]
967f72e2 4093
29bdbac1 4094 if self.verbose:
f2606b17 4095 print("branches: %s" % self.p4BranchesInGit)
29bdbac1
SH
4096
4097 p4Change = 0
4098 for branch in self.p4BranchesInGit:
17f273ff
TK
4099 logMsg = extractLogMessageFromGitCommit(fullP4Ref(branch,
4100 self.importIntoRemotes))
bb6e09b2
HWN
4101
4102 settings = extractSettingsGitLog(logMsg)
29bdbac1 4103
bb6e09b2 4104 self.readOptions(settings)
7a3e83d0 4105 if 'depot-paths' in settings and 'change' in settings:
bb6e09b2 4106 change = int(settings['change']) + 1
29bdbac1
SH
4107 p4Change = max(p4Change, change)
4108
bb6e09b2
HWN
4109 depotPaths = sorted(settings['depot-paths'])
4110 if self.previousDepotPaths == []:
6326aa58 4111 self.previousDepotPaths = depotPaths
29bdbac1 4112 else:
6326aa58
HWN
4113 paths = []
4114 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
4115 prev_list = prev.split("/")
4116 cur_list = cur.split("/")
4117 for i in range(0, min(len(cur_list), len(prev_list))):
fc35c9d5 4118 if cur_list[i] != prev_list[i]:
583e1707 4119 i = i - 1
6326aa58
HWN
4120 break
4121
843d847f 4122 paths.append("/".join(cur_list[:i + 1]))
6326aa58
HWN
4123
4124 self.previousDepotPaths = paths
29bdbac1
SH
4125
4126 if p4Change > 0:
bb6e09b2 4127 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 4128 self.changeRange = "@%s,#head" % p4Change
341dc1c1 4129 if not self.silent and not self.detectBranches:
f2606b17 4130 print("Performing incremental import into %s git branch" % self.branch)
569d1bd4 4131
17f273ff 4132 self.branch = fullP4Ref(self.branch, self.importIntoRemotes)
179caebf 4133
6326aa58 4134 if len(args) == 0 and self.depotPaths:
b984733c 4135 if not self.silent:
f2606b17 4136 print("Depot paths: %s" % ' '.join(self.depotPaths))
b984733c 4137 else:
6326aa58 4138 if self.depotPaths and self.depotPaths != args:
f2606b17 4139 print("previous import used depot path %s and now %s was specified. "
843d847f
JH
4140 "This doesn't work!" % (' '.join(self.depotPaths),
4141 ' '.join(args)))
b984733c 4142 sys.exit(1)
6326aa58 4143
bb6e09b2 4144 self.depotPaths = sorted(args)
b984733c 4145
1c49fc19 4146 revision = ""
b984733c 4147 self.users = {}
b984733c 4148
58c8bc7c
PW
4149 # Make sure no revision specifiers are used when --changesfile
4150 # is specified.
4151 bad_changesfile = False
4152 if len(self.changesFile) > 0:
4153 for p in self.depotPaths:
4154 if p.find("@") >= 0 or p.find("#") >= 0:
4155 bad_changesfile = True
4156 break
4157 if bad_changesfile:
4158 die("Option --changesfile is incompatible with revision specifiers")
4159
6326aa58
HWN
4160 newPaths = []
4161 for p in self.depotPaths:
4162 if p.find("@") != -1:
4163 atIdx = p.index("@")
4164 self.changeRange = p[atIdx:]
4165 if self.changeRange == "@all":
4166 self.changeRange = ""
6a49f8e2 4167 elif ',' not in self.changeRange:
1c49fc19 4168 revision = self.changeRange
6326aa58 4169 self.changeRange = ""
7fcff9de 4170 p = p[:atIdx]
6326aa58
HWN
4171 elif p.find("#") != -1:
4172 hashIdx = p.index("#")
1c49fc19 4173 revision = p[hashIdx:]
7fcff9de 4174 p = p[:hashIdx]
6326aa58 4175 elif self.previousDepotPaths == []:
58c8bc7c
PW
4176 # pay attention to changesfile, if given, else import
4177 # the entire p4 tree at the head revision
4178 if len(self.changesFile) == 0:
4179 revision = "#head"
6326aa58 4180
843d847f 4181 p = re.sub("\.\.\.$", "", p)
6326aa58
HWN
4182 if not p.endswith("/"):
4183 p += "/"
4184
4185 newPaths.append(p)
4186
4187 self.depotPaths = newPaths
4188
e63231e5
PW
4189 # --detect-branches may change this for each branch
4190 self.branchPrefixes = self.depotPaths
4191
b607e71e 4192 self.loadUserMapFromCache()
cb53e1f8
SH
4193 self.labels = {}
4194 if self.detectLabels:
990547aa 4195 self.getLabels()
b984733c 4196
4b97ffb1 4197 if self.detectBranches:
c785e202 4198 # FIXME - what's a P4 projectName ?
df450923
SH
4199 self.projectName = self.guessProjectName()
4200
38f9f5ec
SH
4201 if self.hasOrigin:
4202 self.getBranchMappingFromGitBranches()
4203 else:
4204 self.getBranchMapping()
29bdbac1 4205 if self.verbose:
f2606b17
LD
4206 print("p4-git branches: %s" % self.p4BranchesInGit)
4207 print("initial parents: %s" % self.initialParents)
29bdbac1
SH
4208 for b in self.p4BranchesInGit:
4209 if b != "master":
6326aa58 4210
c785e202 4211 # FIXME
29bdbac1
SH
4212 b = b[len(self.projectName):]
4213 self.createdBranches.add(b)
4b97ffb1 4214
19fa5ac3
LD
4215 p4_check_access()
4216
123f6317 4217 self.openStreams()
b984733c 4218
6026aff5 4219 err = None
341dc1c1 4220
6026aff5
LD
4221 try:
4222 if revision:
4223 self.importHeadRevision(revision)
4224 else:
4225 self.importRevisions(args, branch_arg_given)
b984733c 4226
6026aff5
LD
4227 if gitConfigBool("git-p4.importLabels"):
4228 self.importLabels = True
06804c76 4229
6026aff5
LD
4230 if self.importLabels:
4231 p4Labels = getP4Labels(self.depotPaths)
4232 gitTags = getGitTags()
b984733c 4233
6026aff5
LD
4234 missingP4Labels = p4Labels - gitTags
4235 self.importP4Labels(self.gitStream, missingP4Labels)
4236
4237 except P4CommandException as e:
4238 err = e
4239
4240 finally:
4241 self.closeStreams()
4242
4243 if err:
4244 die(str(err))
b984733c 4245
fed23693
VA
4246 # Cleanup temporary branches created during import
4247 if self.tempBranches != []:
4248 for branch in self.tempBranches:
8a470599 4249 read_pipe(["git", "update-ref", "-d", branch])
fed23693
VA
4250 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
4251
55d12437
PW
4252 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
4253 # a convenient shortcut refname "p4".
4254 if self.importIntoRemotes:
4255 head_ref = self.refPrefix + "HEAD"
4256 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
4257 system(["git", "symbolic-ref", head_ref, self.branch])
4258
b984733c
SH
4259 return True
4260
adf159b4 4261
01ce1fe9
SH
4262class P4Rebase(Command):
4263 def __init__(self):
4264 Command.__init__(self)
06804c76
LD
4265 self.options = [
4266 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
06804c76 4267 ]
06804c76 4268 self.importLabels = False
cebdf5af
HWN
4269 self.description = ("Fetches the latest revision from perforce and "
4270 + "rebases the current work (branch) against it")
01ce1fe9
SH
4271
4272 def run(self, args):
4273 sync = P4Sync()
06804c76 4274 sync.importLabels = self.importLabels
01ce1fe9 4275 sync.run([])
d7e3868c 4276
14594f4b
SH
4277 return self.rebase()
4278
4279 def rebase(self):
36ee4ee4 4280 if os.system("git update-index --refresh") != 0:
990547aa 4281 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.")
8a470599 4282 if len(read_pipe(["git", "diff-index", "HEAD", "--"])) > 0:
990547aa 4283 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.")
36ee4ee4 4284
0874bb01 4285 upstream, settings = findUpstreamBranchPoint()
d7e3868c
SH
4286 if len(upstream) == 0:
4287 die("Cannot find upstream branchpoint for rebase")
4288
4289 # the branchpoint may be p4/foo~3, so strip off the parent
4290 upstream = re.sub("~[0-9]+$", "", upstream)
4291
f2606b17 4292 print("Rebasing the current branch onto %s" % upstream)
8a470599
JH
4293 oldHead = read_pipe(["git", "rev-parse", "HEAD"]).strip()
4294 system(["git", "rebase", upstream])
4295 system(["git", "diff-tree", "--stat", "--summary", "-M", oldHead,
4296 "HEAD", "--"])
01ce1fe9
SH
4297 return True
4298
adf159b4 4299
f9a3a4f7
SH
4300class P4Clone(P4Sync):
4301 def __init__(self):
4302 P4Sync.__init__(self)
4303 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2 4304 self.usage = "usage: %prog [options] //depot/path[@revRange]"
354081d5 4305 self.options += [
bb6e09b2
HWN
4306 optparse.make_option("--destination", dest="cloneDestination",
4307 action='store', default=None,
354081d5 4308 help="where to leave result of the clone"),
38200076
PW
4309 optparse.make_option("--bare", dest="cloneBare",
4310 action="store_true", default=False),
354081d5 4311 ]
bb6e09b2 4312 self.cloneDestination = None
f9a3a4f7 4313 self.needsGit = False
38200076 4314 self.cloneBare = False
f9a3a4f7 4315
6a49f8e2 4316 def defaultDestination(self, args):
c785e202 4317 # TODO: use common prefix of args?
6a49f8e2
HWN
4318 depotPath = args[0]
4319 depotDir = re.sub("(@[^@]*)$", "", depotPath)
4320 depotDir = re.sub("(#[^#]*)$", "", depotDir)
053d9e43 4321 depotDir = re.sub(r"\.\.\.$", "", depotDir)
6a49f8e2
HWN
4322 depotDir = re.sub(r"/$", "", depotDir)
4323 return os.path.split(depotDir)[1]
4324
f9a3a4f7
SH
4325 def run(self, args):
4326 if len(args) < 1:
4327 return False
bb6e09b2
HWN
4328
4329 if self.keepRepoPath and not self.cloneDestination:
4330 sys.stderr.write("Must specify destination for --keep-path\n")
4331 sys.exit(1)
f9a3a4f7 4332
6326aa58 4333 depotPaths = args
5e100b5c
SH
4334
4335 if not self.cloneDestination and len(depotPaths) > 1:
4336 self.cloneDestination = depotPaths[-1]
4337 depotPaths = depotPaths[:-1]
4338
6326aa58
HWN
4339 for p in depotPaths:
4340 if not p.startswith("//"):
0f487d30 4341 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
6326aa58 4342 return False
f9a3a4f7 4343
bb6e09b2 4344 if not self.cloneDestination:
98ad4faf 4345 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 4346
f2606b17 4347 print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
38200076 4348
c3bf3f13
KG
4349 if not os.path.exists(self.cloneDestination):
4350 os.makedirs(self.cloneDestination)
053fd0c1 4351 chdir(self.cloneDestination)
38200076 4352
84af8b85 4353 init_cmd = ["git", "init"]
38200076
PW
4354 if self.cloneBare:
4355 init_cmd.append("--bare")
a235e85c
BC
4356 retcode = subprocess.call(init_cmd)
4357 if retcode:
40e7cfdd 4358 raise subprocess.CalledProcessError(retcode, init_cmd)
38200076 4359
6326aa58 4360 if not P4Sync.run(self, depotPaths):
f9a3a4f7 4361 return False
c595956d
PW
4362
4363 # create a master branch and check out a work tree
4364 if gitBranchExists(self.branch):
84af8b85 4365 system(["git", "branch", currentGitBranch(), self.branch])
c595956d 4366 if not self.cloneBare:
84af8b85 4367 system(["git", "checkout", "-f"])
c595956d 4368 else:
968e29e1 4369 print('Not checking out any branch, use '
f2606b17 4370 '"git checkout -q -b master <branch>"')
86dff6b6 4371
a93d33ee
PW
4372 # auto-set this variable if invoked with --use-client-spec
4373 if self.useClientSpec_from_options:
8a470599 4374 system(["git", "config", "--bool", "git-p4.useclientspec", "true"])
a93d33ee 4375
f7b5ff60
TK
4376 # persist any git-p4 encoding-handling config options passed in for clone:
4377 if gitConfig('git-p4.metadataDecodingStrategy'):
4378 system(["git", "config", "git-p4.metadataDecodingStrategy", gitConfig('git-p4.metadataDecodingStrategy')])
4379 if gitConfig('git-p4.metadataFallbackEncoding'):
4380 system(["git", "config", "git-p4.metadataFallbackEncoding", gitConfig('git-p4.metadataFallbackEncoding')])
4381 if gitConfig('git-p4.pathEncoding'):
4382 system(["git", "config", "git-p4.pathEncoding", gitConfig('git-p4.pathEncoding')])
4383
f9a3a4f7
SH
4384 return True
4385
adf159b4 4386
123f6317
LD
4387class P4Unshelve(Command):
4388 def __init__(self):
4389 Command.__init__(self)
4390 self.options = []
4391 self.origin = "HEAD"
4392 self.description = "Unshelve a P4 changelist into a git commit"
4393 self.usage = "usage: %prog [options] changelist"
4394 self.options += [
4395 optparse.make_option("--origin", dest="origin",
4396 help="Use this base revision instead of the default (%s)" % self.origin),
4397 ]
4398 self.verbose = False
4399 self.noCommit = False
08813127 4400 self.destbranch = "refs/remotes/p4-unshelved"
123f6317
LD
4401
4402 def renameBranch(self, branch_name):
59ef3fc1 4403 """Rename the existing branch to branch_name.N ."""
123f6317 4404
12a77f5b 4405 for i in range(0, 1000):
123f6317
LD
4406 backup_branch_name = "{0}.{1}".format(branch_name, i)
4407 if not gitBranchExists(backup_branch_name):
4768af20
JH
4408 # Copy ref to backup
4409 gitUpdateRef(backup_branch_name, branch_name)
123f6317 4410 gitDeleteRef(branch_name)
123f6317
LD
4411 print("renamed old unshelve branch to {0}".format(backup_branch_name))
4412 break
198551ca
MB
4413 else:
4414 sys.exit("gave up trying to rename existing branch {0}".format(branch_name))
123f6317
LD
4415
4416 def findLastP4Revision(self, starting_point):
59ef3fc1
JH
4417 """Look back from starting_point for the first commit created by git-p4
4418 to find the P4 commit we are based on, and the depot-paths.
4419 """
123f6317
LD
4420
4421 for parent in (range(65535)):
0acbf599 4422 log = extractLogMessageFromGitCommit("{0}~{1}".format(starting_point, parent))
123f6317 4423 settings = extractSettingsGitLog(log)
dba1c9d9 4424 if 'change' in settings:
123f6317
LD
4425 return settings
4426
4427 sys.exit("could not find git-p4 commits in {0}".format(self.origin))
4428
89143ac2 4429 def createShelveParent(self, change, branch_name, sync, origin):
59ef3fc1
JH
4430 """Create a commit matching the parent of the shelved changelist
4431 'change'.
4432 """
89143ac2
LD
4433 parent_description = p4_describe(change, shelved=True)
4434 parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change)
4435 files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)
4436
4437 parent_files = []
4438 for f in files:
4439 # if it was added in the shelved changelist, it won't exist in the parent
4440 if f['action'] in self.add_actions:
4441 continue
4442
4443 # if it was deleted in the shelved changelist it must not be deleted
4444 # in the parent - we might even need to create it if the origin branch
4445 # does not have it
4446 if f['action'] in self.delete_actions:
4447 f['action'] = 'add'
4448
4449 parent_files.append(f)
4450
4451 sync.commit(parent_description, parent_files, branch_name,
4452 parent=origin, allow_empty=True)
4453 print("created parent commit for {0} based on {1} in {2}".format(
4454 change, self.origin, branch_name))
4455
123f6317
LD
4456 def run(self, args):
4457 if len(args) != 1:
4458 return False
4459
4460 if not gitBranchExists(self.origin):
4461 sys.exit("origin branch {0} does not exist".format(self.origin))
4462
4463 sync = P4Sync()
4464 changes = args
123f6317 4465
89143ac2 4466 # only one change at a time
123f6317
LD
4467 change = changes[0]
4468
4469 # if the target branch already exists, rename it
4470 branch_name = "{0}/{1}".format(self.destbranch, change)
4471 if gitBranchExists(branch_name):
4472 self.renameBranch(branch_name)
4473 sync.branch = branch_name
4474
4475 sync.verbose = self.verbose
4476 sync.suppress_meta_comment = True
4477
4478 settings = self.findLastP4Revision(self.origin)
123f6317
LD
4479 sync.depotPaths = settings['depot-paths']
4480 sync.branchPrefixes = sync.depotPaths
4481
4482 sync.openStreams()
4483 sync.loadUserMapFromCache()
4484 sync.silent = True
89143ac2
LD
4485
4486 # create a commit for the parent of the shelved changelist
4487 self.createShelveParent(change, branch_name, sync, self.origin)
4488
4489 # create the commit for the shelved changelist itself
4490 description = p4_describe(change, True)
4491 files = sync.extractFilesFromCommit(description, True, change)
4492
4493 sync.commit(description, files, branch_name, "")
123f6317
LD
4494 sync.closeStreams()
4495
4496 print("unshelved changelist {0} into {1}".format(change, branch_name))
4497
4498 return True
4499
adf159b4 4500
09d89de2
SH
4501class P4Branches(Command):
4502 def __init__(self):
4503 Command.__init__(self)
84af8b85 4504 self.options = []
09d89de2
SH
4505 self.description = ("Shows the git branches that hold imports and their "
4506 + "corresponding perforce depot paths")
4507 self.verbose = False
4508
4509 def run(self, args):
5ca44617
SH
4510 if originP4BranchesExist():
4511 createOrUpdateBranchesFromOrigin()
4512
8a470599 4513 for line in read_pipe_lines(["git", "rev-parse", "--symbolic", "--remotes"]):
09d89de2
SH
4514 line = line.strip()
4515
4516 if not line.startswith('p4/') or line == "p4/HEAD":
4517 continue
4518 branch = line
4519
4520 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4521 settings = extractSettingsGitLog(log)
4522
f2606b17 4523 print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
09d89de2
SH
4524 return True
4525
adf159b4 4526
b984733c
SH
4527class HelpFormatter(optparse.IndentedHelpFormatter):
4528 def __init__(self):
4529 optparse.IndentedHelpFormatter.__init__(self)
4530
4531 def format_description(self, description):
4532 if description:
4533 return description + "\n"
4534 else:
4535 return ""
4f5cf76a 4536
adf159b4 4537
86949eef 4538def printUsage(commands):
f2606b17
LD
4539 print("usage: %s <command> [options]" % sys.argv[0])
4540 print("")
4541 print("valid commands: %s" % ", ".join(commands))
4542 print("")
4543 print("Try %s <command> --help for command specific help." % sys.argv[0])
4544 print("")
86949eef 4545
adf159b4 4546
86949eef 4547commands = {
2bcf6110
JH
4548 "submit": P4Submit,
4549 "commit": P4Submit,
4550 "sync": P4Sync,
4551 "rebase": P4Rebase,
4552 "clone": P4Clone,
4553 "branches": P4Branches,
4554 "unshelve": P4Unshelve,
86949eef
SH
4555}
4556
adf159b4 4557
bb6e09b2
HWN
4558def main():
4559 if len(sys.argv[1:]) == 0:
4560 printUsage(commands.keys())
4561 sys.exit(2)
4f5cf76a 4562
bb6e09b2
HWN
4563 cmdName = sys.argv[1]
4564 try:
b86f7378
HWN
4565 klass = commands[cmdName]
4566 cmd = klass()
bb6e09b2 4567 except KeyError:
f2606b17
LD
4568 print("unknown command %s" % cmdName)
4569 print("")
bb6e09b2
HWN
4570 printUsage(commands.keys())
4571 sys.exit(2)
4572
4573 options = cmd.options
b86f7378 4574 cmd.gitdir = os.environ.get("GIT_DIR", None)
bb6e09b2
HWN
4575
4576 args = sys.argv[2:]
4577
b0ccc80d 4578 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
6a10b6aa
LD
4579 if cmd.needsGit:
4580 options.append(optparse.make_option("--git-dir", dest="gitdir"))
bb6e09b2 4581
6a10b6aa
LD
4582 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4583 options,
57fe2ce0
JH
4584 description=cmd.description,
4585 formatter=HelpFormatter())
bb6e09b2 4586
608e3805 4587 try:
0874bb01 4588 cmd, args = parser.parse_args(sys.argv[2:], cmd)
608e3805
BK
4589 except:
4590 parser.print_help()
4591 raise
4592
bb6e09b2
HWN
4593 global verbose
4594 verbose = cmd.verbose
4595 if cmd.needsGit:
da0134f6 4596 if cmd.gitdir is None:
b86f7378
HWN
4597 cmd.gitdir = os.path.abspath(".git")
4598 if not isValidGitDir(cmd.gitdir):
378f7be1 4599 # "rev-parse --git-dir" without arguments will try $PWD/.git
8a470599 4600 cmd.gitdir = read_pipe(["git", "rev-parse", "--git-dir"]).strip()
b86f7378 4601 if os.path.exists(cmd.gitdir):
8a470599 4602 cdup = read_pipe(["git", "rev-parse", "--show-cdup"]).strip()
bb6e09b2 4603 if len(cdup) > 0:
990547aa 4604 chdir(cdup)
e20a9e53 4605
b86f7378
HWN
4606 if not isValidGitDir(cmd.gitdir):
4607 if isValidGitDir(cmd.gitdir + "/.git"):
4608 cmd.gitdir += "/.git"
bb6e09b2 4609 else:
b86f7378 4610 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
e20a9e53 4611
378f7be1 4612 # so git commands invoked from the P4 workspace will succeed
b86f7378 4613 os.environ["GIT_DIR"] = cmd.gitdir
86949eef 4614
bb6e09b2
HWN
4615 if not cmd.run(args):
4616 parser.print_help()
09fca77b 4617 sys.exit(2)
4f5cf76a 4618
4f5cf76a 4619
bb6e09b2
HWN
4620if __name__ == '__main__':
4621 main()