--- /dev/null
+#!/usr/bin/env python3
+"""Property-level coverage of --link-dest / --copy-dest / --compare-dest at
+depth and across directory boundaries.
+
+The existing alt-dest_test.py is a faithful port that checks tree equality;
+this companion asserts the *distinguishing* property of each option, at every
+level of a >=3-deep tree, with the alternate-destination tree placed OUTSIDE
+both the source and destination trees (a sibling, not a parent/child):
+
+ --link-dest unchanged files are HARD-LINKED to the reference (same inode);
+ a changed file is transferred fresh (not linked).
+ --copy-dest unchanged files are COPIED from the reference (never linked);
+ dest is complete and byte-identical to the source.
+ --compare-dest unchanged files are NEITHER transferred NOR created in dest;
+ only a changed/new file lands in dest.
+
+These options drive the two-dirfd / outside-tree path handling the resolver
+restructure rewrites, so they must be guarded at depth.
+"""
+
+import os
+
+from rsyncfns import (
+ FROMDIR, SCRATCHDIR, TODIR,
+ assert_exists, assert_hardlinked, assert_not_exists, assert_not_hardlinked,
+ assert_same, make_tree, rmtree, run_rsync, walk_files,
+)
+
+src = FROMDIR
+ref = SCRATCHDIR / 'altref' # sibling of from/ and to/ -- outside both trees
+
+rmtree(src)
+rmtree(ref)
+rmtree(TODIR)
+
+# A >=3-deep source: f0 at the root, then d1/f1, d1/d2/f2, d1/d2/d3/f3.
+make_tree(src, depth=3, data=True)
+
+# Reference tree == an exact copy of the source, so every file is "identical"
+# for the alt-dest comparison.
+run_rsync('-a', f'{src}/', f'{ref}/')
+
+# Now make the deepest file differ so it must be transferred in every mode.
+changed = os.path.join('d1', 'd2', 'd3', 'f3')
+with open(src / changed, 'ab') as f:
+ f.write(b'a changed deep tail\n')
+
+rels = [p.relative_to(src) for p in walk_files(src)]
+assert os.path.join('d1', 'd2', 'd3', 'f3') in [str(r) for r in rels]
+
+
+def run_to(opt):
+ rmtree(TODIR)
+ run_rsync('-a', f'--{opt}={ref}', f'{src}/', f'{TODIR}/')
+
+
+# --- --link-dest: unchanged -> hardlink to ref; changed -> fresh copy -------
+run_to('link-dest')
+for rel in rels:
+ d, r = TODIR / rel, ref / rel
+ if str(rel) == changed:
+ assert_not_hardlinked(d, r, label=f'link-dest changed {rel}')
+ assert_same(d, src / rel, label=f'link-dest changed {rel}')
+ else:
+ assert_hardlinked(d, r, label=f'link-dest unchanged {rel}')
+
+# --- --copy-dest: every file copied (never linked), dest complete -----------
+run_to('copy-dest')
+for rel in rels:
+ d, r = TODIR / rel, ref / rel
+ assert_exists(d, label=f'copy-dest {rel}')
+ assert_same(d, src / rel, label=f'copy-dest {rel}')
+ assert_not_hardlinked(d, r, label=f'copy-dest {rel}')
+
+# --- --compare-dest: unchanged absent from dest; only the changed file lands -
+run_to('compare-dest')
+for rel in rels:
+ d = TODIR / rel
+ if str(rel) == changed:
+ assert_exists(d, label=f'compare-dest changed {rel}')
+ assert_same(d, src / rel, label=f'compare-dest changed {rel}')
+ else:
+ assert_not_exists(d, label=f'compare-dest unchanged {rel}')
+
+print("alt-dest-deep: link-dest/copy-dest/compare-dest verified at depth")
--- /dev/null
+#!/usr/bin/env python3
+"""Coverage of --append and --append-verify at depth.
+
+--append assumes each destination file is a prefix of the (longer) source and
+transfers only the bytes past the existing size; it does NOT re-check the data
+already present, and it never touches a file that is already the same size or
+larger. --append-verify works the same way but folds the existing data into the
+whole-file checksum, so a transfer whose result fails verification is re-sent
+with a normal --inplace pass. Exercise both on files >=3 levels deep.
+"""
+
+import os
+
+from rsyncfns import (
+ FROMDIR, TODIR,
+ assert_same, forced_protocol, make_tree, rmtree, run_rsync, test_fail,
+ walk_files,
+)
+
+src = FROMDIR
+deep = os.path.join('d1', 'd2', 'd3', 'f3')
+
+
+def seed_source():
+ rmtree(src)
+ make_tree(src, depth=3, data=True, data_size=8192)
+ return [p.relative_to(src) for p in walk_files(src)]
+
+
+def dest_prefix(rels, *, corrupt=False, frac=0.5):
+ """Build a destination holding the first `frac` of each source file (a
+ valid prefix), optionally corrupting the deep file's leading bytes."""
+ rmtree(TODIR)
+ for rel in rels:
+ dst = TODIR / rel
+ dst.parent.mkdir(parents=True, exist_ok=True)
+ full = (src / rel).read_bytes()
+ dst.write_bytes(full[: int(len(full) * frac)])
+ if corrupt:
+ p = TODIR / deep
+ bad = bytearray(p.read_bytes())
+ bad[0:64] = b'\x00' * 64
+ p.write_bytes(bytes(bad))
+
+
+# --- --append completes truncated destinations at every level ---------------
+rels = seed_source()
+dest_prefix(rels)
+run_rsync('-a', '--append', f'{src}/', f'{TODIR}/')
+for rel in rels:
+ assert_same(TODIR / rel, src / rel, label=f'append {rel}')
+
+# The split between non-verifying --append and verifying --append-verify only
+# exists at protocol >= 30; at protocol 29 plain --append still verifies, so
+# skip the distinguishing sub-cases there.
+proto = forced_protocol()
+if proto is not None and proto < 30:
+ print(f"append: protocol {proto} -- skipping the --append/--append-verify "
+ "split (verifying-append behaviour predates the protocol-30 split)")
+else:
+ # plain --append trusts a corrupted prefix (leaves it wrong)
+ dest_prefix(rels, corrupt=True)
+ run_rsync('-a', '--append', f'{src}/', f'{TODIR}/')
+ if (TODIR / deep).read_bytes() == (src / deep).read_bytes():
+ test_fail("plain --append unexpectedly repaired a corrupted prefix "
+ "(it should append only and trust the existing data)")
+
+ # --append-verify detects the bad prefix and re-sends the whole file
+ dest_prefix(rels, corrupt=True)
+ run_rsync('-a', '--append-verify', f'{src}/', f'{TODIR}/')
+ assert_same(TODIR / deep, src / deep, label='append-verify deep')
+
+print("append: tail-only completion at depth; append-verify repairs prefix")
--- /dev/null
+#!/usr/bin/env python3
+"""Property-level coverage of --backup / --suffix / --backup-dir at depth.
+
+backup_test.py is the ported regression test (2 levels, no custom suffix); this
+companion checks the concrete outcome of each backup mode at >=3 levels and,
+for --backup-dir, with the backup tree placed OUTSIDE the destination (a
+sibling) so the old file is renamed across directories into a parallel deep
+path -- the cross-directory operation the resolver restructure rewrites.
+
+Asserts, at every level of the tree:
+ * --backup --suffix=S saves the overwritten file as <name>S beside the new
+ one (old content in the backup, new content in place);
+ * --backup --backup-dir=DIR relocates the old file to DIR/<relpath>,
+ preserving the deep structure, while the destination gets the new content;
+ * --backup-dir together with --delete routes a deletion into the backup
+ tree instead of losing it.
+"""
+
+import os
+
+from rsyncfns import (
+ FROMDIR, SCRATCHDIR, TODIR,
+ assert_not_exists, assert_same, make_tree, rmtree, run_rsync, test_fail,
+ walk_files,
+)
+
+src = FROMDIR
+bak = SCRATCHDIR / 'backups' # sibling of from/ and to/ -- outside both trees
+
+
+def seed():
+ """Fresh v1 source, a matching destination, and the old (v1) bytes; then
+ mutate the source to v2 so the next sync overwrites every file."""
+ rmtree(src)
+ rmtree(TODIR)
+ rmtree(bak)
+ make_tree(src, depth=3, data=True, data_size=4096)
+ rels = [p.relative_to(src) for p in walk_files(src)]
+ run_rsync('-a', f'{src}/', f'{TODIR}/')
+ old = {rel: (src / rel).read_bytes() for rel in rels}
+ for rel in rels: # v1 -> v2
+ with open(src / rel, 'ab') as f:
+ f.write(b'\nversion-2 tail\n')
+ return rels, old
+
+
+# --- --backup --suffix=.old (same directory) --------------------------------
+rels, old = seed()
+run_rsync('-a', '-b', '--suffix=.old', '--no-whole-file',
+ f'{src}/', f'{TODIR}/')
+for rel in rels:
+ assert_same(TODIR / rel, src / rel, label=f'suffix new {rel}')
+ backup = (TODIR / rel)
+ backup = backup.with_name(backup.name + '.old')
+ if not backup.is_file():
+ test_fail(f"--suffix backup missing for {rel}: {backup}")
+ if backup.read_bytes() != old[rel]:
+ test_fail(f"--suffix backup of {rel} does not hold the old content")
+
+# --- --backup-dir at depth, outside the dest tree (cross-dir) ---------------
+rels, old = seed()
+run_rsync('-a', '-b', f'--backup-dir={bak}', '--no-whole-file',
+ f'{src}/', f'{TODIR}/')
+for rel in rels:
+ assert_same(TODIR / rel, src / rel, label=f'backup-dir new {rel}')
+ saved = bak / rel
+ if not saved.is_file():
+ test_fail(f"--backup-dir did not preserve deep path for {rel}: {saved}")
+ if saved.read_bytes() != old[rel]:
+ test_fail(f"--backup-dir copy of {rel} does not hold the old content")
+
+# --- --backup-dir captures a deletion under --delete ------------------------
+rels, old = seed()
+# Add a deep file to the destination that is absent from the source.
+extra = os.path.join('d1', 'd2', 'd3', 'goner')
+(TODIR / extra).write_bytes(b'about to be deleted\n')
+run_rsync('-a', '--delete', '-b', f'--backup-dir={bak}', '--no-whole-file',
+ f'{src}/', f'{TODIR}/')
+assert_not_exists(TODIR / extra, label='deleted file removed from dest')
+saved = bak / extra
+if not saved.is_file():
+ test_fail(f"--backup-dir did not capture the deletion of {extra}")
+if saved.read_bytes() != b'about to be deleted\n':
+ test_fail("captured deletion has the wrong content")
+
+print("backup-deep: suffix / backup-dir / delete-capture verified at depth")
--- /dev/null
+#!/usr/bin/env python3
+"""Property-level coverage of --delay-updates at depth.
+
+--delay-updates writes each updated file into a per-directory staging dir
+(.~tmp~) during the transfer and only renames them into place at the very end,
+so an interrupted run never leaves a half-written file visible. The staging dir
+sits inside each destination directory, so the staging write and the
+end-of-run rename are parent-directory operations the resolver restructure
+rewrites; the ported delay-updates_test.py only exercises the tree root, so
+this companion drives a >=3-deep tree.
+
+Asserts, at every level of the tree:
+ * a --delay-updates transfer reproduces the source and leaves no .~tmp~
+ staging directory behind;
+ * a stale file pre-planted in a deep .~tmp~ staging dir is overwritten
+ cleanly and the staging dir is removed.
+"""
+
+import os
+
+from rsyncfns import (
+ FROMDIR, TODIR,
+ assert_same, make_tree, rmtree, run_rsync, test_fail, walk_dirs,
+ walk_files,
+)
+
+src = FROMDIR
+deepdir = os.path.join('d1', 'd2', 'd3')
+
+
+def no_staging_left():
+ leftover = [p for p in walk_dirs(TODIR) if p.name == '.~tmp~']
+ if leftover:
+ test_fail(f"--delay-updates left staging dirs behind: {leftover}")
+
+
+# --- initial --delay-updates over a deep tree -------------------------------
+rmtree(src)
+rmtree(TODIR)
+make_tree(src, depth=3, data=True, data_size=4096)
+rels = [p.relative_to(src) for p in walk_files(src)]
+
+run_rsync('-a', '--delay-updates', f'{src}/', f'{TODIR}/')
+for rel in rels:
+ assert_same(TODIR / rel, src / rel, label=f'delay-updates initial {rel}')
+no_staging_left()
+
+# --- update every file, with a stale staging file planted deep --------------
+for rel in rels:
+ with open(src / rel, 'ab') as f:
+ f.write(b'\nupdated content\n')
+
+stage = TODIR / deepdir / '.~tmp~'
+stage.mkdir(parents=True, exist_ok=True)
+(stage / 'f3').write_bytes(b'stale staged junk\n') # must be overwritten
+
+run_rsync('-a', '--delay-updates', f'{src}/', f'{TODIR}/')
+for rel in rels:
+ assert_same(TODIR / rel, src / rel, label=f'delay-updates update {rel}')
+no_staging_left()
+
+print("delay-updates-deep: staging + clean overwrite verified at depth")
--- /dev/null
+#!/usr/bin/env python3
+"""Coverage of --inplace at depth.
+
+--inplace updates the destination file directly instead of writing a temp copy
+and renaming it over the original, so across a delta update the destination
+keeps the SAME inode. Without --inplace the receiver creates a fresh temp file
+and renames it, giving the destination a NEW inode. Both behaviours hinge on
+how the receiver resolves the destination directory and (for the default mode)
+performs the temp->final rename, which the path restructure rewrites; verify
+them on a file >=3 levels deep.
+"""
+
+import os
+
+from rsyncfns import (
+ FROMDIR, TODIR,
+ assert_same, make_tree, rmtree, run_rsync, test_fail,
+)
+
+src = FROMDIR
+deep = os.path.join('d1', 'd2', 'd3', 'f3')
+
+
+def seed():
+ rmtree(src)
+ rmtree(TODIR)
+ make_tree(src, depth=3, data=True, data_size=200000)
+
+
+def inode(path):
+ return os.stat(path).st_ino
+
+
+def modify_deep():
+ # Rewrite a chunk in the middle so it is a genuine delta, not just a tail
+ # append. Bump mtime by a clear margin (the whole test runs inside one
+ # second, so a "now" touch would collide with the destination's mtime and
+ # the size-unchanged file would be skipped by the quick check).
+ p = src / deep
+ data = bytearray(p.read_bytes())
+ data[1000:1100] = bytes((b ^ 0xFF) for b in data[1000:1100])
+ p.write_bytes(bytes(data))
+ st = os.stat(p)
+ os.utime(p, (st.st_atime, st.st_mtime + 100))
+
+
+# --- --inplace keeps the destination inode across a delta update ------------
+seed()
+run_rsync('-a', f'{src}/', f'{TODIR}/')
+ino_before = inode(TODIR / deep)
+
+modify_deep()
+run_rsync('-a', '--inplace', '--no-whole-file', f'{src}/', f'{TODIR}/')
+assert_same(TODIR / deep, src / deep, label='inplace content')
+if inode(TODIR / deep) != ino_before:
+ test_fail("--inplace changed the destination inode at depth "
+ f"({ino_before} -> {inode(TODIR / deep)})")
+
+# --- control: the default (temp+rename) path replaces the inode -------------
+seed()
+run_rsync('-a', f'{src}/', f'{TODIR}/')
+ino_before = inode(TODIR / deep)
+
+modify_deep()
+run_rsync('-a', '--no-whole-file', f'{src}/', f'{TODIR}/')
+assert_same(TODIR / deep, src / deep, label='default content')
+if inode(TODIR / deep) == ino_before:
+ test_fail("default (non-inplace) delta update unexpectedly kept the "
+ "destination inode at depth -- temp+rename did not run")
+
+print("inplace: same-inode update at depth verified; default replaces inode")
--- /dev/null
+#!/usr/bin/env python3
+"""Coverage of --partial / --partial-dir at depth and across directory
+boundaries.
+
+--partial keeps a partially transferred file so a later run can resume it.
+--partial-dir=DIR keeps the partial in DIR instead of the destination file:
+a RELATIVE dir is created in (and removed from) each destination file's own
+directory; an ABSOLUTE dir is a reserved location that holds partials by
+basename. All of this is parent- and cross-directory path resolution -- what
+the resolver restructure rewrites -- so exercise it on a file several levels
+deep, with the absolute partial-dir kept OUTSIDE the destination tree.
+
+Note: a *delta* resume from an absolute partial-dir currently fails whole-file
+verification on master (it re-puts the partial and never converges). This test
+therefore only asserts the cross-directory WRITE of the partial for that case
+and completes it with --whole-file, which is the clearly-correct baseline.
+"""
+
+import os
+import signal
+import subprocess
+import time
+
+from rsyncfns import (
+ FROMDIR, SCRATCHDIR, TODIR,
+ assert_same, make_data_file, makepath, rmtree, rsync_argv, run_rsync,
+ test_fail,
+)
+
+src = FROMDIR
+deepdir = os.path.join('d1', 'd2', 'd3')
+deep = os.path.join(deepdir, 'f3')
+FULL = 12_000_000
+
+
+def seed_big():
+ rmtree(src)
+ rmtree(TODIR)
+ makepath(src / deepdir)
+ make_data_file(src / deep, FULL)
+
+
+def is_prefix(partial) -> bool:
+ pb = partial.read_bytes()
+ return 0 < len(pb) < FULL and (src / deep).read_bytes()[:len(pb)] == pb
+
+
+def interrupt_transfer(extra_args, partial_path):
+ """Start a deliberately slow transfer, SIGTERM it once the receiver's
+ in-progress temp (.f3.XXXXXX) has some data, and wait for `partial_path`
+ (where this mode keeps the partial) to materialise.
+
+ The bandwidth limit is low so the multi-second transfer cannot finish
+ before we interrupt it -- important under a loaded parallel run (-j16),
+ where the polling loop can lag by seconds. We then poll for the partial,
+ since rsync moves it into place from its exit_cleanup handler."""
+ proc = subprocess.Popen(
+ rsync_argv('-a', '--no-whole-file', '--bwlimit=400', *extra_args,
+ f'{src}/', f'{TODIR}/'),
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+ tdir = TODIR / deepdir
+ caught = False
+ deadline = time.monotonic() + 30
+ while time.monotonic() < deadline:
+ if proc.poll() is not None:
+ break # exited before we caught it
+ if tdir.is_dir():
+ temps = [p for p in tdir.glob('.f3.*')
+ if p.is_file() and p.stat().st_size > 0]
+ if temps:
+ caught = True
+ break
+ time.sleep(0.02)
+ proc.send_signal(signal.SIGTERM)
+ proc.wait()
+ if not caught:
+ test_fail("never caught an in-progress temp (transfer finished too "
+ "fast to interrupt)")
+ # rsync moves the partial into place from exit_cleanup; give it a moment.
+ pdeadline = time.monotonic() + 5
+ while time.monotonic() < pdeadline:
+ if partial_path.is_file() and partial_path.stat().st_size > 0:
+ return
+ time.sleep(0.05)
+
+
+# --- 1. --partial (no dir): partial kept in the dest file itself, at depth --
+seed_big()
+interrupt_transfer(['--partial'], TODIR / deep)
+if not (TODIR / deep).is_file() or not is_prefix(TODIR / deep):
+ test_fail("--partial did not leave a valid partial in the dest file")
+run_rsync('-a', '--partial', '--no-whole-file', f'{src}/', f'{TODIR}/')
+assert_same(TODIR / deep, src / deep, label='--partial resume')
+
+# --- 2. relative --partial-dir at depth: deterministic clean pre-seed -------
+rmtree(src)
+rmtree(TODIR)
+makepath(src / deepdir, TODIR / deepdir / '.rsync-partial')
+make_data_file(src / deep, 1_000_000)
+full = (src / deep).read_bytes()
+(TODIR / deepdir / '.rsync-partial' / 'f3').write_bytes(full[:400_000])
+run_rsync('-a', '--partial-dir=.rsync-partial', '--no-whole-file',
+ f'{src}/', f'{TODIR}/')
+assert_same(TODIR / deep, src / deep, label='rel partial-dir preseed')
+if (TODIR / deepdir / '.rsync-partial').exists():
+ test_fail("relative --partial-dir not removed after the partial was used")
+
+# --- 3. relative --partial-dir at depth: interrupt then resume -------------
+seed_big()
+part = TODIR / deepdir / '.rsync-partial' / 'f3'
+interrupt_transfer(['--partial-dir=.rsync-partial'], part)
+if not part.is_file() or not is_prefix(part):
+ test_fail("relative --partial-dir did not keep a valid partial at depth")
+run_rsync('-a', '--partial-dir=.rsync-partial', '--no-whole-file',
+ f'{src}/', f'{TODIR}/')
+assert_same(TODIR / deep, src / deep, label='rel partial-dir resume')
+
+# --- 4. absolute --partial-dir OUTSIDE the tree (cross-dir): interrupt -----
+ext = SCRATCHDIR / 'partials' # sibling of from/ and to/ -- outside both
+rmtree(ext)
+ext.mkdir()
+seed_big()
+interrupt_transfer([f'--partial-dir={ext}'], ext / 'f3')
+if not (ext / 'f3').is_file() or not is_prefix(ext / 'f3'):
+ test_fail("absolute --partial-dir did not write the partial to the "
+ "outside-tree dir")
+run_rsync('-a', f'--partial-dir={ext}', '--whole-file', f'{src}/', f'{TODIR}/')
+assert_same(TODIR / deep, src / deep, label='abs partial-dir resume')
+
+print("partial: --partial + relative/absolute --partial-dir verified at depth")
--- /dev/null
+#!/usr/bin/env python3
+"""Coverage of --temp-dir (-T) at depth and across directory boundaries.
+
+--temp-dir tells the receiver to create its scratch/temp copies in DIR rather
+than in the destination directory, then rename the finished file into place.
+That rename crosses from the temp directory to a destination directory several
+levels deep -- exactly the two-directory operation the path-resolution
+restructure rewrites -- so it must be guarded at depth with the temp dir kept
+OUTSIDE the destination tree.
+
+Asserts:
+ * a transfer with --temp-dir pointing outside the dest tree reproduces the
+ source byte-for-byte at every level;
+ * no temp/scratch files are left behind in the temp dir or the dest tree;
+ * a non-existent --temp-dir makes the receiver fail (so we know the option
+ is actually consulted, not silently ignored).
+"""
+
+import os
+
+from rsyncfns import (
+ FROMDIR, SCRATCHDIR, TODIR,
+ assert_same, make_tree, rmtree, run_rsync, test_fail, walk_files,
+)
+
+src = FROMDIR
+tmp = SCRATCHDIR / 'scratch-temp' # sibling of from/ and to/ -- outside both
+rmtree(src)
+rmtree(TODIR)
+rmtree(tmp)
+tmp.mkdir()
+
+make_tree(src, depth=3, data=True)
+rels = [p.relative_to(src) for p in walk_files(src)]
+
+# Transfer with the temp dir outside the destination tree.
+run_rsync('-a', f'--temp-dir={tmp}', f'{src}/', f'{TODIR}/')
+
+for rel in rels:
+ assert_same(TODIR / rel, src / rel, label=f'temp-dir {rel}')
+
+# The temp dir must be clean afterwards (every scratch file renamed away).
+leftover = sorted(p for p in tmp.rglob('*'))
+if leftover:
+ test_fail(f"--temp-dir left scratch files behind: {leftover}")
+
+# No stray rsync temp files (.name.XXXXXX) anywhere in the dest tree.
+strays = [p for p in TODIR.rglob('.*') if p.is_file()]
+if strays:
+ test_fail(f"dest tree contains stray temp files: {strays}")
+
+# Negative: a missing temp dir must cause a receiver failure, proving the
+# option is honoured rather than ignored.
+rmtree(TODIR)
+proc = run_rsync('-a', f'--temp-dir={SCRATCHDIR}/does-not-exist',
+ f'{src}/', f'{TODIR}/', check=False)
+if proc.returncode == 0:
+ test_fail("--temp-dir pointing at a missing directory unexpectedly "
+ "succeeded")
+
+print("temp-dir: cross-dir rename at depth verified; missing temp dir fails")
projects / thirdparty / rsync.git / commitdiff
