diff --git a/checksum.c b/checksum.c index 4c91c2b2b..5c387dd66 100644 --- a/checksum.c +++ b/checksum.c @@ -59,6 +59,9 @@ struct name_num_item valid_checksums_items[] = { { CSUM_MD4, NNI_BUILTIN|NNI_EVP, "md4", NULL }, #ifdef SHA_DIGEST_LENGTH { CSUM_SHA1, NNI_EVP, "sha1", NULL }, +#endif +#ifdef SHA256_DIGEST_LENGTH + { CSUM_SHA256, NNI_EVP, "sha256", NULL }, #endif { CSUM_NONE, 0, "none", NULL }, { 0, 0, NULL, NULL } diff --git a/rsync.1.md b/rsync.1.md index 40ea438c1..f584962cf 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -2007,6 +2007,7 @@ sign) if you want the local shell to expand it. - `md5` - `md4` - `sha1` + - `sha256` - `none` Run `rsync --version` to see the default checksum list compiled into your diff --git a/testsuite/checksum-sha256-absent_test.py b/testsuite/checksum-sha256-absent_test.py new file mode 100755 index 000000000..0d029036d --- /dev/null +++ b/testsuite/checksum-sha256-absent_test.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""The compiled algorithm list must stay honest about sha256. + +sha256 is available only when the build's OpenSSL provides it: both the +transfer list (valid_checksums_items[]) and the daemon auth list +(valid_auth_checksums_items[]) wrap their sha256 entry in +`#ifdef SHA256_DIGEST_LENGTH`. A build without OpenSSL SHA-256 -- rsync +configured --disable-openssl, or a platform whose CI installs no openssl at +all (the FreeBSD and Solaris jobs) -- must therefore not offer sha256 +anywhere, and must refuse a request for it instead of advertising something +it cannot compute. + +This test never skips: it asserts the contract that holds on *either* build +shape, so the sha256-absent branch is genuinely exercised wherever such a +build runs, and the sha256-present branch elsewhere. + + * the two lists agree -- sha256 is in both or in neither, since one #ifdef + governs both; + * where sha256 is advertised it must actually work; + * where it is not, asking for it must fail closed with a clear error, and + ordinary negotiation must still succeed; + * the rejection path itself works, checked with a name no build has. + +checksum-sha256 covers what a working sha256 transfer must get right. +""" + +import json +import re + +from rsyncfns import ( + FROMDIR, TODIR, assert_same, make_data_file, makepath, rmtree, run_rsync, + test_fail, +) + +vv = json.loads(run_rsync('-VV', check=True, capture_output=True).stdout) +checksums = vv.get('checksum_list', []) +auth = vv.get('daemon_auth_list', []) + +in_transfer = 'sha256' in checksums +in_auth = 'sha256' in auth + +# --- 1. one #ifdef governs both tables, so they cannot disagree ------------- + +if in_auth and not in_transfer: + test_fail( + "this build's OpenSSL provides SHA-256 -- daemon_auth_list offers " + "sha256 -- but the transfer checksum list does not, so the guarded " + "sha256 entry is missing from valid_checksums_items[]; --version " + f"reports checksum_list={checksums} daemon_auth_list={auth}") +if in_transfer and not in_auth: + test_fail( + "the transfer checksum list offers sha256 while daemon_auth_list does " + "not; one #ifdef SHA256_DIGEST_LENGTH governs both entries, so they " + "cannot legitimately disagree; --version reports " + f"checksum_list={checksums} daemon_auth_list={auth}") + +src, dst = FROMDIR, TODIR +rmtree(src) +makepath(src) +make_data_file(src / 'payload.bin', 40000) + + +def copy(*extra: str): + rmtree(dst) + return run_rsync('-a', '--debug=NSTR', *extra, f'{src}/', f'{dst}/', + check=False, capture_output=True) + + +# --- 2. the rejection path works at all ------------------------------------- +# Checked on every build with a name no rsync can ever have, so the assertion +# the sha256-absent branch relies on is itself proven here. + +proc = copy('--checksum-choice=nosuchalgo') +if proc.returncode == 0: + test_fail('--checksum-choice=nosuchalgo was accepted; an unknown ' + 'algorithm name must be refused') +if 'unknown checksum name' not in proc.stderr: + test_fail('--checksum-choice with an unknown name should report "unknown ' + f'checksum name", got: {proc.stderr!r}') + +# --- 3. whichever way this build was compiled, it must be consistent -------- + +if in_transfer: + proc = copy('--checksum', '--checksum-choice=sha256') + if proc.returncode != 0: + test_fail("sha256 is advertised in checksum_list but a transfer " + f"asking for it failed: rc={proc.returncode} {proc.stderr}") + if not re.search(r'checksum: sha256\b', proc.stdout): + test_fail("sha256 is advertised but --checksum-choice=sha256 did not " + f"select it: {proc.stdout!r}") + assert_same(src / 'payload.bin', dst / 'payload.bin', 'advertised sha256') + verdict = ('sha256 advertised in both lists and usable') +else: + proc = copy('--checksum', '--checksum-choice=sha256') + if proc.returncode == 0: + test_fail('this build does not advertise sha256, yet ' + '--checksum-choice=sha256 succeeded; the ' + '#ifdef SHA256_DIGEST_LENGTH guard is not holding') + if 'unknown checksum name' not in proc.stderr: + test_fail('a build without OpenSSL SHA-256 must refuse ' + '--checksum-choice=sha256 with "unknown checksum name", ' + f'got: rc={proc.returncode} {proc.stderr!r}') + + # Losing sha256 must not cost the build ordinary negotiation. + proc = copy() + if proc.returncode != 0: + test_fail("a build without sha256 could not complete an ordinary " + f"negotiated transfer: rc={proc.returncode} {proc.stderr}") + assert_same(src / 'payload.bin', dst / 'payload.bin', 'no-sha256 build') + verdict = ('sha256 absent from both lists, refused when asked for, ' + 'negotiation still fine') + +print(f"checksum-sha256-absent: {verdict}; unknown names rejected") diff --git a/testsuite/checksum-sha256-negotiate-old_test.py b/testsuite/checksum-sha256-negotiate-old_test.py new file mode 100755 index 000000000..39414b658 --- /dev/null +++ b/testsuite/checksum-sha256-negotiate-old_test.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Adding sha256 to the checksum list must not disturb an older peer. + +valid_checksums_items[] is what both sides advertise when they negotiate a +transfer checksum, so appending sha256 to it changes the wire conversation +with every peer -- including ones built before sha256 was offered, and ones +whose OpenSSL never provided it. The failure this guards against is a new +entry making negotiation pick something the far side cannot compute, or +shifting the agreed algorithm for peers that were previously fine. + +Three properties, against the in-tree old_versions binaries: + + * a negotiation-capable peer without sha256 (3.2.7, 3.4.1) still agrees on a + mutually supported algorithm, and it is not sha256; + * a pre-negotiation peer (3.1.3) still falls back to md5 and copies + correctly; + * asking for sha256 explicitly against a peer that lacks it fails closed -- + a clear "unknown checksum name" error and a non-zero exit, not a hang, a + silent downgrade, or a corrupt destination. + +The old_versions binaries are static Linux builds. Where they cannot run +(macOS, Cygwin, the BSDs) the portable control still runs and the old-peer +cases report that they were not exercised, rather than skipping the test -- +those platforms enforce testsuite/skiplist and a skip there would be a CI +failure, not a note. +""" + +import json +import os +import re +import subprocess + +from rsyncfns import ( + FROMDIR, SRCDIR, TODIR, assert_same, make_data_file, makepath, rmtree, + rsh_cmd, rsync_path_arg, run_rsync, test_fail, test_skipped, +) + +vv = json.loads(run_rsync('-VV', check=True, capture_output=True).stdout) +if 'sha256' not in vv.get('checksum_list', []): + test_skipped("sha256 not in this build's checksum list (no OpenSSL SHA-256)") + +os.environ['RSYNC_RSH'] = rsh_cmd() + +src, dst = FROMDIR, TODIR +rmtree(src) +makepath(src) +make_data_file(src / 'payload.bin', 80000) +(src / 'note.txt').write_text('negotiated against an older peer\n') + + +def transfer(peer_path: str, *extra: str): + """Copy src -> a fresh dst with `peer_path` as the far-side rsync.""" + rmtree(dst) + return run_rsync('-a', '--debug=NSTR', f'--rsync-path={peer_path}', *extra, + f'lh:{src}/', f'{dst}/', check=False, capture_output=True) + + +def agreed_checksum(proc): + """The algorithm the run reported settling on, or None.""" + m = re.search(r'checksum: (\S+)', proc.stdout) + return m.group(1) if m else None + + +def check_copy(label: str) -> 'None': + assert_same(src / 'payload.bin', dst / 'payload.bin', label) + assert_same(src / 'note.txt', dst / 'note.txt', label) + + +# --- portable control: both sides are this build ---------------------------- + +proc = transfer(rsync_path_arg()) +if proc.returncode != 0: + test_fail(f"control: same-build peer transfer failed: {proc.stderr}") +check_copy('control (same-build peer)') +if agreed_checksum(proc) is None: + test_fail(f"control: no checksum reported by --debug=NSTR: {proc.stdout!r}") + +proc = transfer(rsync_path_arg(), '--checksum', '--checksum-choice=sha256') +if proc.returncode != 0: + test_fail("control: --checksum-choice=sha256 failed between two builds " + f"that both advertise it: {proc.stderr}") +if agreed_checksum(proc) != 'sha256': + test_fail("control: --checksum-choice=sha256 did not select sha256 " + f"between two sha256-capable builds: {proc.stdout!r}") +check_copy('control (forced sha256, same-build peer)') + +# --- the old peers ---------------------------------------------------------- + +NNI_PEERS = ('rsync_3.4.1', 'rsync_3.2.7') # negotiate, but no sha256 +PRE_NNI_PEERS = ('rsync_3.1.3',) # older than checksum negotiation + + +def usable(name: str): + """Path to an old_versions binary that runs here, else None.""" + path = SRCDIR / 'old_versions' / name + if not path.is_file() or not os.access(path, os.X_OK): + return None + want = name.replace('rsync_', 'version ') + try: + probe = subprocess.run([str(path), '--version'], stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, timeout=30) + except (OSError, subprocess.TimeoutExpired): + return None + return path if probe.returncode == 0 and want in probe.stdout else None + + +exercised = [] + +for name in NNI_PEERS: + peer = usable(name) + if peer is None: + continue + peer_algos = set(json.loads( + subprocess.run([str(peer), '-VV'], stdout=subprocess.PIPE, text=True, + check=True).stdout).get('checksum_list', [])) + if 'sha256' in peer_algos: + test_fail(f"test bug: {name} advertises sha256, so it cannot stand in " + "for a peer that lacks it") + + # 1. auto negotiation still lands on something both sides have. + proc = transfer(rsync_path_arg(str(peer))) + if proc.returncode != 0: + test_fail(f"{name}: adding sha256 broke plain negotiation with an " + f"older peer: rc={proc.returncode} {proc.stderr}") + algo = agreed_checksum(proc) + if algo == 'sha256': + test_fail(f"{name}: negotiation selected sha256 against a peer whose " + f"checksum list is {sorted(peer_algos)}") + if algo not in peer_algos: + test_fail(f"{name}: negotiation selected {algo!r}, which the peer does " + f"not advertise ({sorted(peer_algos)})") + check_copy(f'{name} negotiated {algo}') + + # 2. demanding sha256 from a peer without it must fail closed. + proc = transfer(rsync_path_arg(str(peer)), '--checksum', + '--checksum-choice=sha256') + if proc.returncode == 0: + test_fail(f"{name}: --checksum-choice=sha256 unexpectedly succeeded " + "against a peer that does not support sha256") + if 'unknown checksum name' not in proc.stderr: + test_fail(f"{name}: --checksum-choice=sha256 against a peer without it " + "should fail with 'unknown checksum name', got: " + f"rc={proc.returncode} {proc.stderr!r}") + exercised.append(f'{name} (negotiated {algo}, forced sha256 refused)') + +for name in PRE_NNI_PEERS: + peer = usable(name) + if peer is None: + continue + proc = transfer(rsync_path_arg(str(peer))) + if proc.returncode != 0: + test_fail(f"{name}: adding sha256 broke the pre-negotiation md5 " + f"fallback: rc={proc.returncode} {proc.stderr}") + algo = agreed_checksum(proc) + if algo != 'md5': + test_fail(f"{name}: a pre-negotiation peer should still use md5, " + f"got {algo!r}") + check_copy(f'{name} md5 fallback') + exercised.append(f'{name} (md5 fallback)') + +if not exercised: + print("checksum-sha256-negotiate-old: same-build control verified " + "(negotiation works and forced sha256 selects sha256); no " + "old_versions binary runs on this platform, so the older-peer " + "cases were not exercised") + raise SystemExit(0) + +print("checksum-sha256-negotiate-old: sha256 in the checksum list leaves " + "older peers undisturbed -- " + '; '.join(exercised)) diff --git a/testsuite/checksum-sha256_test.py b/testsuite/checksum-sha256_test.py new file mode 100755 index 000000000..8039f492c --- /dev/null +++ b/testsuite/checksum-sha256_test.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Focused coverage for sha256 as the transfer checksum. + +sha256 is in valid_checksums_items[], so --checksum-choice=sha256 (and +automatic negotiation) can select it on a build whose OpenSSL provides +SHA-256. compress-options already walks *every* advertised algorithm for the +basic "was it selected, did the copy land" property, so this test covers what +is specific to sha256: its digest is 32 bytes, i.e. *longer* than SUM_LENGTH +(16), the legacy MD4/MD5 size the block-checksum paths were written around. + +append-shortsum guards the short end of that range (xxh64, 8 bytes, which used +to make the sender die on an over-stated s2length). This guards the long end, +with the delta algorithm actually engaged (--no-whole-file; a local transfer +defaults to --whole-file and would never compute a block checksum at all) and +on the --append-verify redo path, plus the stealth-change detection that -c +exists for. + +Skipped on a build whose OpenSSL lacks SHA-256; checksum-sha256-absent covers +that build shape. +""" + +import json +import os +import re + +from rsyncfns import ( + FROMDIR, TODIR, assert_same, make_data_file, makepath, rmtree, run_rsync, + test_fail, test_skipped, +) + +vv = json.loads(run_rsync('-VV', check=True, capture_output=True).stdout) +if 'sha256' not in vv.get('checksum_list', []): + test_skipped("sha256 not in this build's checksum list (no OpenSSL SHA-256)") + +CHOICE = '--checksum-choice=sha256' +src, dst = FROMDIR, TODIR + + +def selected_sha256(proc, what: str) -> 'None': + """Fail unless the run reported sha256 as the checksum it settled on.""" + if not re.search(r'checksum: sha256\b', proc.stdout): + test_fail(f"{what}: sha256 was not the selected checksum; " + f"--debug=NSTR said: {proc.stdout!r}") + + +# --- 1. whole-file -c transfer at depth ------------------------------------- + +rmtree(src) +rmtree(dst) +makepath(src / 'a' / 'b' / 'c') +make_data_file(src / 'a' / 'b' / 'c' / 'deep.bin', 120000) +(src / 'a' / 'b' / 'top.txt').write_text('sha256 at depth\n') + +proc = run_rsync('-a', '-c', CHOICE, '--debug=NSTR', + f'{src}/', f'{dst}/', capture_output=True) +selected_sha256(proc, '-c transfer') +assert_same(src / 'a' / 'b' / 'c' / 'deep.bin', dst / 'a' / 'b' / 'c' / 'deep.bin', + 'sha256 -c transfer at depth') +assert_same(src / 'a' / 'b' / 'top.txt', dst / 'a' / 'b' / 'top.txt', + 'sha256 -c transfer at depth') + +# --- 2. stealth change: same size, same mtime, different bytes --------------- +# -c must re-send on the digest alone. Without a working sha256 comparison the +# quick check would call these identical and the file would silently stay stale. + +stealth_src = src / 'a' / 'b' / 'c' / 'deep.bin' +stealth_dst = dst / 'a' / 'b' / 'c' / 'deep.bin' +before = stealth_src.stat() +data = bytearray(stealth_src.read_bytes()) +data[len(data) // 2] ^= 0xff # one flipped bit, size unchanged +stealth_src.write_bytes(bytes(data)) +os.utime(stealth_src, (before.st_atime, before.st_mtime)) # mtime unchanged too + +if stealth_src.stat().st_size != before.st_size: + test_fail('test bug: the stealth edit changed the file size') + +proc = run_rsync('-a', '-c', CHOICE, '--debug=NSTR', + f'{src}/', f'{dst}/', capture_output=True) +selected_sha256(proc, 'stealth-change -c transfer') +assert_same(stealth_src, stealth_dst, + 'sha256 -c did not re-send a same-size same-mtime edit') + +# --- 3. delta transfer with the block checksum actually engaged ------------- +# --no-whole-file makes the receiver generate block checksums and the sender +# match against them, so the 32-byte digest goes through get_checksum2() and +# the s2length plumbing rather than being skipped by a whole-file copy. + +rmtree(src) +rmtree(dst) +makepath(src, dst) +make_data_file(src / 'delta.bin', 400000) +run_rsync('-a', f'{src}/delta.bin', f'{dst}/delta.bin') + +data = bytearray((src / 'delta.bin').read_bytes()) +data[150000:150100] = bytes(100) # rewrite a middle chunk, same total size +(src / 'delta.bin').write_bytes(bytes(data)) + +proc = run_rsync('-a', '-c', '--no-whole-file', CHOICE, '--debug=NSTR', + f'{src}/delta.bin', f'{dst}/delta.bin', capture_output=True) +selected_sha256(proc, 'delta transfer') +assert_same(src / 'delta.bin', dst / 'delta.bin', + 'sha256 delta transfer (--no-whole-file)') + +# --- 4. --append-verify redo with a >16-byte digest ------------------------- +# The dest is a *corrupted* prefix, so --append-verify's check of the existing +# bytes fails and the file is redone with a full checksum -- the same path +# append-shortsum drives with an 8-byte digest, here with a 32-byte one. + +make_data_file(src / 'append.bin', 60000) +full = (src / 'append.bin').read_bytes() +bad_prefix = bytearray(full[:30000]) +bad_prefix[10000] ^= 0xff +(dst / 'append.bin').write_bytes(bytes(bad_prefix)) + +proc = run_rsync('-a', '--append-verify', '--no-whole-file', CHOICE, + '--debug=NSTR', f'{src}/append.bin', f'{dst}/append.bin', + capture_output=True) +selected_sha256(proc, '--append-verify redo') +assert_same(src / 'append.bin', dst / 'append.bin', + 'sha256 --append-verify redo of a corrupted prefix') + +print("checksum-sha256: sha256 selected and correct for -c at depth, a " + "stealth same-size same-mtime edit, a --no-whole-file delta transfer " + "and an --append-verify redo")