From 403d82a0bf9e350642933ec44cd4e0ef1fdc0c28 Mon Sep 17 00:00:00 2001 From: Collin Funk Date: Sun, 31 Aug 2025 16:56:08 -0700 Subject: [PATCH] cksum: add support for SHA-3 * src/digest.c: Include sha3.h. (BLAKE2B_MAX_LEN): Rename to DIGEST_MAX_LEN since it is also used for SHA-3. (sha3_sum_stream): New function. (enum Algorithm, algorithm_args, algorithm_args, algorithm_types) algorithm_tags, algorithm_bits, cksumfns, cksum_output_fns): Add entries for SHA-3. (usage): Mention that SHA-3 is supported. Mention requirements for --length with SHA-3. (split_3): Use DIGEST_MAX_LEN instead of BLAKE2B_MAX_LEN. Determine the length of the digest for SHA-3. Make sure it is 224, 256, 384, or 512. (digest_file): Set the digest length in bytes. Use DIGEST_MAX_LEN instead of BLAKE2B_MAX_LEN. Always append the digest length to SHA3 in the output. (main): Allow the use of --length with 'cksum -a sha3'. Use DIGEST_MAX_LEN instead of BLAKE2B_MAX_LEN. Make sure it is 224, 256, 384, or 512. * tests/cksum/cksum-base64.pl (@pairs): Add expected sha3 output. (fmt): Modify the output to use SHA3-512 since that is the default. (@Tests): Modify arguments for sha3 to use --length=512. * tests/cksum/cksum-sha3.sh: New test, based on tests/cksum/b2sum.sh. * tests/local.mk (all_tests): Add the test. * bootstrap.conf: Add crypto/sha3. * gnulib: Update to latest commit. * NEWS: Mention the change. * doc/coreutils.texi (cksum general options): Mention sha3 as a supported argument to the -a option. Mention that 'cksum -a sha3' supports the --length option. Mention that SHA-3 is considered secure. --- NEWS | 4 ++ bootstrap.conf | 1 + doc/coreutils.texi | 28 ++++++-- gnulib | 2 +- src/digest.c | 136 ++++++++++++++++++++++++++++-------- tests/cksum/cksum-base64.pl | 6 +- tests/cksum/cksum-sha3.sh | 81 +++++++++++++++++++++ tests/local.mk | 1 + 8 files changed, 220 insertions(+), 39 deletions(-) create mode 100755 tests/cksum/cksum-sha3.sh diff --git a/NEWS b/NEWS index 2da3b04b9e..7f323a86cf 100644 --- a/NEWS +++ b/NEWS @@ -87,6 +87,10 @@ GNU coreutils NEWS -*- outline -*- basenc supports the --base58 option to encode and decode the visually unambiguous Base58 encoding. + 'cksum -a' now supports the 'sha3' argument, to use the SHA3-224, + SHA3-256, SHA3-384, SHA3-512 message digest algorithms depending on + the argument passed to the required --length (-l) option. + 'date' now outputs dates in the country's native calendar for the Iranian locale (fa_IR) and for the Ethiopian locale (am_ET), and also does so more consistently for the Thailand locale (th_TH.UTF-8). diff --git a/bootstrap.conf b/bootstrap.conf index 7d09752bc4..03848e9ea4 100644 --- a/bootstrap.conf +++ b/bootstrap.conf @@ -68,6 +68,7 @@ gnulib_modules=" crc-x86_64 crypto/md5 crypto/sha1 + crypto/sha3 crypto/sha256 crypto/sha512 crypto/sm3 diff --git a/doc/coreutils.texi b/doc/coreutils.texi index 3f0931e1a9..5618328552 100644 --- a/doc/coreutils.texi +++ b/doc/coreutils.texi @@ -4151,6 +4151,7 @@ Supported more modern digest algorithms are: @samp{sha256} equivalent to @command{sha256sum} @samp{sha384} equivalent to @command{sha384sum} @samp{sha512} equivalent to @command{sha512sum} +@samp{sha3} only available through @command{cksum} @samp{blake2b} equivalent to @command{b2sum} @samp{sm3} only available through @command{cksum} @end example @@ -4174,18 +4175,19 @@ input digest string as what is output. I.e., removing or adding any Output extra information to standard error, like the checksum implementation being used. -@macro cksumLengthOption @item -l @itemx --length @opindex -l @opindex --length @cindex BLAKE2 hash length -Change (shorten) the default digest length. -This is specified in bits and thus must be a multiple of 8. +@cindex SHA-3 hash length +Specify the digest size used with @option{-a sha3} or @option{-a blake2b}. +For @samp{blake2b} this is optional, with 512 being the default. If the +option is given it must be a multiple of 8. For @samp{sha3} this option +is required, and the @var{length} must be one of 224, 256, 384, or 512. + This option is ignored when @option{--check} is specified, as the length is automatically determined when checking. -@end macro -@cksumLengthOption @item --raw @opindex --raw @@ -4368,7 +4370,7 @@ against malicious tampering: although finding a file with a given \hash\ fingerprint is considered infeasible at the moment, it is known how to modify certain files, including digital certificates, so that they appear valid when signed with an \hash\ digest. For more secure hashes, -consider using SHA-2 or @command{b2sum}. +consider using SHA-2, SHA-3, or @command{b2sum}. @xref{sha2 utilities}. @xref{b2sum invocation}. @end macro @weakHash{MD5} @@ -4411,7 +4413,19 @@ The program accepts @ref{cksum common options}. Also see @ref{Common options}. In addition @command{b2sum} supports the following options. @table @samp -@cksumLengthOption + +@item -l +@itemx --length +@opindex -l +@opindex --length +@cindex BLAKE2 hash length +Specify the digest size used by the algorithm. This option is optional. +By default a 512 bit digest will be used. If the option is given it +must be a multiple of 8. + +This option is ignored when @option{--check} is specified, +as the length is automatically determined when checking. + @end table diff --git a/gnulib b/gnulib index 6fd6098f51..06e7da510b 160000 --- a/gnulib +++ b/gnulib @@ -1 +1 @@ -Subproject commit 6fd6098f510255c0b9134088b1526a50fef90edd +Subproject commit 06e7da510bd055f524c1baff0890c861bef2b6a4 diff --git a/src/digest.c b/src/digest.c index 0e4e62dee7..5acf54997a 100644 --- a/src/digest.c +++ b/src/digest.c @@ -55,6 +55,9 @@ # include "sha512.h" #endif #if HASH_ALGO_CKSUM +# include "sha3.h" +#endif +#if HASH_ALGO_CKSUM # include "sm3.h" #endif #include "fadvise.h" @@ -216,10 +219,15 @@ static bool base64_digest = false; /* If true, print binary digests, not hex. */ static bool raw_digest = false; +/* blake2b and sha3 allow the -l option. Luckily they both have the same + maximum digest size. */ #if HASH_ALGO_BLAKE2 || HASH_ALGO_CKSUM -# define BLAKE2B_MAX_LEN BLAKE2B_OUTBYTES +# if HASH_ALGO_CKSUM +static_assert (BLAKE2B_OUTBYTES == SHA3_512_DIGEST_SIZE); +# endif +# define DIGEST_MAX_LEN BLAKE2B_OUTBYTES static uintmax_t digest_length; -#endif /* HASH_ALGO_BLAKE2 */ +#endif /* HASH_ALGO_BLAKE2 || HASH_ALGO_CKSUM */ typedef void (*digest_output_fn)(char const *, int, void const *, bool, bool, unsigned char, bool, uintmax_t); @@ -279,6 +287,23 @@ sha512_sum_stream (FILE *stream, void *resstream, return sha512_stream (stream, resstream); } static int +sha3_sum_stream (FILE *stream, void *resstream, uintmax_t *length) +{ + switch (*length) + { + case SHA3_224_DIGEST_SIZE: + return sha3_224_stream (stream, resstream); + case SHA3_256_DIGEST_SIZE: + return sha3_256_stream (stream, resstream); + case SHA3_384_DIGEST_SIZE: + return sha3_384_stream (stream, resstream); + case SHA3_512_DIGEST_SIZE: + return sha3_512_stream (stream, resstream); + default: + unreachable (); + } +} +static int blake2b_sum_stream (FILE *stream, void *resstream, uintmax_t *length) { return blake2b_stream (stream, resstream, *length); @@ -301,6 +326,7 @@ enum Algorithm sha256, sha384, sha512, + sha3, blake2b, sm3, }; @@ -308,24 +334,24 @@ enum Algorithm static char const *const algorithm_args[] = { "bsd", "sysv", "crc", "crc32b", "md5", "sha1", "sha224", - "sha256", "sha384", "sha512", "blake2b", "sm3", nullptr + "sha256", "sha384", "sha512", "sha3", "blake2b", "sm3", nullptr }; static enum Algorithm const algorithm_types[] = { bsd, sysv, crc, crc32b, md5, sha1, sha224, - sha256, sha384, sha512, blake2b, sm3, + sha256, sha384, sha512, sha3, blake2b, sm3, }; ARGMATCH_VERIFY (algorithm_args, algorithm_types); static char const *const algorithm_tags[] = { "BSD", "SYSV", "CRC", "CRC32B", "MD5", "SHA1", "SHA224", - "SHA256", "SHA384", "SHA512", "BLAKE2b", "SM3", nullptr + "SHA256", "SHA384", "SHA512", "SHA3", "BLAKE2b", "SM3", nullptr }; static int const algorithm_bits[] = { 16, 16, 32, 32, 128, 160, 224, - 256, 384, 512, 512, 256, 0 + 256, 384, 512, 512, 512, 256, 0 }; static_assert (ARRAY_CARDINALITY (algorithm_bits) @@ -345,6 +371,7 @@ static sumfn cksumfns[]= sha256_sum_stream, sha384_sum_stream, sha512_sum_stream, + sha3_sum_stream, blake2b_sum_stream, sm3_sum_stream, }; @@ -362,6 +389,7 @@ static digest_output_fn cksum_output_fns[]= output_file, output_file, output_file, + output_file, }; bool cksum_debug; #endif @@ -478,8 +506,9 @@ Print or check %s (%d-bit) checksums.\n\ "), stdout); # if HASH_ALGO_BLAKE2 || HASH_ALGO_CKSUM fputs (_("\ - -l, --length=BITS digest length in bits; must not exceed the max for\n\ - the blake2 algorithm and must be a multiple of 8\n\ + -l, --length=BITS digest length in bits; must not exceed the max size\n\ + and must be a multiple of 8 for blake2b;\n\ + must be 224, 256, 384, or 512 for sha3\n\ "), stdout); # endif # if HASH_ALGO_CKSUM @@ -544,6 +573,7 @@ DIGEST determines the digest algorithm and default output format:\n\ sha256 (equivalent to sha256sum)\n\ sha384 (equivalent to sha384sum)\n\ sha512 (equivalent to sha512sum)\n\ + sha3 (only available through cksum)\n\ blake2b (equivalent to b2sum)\n\ sm3 (only available through cksum)\n\ \n"), stdout); @@ -823,7 +853,7 @@ split_3 (char *s, size_t s_len, s[--i] = '('; # if HASH_ALGO_BLAKE2 - digest_length = BLAKE2B_MAX_LEN * 8; + digest_length = DIGEST_MAX_LEN * 8; # else digest_length = algorithm_bits[cksum_algorithm]; # endif @@ -831,9 +861,19 @@ split_3 (char *s, size_t s_len, { uintmax_t length; char *siend; - if (! (xstrtoumax (s + i, &siend, 0, &length, nullptr) == LONGINT_OK - && 0 < length && length <= digest_length - && length % 8 == 0)) + if (xstrtoumax (s + i, &siend, 0, &length, nullptr) != LONGINT_OK) + return false; +# if HASH_ALGO_CKSUM + else if (cksum_algorithm == sha3) + { + if (length != SHA3_224_DIGEST_SIZE * 8 + && length != SHA3_256_DIGEST_SIZE * 8 + && length != SHA3_384_DIGEST_SIZE * 8 + && length != SHA3_512_DIGEST_SIZE * 8) + return false; + } +# endif + else if (!(0 < length && length <= digest_length && length % 8 == 0)) return false; i = siend - s; @@ -865,14 +905,21 @@ split_3 (char *s, size_t s_len, #if HASH_ALGO_BLAKE2 || HASH_ALGO_CKSUM /* Auto determine length. */ # if HASH_ALGO_CKSUM - if (cksum_algorithm == blake2b) { + if (cksum_algorithm == blake2b || cksum_algorithm == sha3) { # endif unsigned char const *hp = *digest; digest_hex_bytes = 0; while (c_isxdigit (*hp++)) digest_hex_bytes++; +# if HASH_ALGO_CKSUM + if (cksum_algorithm == sha3 && digest_hex_bytes / 2 != SHA3_224_DIGEST_SIZE + && digest_hex_bytes / 2 != SHA3_256_DIGEST_SIZE + && digest_hex_bytes / 2 != SHA3_384_DIGEST_SIZE + && digest_hex_bytes / 2 != SHA3_512_DIGEST_SIZE) + return false; +# endif if (digest_hex_bytes < 2 || digest_hex_bytes % 2 - || BLAKE2B_MAX_LEN * 2 < digest_hex_bytes) + || DIGEST_MAX_LEN * 2 < digest_hex_bytes) return false; digest_length = digest_hex_bytes * 4; # if HASH_ALGO_CKSUM @@ -1013,7 +1060,7 @@ digest_file (char const *filename, int *binary, unsigned char *bin_result, fadvise (fp, FADVISE_SEQUENTIAL); #if HASH_ALGO_CKSUM - if (cksum_algorithm == blake2b) + if (cksum_algorithm == blake2b || cksum_algorithm == sha3) *length = digest_length / 8; err = DIGEST_STREAM (fp, bin_result, length); #elif HASH_ALGO_SUM @@ -1064,12 +1111,14 @@ output_file (char const *file, int binary_file, void const *digest, { fputs (DIGEST_TYPE_STRING, stdout); # if HASH_ALGO_BLAKE2 - if (digest_length < BLAKE2B_MAX_LEN * 8) + if (digest_length < DIGEST_MAX_LEN * 8) printf ("-%ju", digest_length); # elif HASH_ALGO_CKSUM + if (cksum_algorithm == sha3) + printf ("-%ju", digest_length); if (cksum_algorithm == blake2b) { - if (digest_length < BLAKE2B_MAX_LEN * 8) + if (digest_length < DIGEST_MAX_LEN * 8) printf ("-%ju", digest_length); } # endif @@ -1478,27 +1527,54 @@ main (int argc, char **argv) min_digest_line_length = MIN_DIGEST_LINE_LENGTH; #if HASH_ALGO_BLAKE2 || HASH_ALGO_CKSUM # if HASH_ALGO_CKSUM - if (digest_length && cksum_algorithm != blake2b) + if (digest_length && (cksum_algorithm != blake2b && cksum_algorithm != sha3)) error (EXIT_FAILURE, 0, - _("--length is only supported with --algorithm=blake2b")); -# endif - if (digest_length > BLAKE2B_MAX_LEN * 8) + _("--length is only supported with --algorithm=blake2b or " + "--algorithm=sha3")); + if (cksum_algorithm == sha3) { - error (0, 0, _("invalid length: %s"), quote (digest_length_str)); - error (EXIT_FAILURE, 0, - _("maximum digest length for %s is %d bits"), - quote (DIGEST_TYPE_STRING), - BLAKE2B_MAX_LEN * 8); + /* Do not require --length with --check. */ + if (digest_length == 0 && *digest_length_str == '\0' && ! do_check) + error (EXIT_FAILURE, 0, _("--algorithm=sha3 requires specifying " + "--length 224, 256, 384, or 512")); + /* If --check and --length are used we verify the digest length. */ + if ((! do_check || *digest_length_str != '\0') + && digest_length != SHA3_224_DIGEST_SIZE * 8 + && digest_length != SHA3_256_DIGEST_SIZE * 8 + && digest_length != SHA3_384_DIGEST_SIZE * 8 + && digest_length != SHA3_512_DIGEST_SIZE * 8) + { + error (0, 0, _("invalid length: %s"), quote (digest_length_str)); + error (EXIT_FAILURE, 0, _("digest length for %s must be " + "224, 256, 384, or 512"), + quote (DIGEST_TYPE_STRING)); + } } - if (digest_length % 8 != 0) + else { - error (0, 0, _("invalid length: %s"), quote (digest_length_str)); - error (EXIT_FAILURE, 0, _("length is not a multiple of 8")); + /* If the digest length checks for SHA-3 are satisfied, the less strict + checks for BLAKE2 will also be. */ +# else + { +# endif + if (digest_length > DIGEST_MAX_LEN * 8) + { + error (0, 0, _("invalid length: %s"), quote (digest_length_str)); + error (EXIT_FAILURE, 0, + _("maximum digest length for %s is %d bits"), + quote (DIGEST_TYPE_STRING), + DIGEST_MAX_LEN * 8); + } + if (digest_length % 8 != 0) + { + error (0, 0, _("invalid length: %s"), quote (digest_length_str)); + error (EXIT_FAILURE, 0, _("length is not a multiple of 8")); + } } if (digest_length == 0) { # if HASH_ALGO_BLAKE2 - digest_length = BLAKE2B_MAX_LEN * 8; + digest_length = DIGEST_MAX_LEN * 8; # else digest_length = algorithm_bits[cksum_algorithm]; # endif diff --git a/tests/cksum/cksum-base64.pl b/tests/cksum/cksum-base64.pl index a174c0ebe5..fe629c1242 100755 --- a/tests/cksum/cksum-base64.pl +++ b/tests/cksum/cksum-base64.pl @@ -36,6 +36,7 @@ my @pairs = ['sha256', "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="], ['sha384', "OLBgp1GsljhM2TJ+sbHjaiH9txEUvgdDTAzHv2P24donTt6/529l+9Ua0vFImLlb"], ['sha512', "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="], + ['sha3', "pp9zzKI6msXItWfcGFp1bpfJghZP4lhZ4NHcwUdcgKYVshI68fX5TBHj6UAsOsVY9QAZnZW20+MBdYWGKB3NJg=="], ['blake2b', "eGoC90IBWQPGxv2FJVLScpEvR0DhWEdhiobiF/cfVBnSXhAxr+5YUxOJZESTTrBLkDpoWxRIt1XVb3Aa/pvizg=="], ['sm3', "GrIdg1XPoX+OYRlIMegajyK+yMco/vt0ftA161CCqis="], ); @@ -47,6 +48,8 @@ sub fmt ($$) { $h !~ m{^(sysv|bsd|crc|crc32b)$} and $v = uc($h). " (f) = $v"; # BLAKE2b is inconsistent: $v =~ s{BLAKE2B}{BLAKE2b}; + # Our tests use 'cksum -a sha3 --length=512'. + $v =~ s/^SHA3\b/SHA3-512/; return "$v" } @@ -54,7 +57,8 @@ my @Tests = ( # Ensure that each of the above works with --base64: (map {my ($h,$v)= @$_; my $o=fmt $h,$v; - [$h, "--base64 -a $h", {IN=>{f=>''}}, {OUT=>"$o\n"}]} @pairs), + (my $opts = $h) =~ s/^sha3$/sha3 --length=512/; + [$h, "--base64 -a $opts", {IN=>{f=>''}}, {OUT=>"$o\n"}]} @pairs), # For each that accepts --check, ensure that works with base64 digests: (map {my ($h,$v)= @$_; my $o=fmt $h,$v; diff --git a/tests/cksum/cksum-sha3.sh b/tests/cksum/cksum-sha3.sh new file mode 100755 index 0000000000..4c414f9371 --- /dev/null +++ b/tests/cksum/cksum-sha3.sh @@ -0,0 +1,81 @@ +#!/bin/sh +# 'cksum -a sha3' tests. + +# Copyright (C) 2025 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src +print_ver_ cksum +getlimits_ + +# Ensure we can --check the --tag format we produce +for i in 'a' ' b' '*c' '44' ' '; do + echo "$i" > "$i" + for l in 224 256 384 512; do + cksum -a sha3 -l $l "$i" >> check.sha3 + done +done +# Note -l is inferred from the tags in the mixed format file +cksum -a sha3 --strict -c check.sha3 || fail=1 + +# Also ensure the openssl tagged variant works +sed 's/ //; s/ =/=/' < check.sha3 > openssl.sha3 || framework_failure_ +cksum -a sha3 --strict -c openssl.sha3 || fail=1 + +# Ensure we can check non tagged format +for l in 224 256 384 512; do + cksum -a sha3 --untagged --text -l $l /dev/null \ + | tee -a check.vals > check.sha3 + cksum -a sha3 -l $l --strict -c check.sha3 || fail=1 + cksum -a sha3 --strict -c check.sha3 || fail=1 +done + +# Ensure the checksum values are correct. The reference +# check.vals was created using OpenSSL. +cksum -a sha3 --length=256 check.vals > out.tmp || fail=1 +tr '*' ' ' < out.tmp > out || framework_failure_ # Remove binary tag on cygwin +printf '%s' 'SHA3-256 (check.vals) = ' > exp +echo 'b4753bf1696fda712821b665494c89090ffb0e87b8645559ad9f5db25b42d4f3' >> exp +compare exp out || fail=1 + +# Make sure --check does not handle unsupported digest sizes, e.g. truncated +# digests. +printf '%s' 'SHA3-248 (check.vals) = ' > inp +echo 'b4753bf1696fda712821b665494c89090ffb0e87b8645559ad9f5db25b42d4' >> inp +returns_ 1 cksum -a sha3 -c --warn inp 2>err || fail=1 +cat < exp || framework_failure_ +cksum: inp: 1: improperly formatted SHA3 checksum line +cksum: inp: no properly formatted checksum lines found +EOF +compare exp err || fail=1 + +# Only validate the last specified, used length +cksum -a sha3 -l 253 -l 256 /dev/null || fail=1 + +# SHA-3 only allows values for --length to be 224, 256, 384, and 512. +# Check that multiples of 8 that are allowed by BLAKE2 are disallowed. +for len in 216 248 376 504 513 1024 $UINTMAX_OFLOW; do + returns_ 1 cksum -a sha3 -l $len /dev/null 2>err || fail=1 + cat < exp || framework_failure_ +cksum: invalid length: '$len' +cksum: digest length for 'SHA3' must be 224, 256, 384, or 512 +EOF + compare exp err || fail=1 + # We still check --length when --check is used. + returns_ 1 cksum -a sha3 -l $len --check /dev/null 2>err || fail=1 + compare exp err +done + +Exit $fail diff --git a/tests/local.mk b/tests/local.mk index bcb01dd8ed..a42a20fbeb 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -380,6 +380,7 @@ all_tests = \ tests/cksum/sha256sum.pl \ tests/cksum/sha384sum.pl \ tests/cksum/sha512sum.pl \ + tests/cksum/cksum-sha3.sh \ tests/shred/shred-exact.sh \ tests/shred/shred-passes.sh \ tests/shred/shred-remove.sh \ -- 2.47.3