--- /dev/null
+/*
+ * compare files content
+ *
+ * The goal is to minimize number of data we need to read from the files and be
+ * ready to compare large set of files, it means reuse the previous data if
+ * possible. It never read entire file if not necessary.
+ *
+ * The another goal is to minimize number of open files (imagine "hardlink /"),
+ * the code can open only two files and reopen the file next time if
+ * necessary.
+ *
+ * This code supports multiple comparison methods. The very basic step which is
+ * generic for all methods is to read and compare an "intro" (a few bytes from
+ * the begging of the file). This intro buffer is always cached in 'struct
+ * ul_fileeq_data', this intro buffer is addresses as block=0. This primitive
+ * thing can reduce a lot ...
+ *
+ * The next steps depend on selected method:
+ *
+ * * memcmp method: always read data to userspace, nothing is cached, compare
+ * directly files content; fast for small sets of the small files.
+ *
+ * * Linux crypto API: zero-copy method based on sendfile(), data blocks are
+ * send to the kernel hash functions (sha1, ...), and only hash digest is read
+ * and cached in usersapce. Fast for large set of (large) files.
+ *
+ *
+ * No copyright is claimed. This code is in the public domain; do with
+ * it what you wish.
+ *
+ * Written by Karel Zak <kzak@redhat.com> [October 2021]
+ */
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+/* Linux crypto */
+#ifdef HAVE_LINUX_IF_ALG_H
+# include <sys/socket.h>
+# include <linux/if_alg.h>
+# include <sys/param.h>
+# include <sys/sendfile.h>
+# define USE_HARDLINK_CRYPTOAPI 1
+#endif
+
+#include "c.h"
+#include "all-io.h"
+#include "fileeq.h"
+#include "debug.h"
+
+static UL_DEBUG_DEFINE_MASK(ulfileeq);
+UL_DEBUG_DEFINE_MASKNAMES(ulfileeq) = UL_DEBUG_EMPTY_MASKNAMES;
+
+#define ULFILEEQ_DEBUG_INIT (1 << 1)
+#define ULFILEEQ_DEBUG_CRYPTO (1 << 2)
+#define ULFILEEQ_DEBUG_DATA (1 << 3)
+#define ULFILEEQ_DEBUG_EQ (1 << 4)
+
+#define DBG(m, x) __UL_DBG(ulfileeq, ULFILEEQ_DEBUG_, m, x)
+#define ON_DBG(m, x) __UL_DBG_CALL(ulfileeq, ULFILEEQ_DEBUG_, m, x)
+
+#define UL_DEBUG_CURRENT_MASK UL_DEBUG_MASK(ulfileeq)
+#include "debugobj.h"
+
+static void ul_fileeq_init_debug(void)
+{
+ if (ulfileeq_debug_mask)
+ return;
+ __UL_INIT_DEBUG_FROM_ENV(ulfileeq, ULFILEEQ_DEBUG_, 0, ULFILEEQ_DEBUG);
+}
+
+enum {
+ UL_FILEEQ_MEMCMP,
+ UL_FILEEQ_SHA1,
+ UL_FILEEQ_SHA256,
+ UL_FILEEQ_CRC32
+};
+
+struct ul_fileeq_method {
+ int id;
+ const char *name; /* name used by applications */
+ const char *kname; /* name used by kernel crypto */
+ short digsiz;
+};
+
+static const struct ul_fileeq_method ul_eq_methods[] = {
+ [UL_FILEEQ_MEMCMP] = {
+ .id = UL_FILEEQ_MEMCMP, .name = "memcmp"
+ },
+#ifdef USE_HARDLINK_CRYPTOAPI
+ [UL_FILEEQ_SHA1] = {
+ .id = UL_FILEEQ_SHA1, .name = "sha1",
+ .digsiz = 20, .kname = "sha1"
+ },
+ [UL_FILEEQ_SHA256] = {
+ .id = UL_FILEEQ_SHA256, .name = "sha256",
+ .digsiz = 32, .kname = "sha256"
+ },
+
+ [UL_FILEEQ_CRC32] = {
+ .id = UL_FILEEQ_CRC32, .name = "crc32",
+ .digsiz = 4, .kname = "crc32c"
+ }
+#endif
+};
+
+#ifdef USE_HARDLINK_CRYPTOAPI
+static void deinit_crypto_api(struct ul_fileeq *eq)
+{
+ if (!eq)
+ return;
+
+ DBG(CRYPTO, ul_debugobj(eq, "deinit"));
+
+ if (eq->fd_cip >= 0)
+ close(eq->fd_cip);
+ if (eq->fd_api >= 0)
+ close(eq->fd_api);
+
+ eq->fd_cip = eq->fd_api = -1;
+
+}
+
+static int init_crypto_api(struct ul_fileeq *eq)
+{
+ struct sockaddr_alg sa = {
+ .salg_family = AF_ALG,
+ .salg_type = "hash",
+ };
+
+ assert(eq->method);
+ assert(eq->method->kname);
+ assert(eq->fd_api == -1);
+ assert(eq->fd_cip == -1);
+
+ DBG(CRYPTO, ul_debugobj(eq, "init [%s]", eq->method->kname));
+
+ assert(sizeof(sa.salg_name) > strlen(eq->method->kname) + 1);
+ memcpy(&sa.salg_name, eq->method->kname, strlen(eq->method->kname) + 1);
+
+ if ((eq->fd_api = socket(AF_ALG, SOCK_SEQPACKET, 0)) < 0)
+ goto fail;
+ if (bind(eq->fd_api, (struct sockaddr *) &sa, sizeof(sa)) != 0)
+ goto fail;
+ if ((eq->fd_cip = accept(eq->fd_api, NULL, 0)) < 0)
+ goto fail;
+ return 0;
+fail:
+ deinit_crypto_api(eq);
+ return -1;
+}
+#endif
+
+int ul_fileeq_init(struct ul_fileeq *eq, const char *method)
+{
+ size_t i;
+
+ ul_fileeq_init_debug();
+ DBG(EQ, ul_debugobj(eq, "init [%s]", method));
+
+ memset(eq, 0, sizeof(*eq));
+ eq->fd_api = -1;
+ eq->fd_cip = -1;
+
+ for (i = 0; i < ARRAY_SIZE(ul_eq_methods); i++) {
+ const struct ul_fileeq_method *m = &ul_eq_methods[i];
+
+ if (strcmp(m->name, method) == 0) {
+ eq->method = m;
+ break;
+ }
+ }
+
+ if (!eq->method)
+ return -1;
+#ifdef USE_HARDLINK_CRYPTOAPI
+ if (eq->method->id != UL_FILEEQ_MEMCMP
+ && init_crypto_api(eq) != 0)
+ return -1;
+#endif
+ return 0;
+}
+
+void ul_fileeq_deinit(struct ul_fileeq *eq)
+{
+ if (!eq)
+ return;
+
+ DBG(EQ, ul_debugobj(eq, "deinit"));
+#ifdef USE_HARDLINK_CRYPTOAPI
+ deinit_crypto_api(eq);
+#endif
+ free(eq->buf_a);
+ free(eq->buf_b);
+}
+
+void ul_fileeq_data_close_file(struct ul_fileeq_data *data)
+{
+ assert(data);
+
+ if (data->fd >= 0) {
+ DBG(DATA, ul_debugobj(data, "close"));
+ close(data->fd);
+ }
+ data->fd = -1;
+}
+
+void ul_fileeq_data_init(struct ul_fileeq_data *data)
+{
+ DBG(DATA, ul_debugobj(data, "init"));
+ memset(data, 0, sizeof(*data));
+ data->fd = -1;
+}
+
+void ul_fileeq_data_deinit(struct ul_fileeq_data *data)
+{
+ assert(data);
+
+ DBG(DATA, ul_debugobj(data, "deinit"));
+ free(data->blocks);
+ data->blocks = NULL;
+ data->nblocks = 0;
+ data->maxblocks = 0;
+ data->is_eof = 0;
+ data->name = NULL;
+
+ ul_fileeq_data_close_file(data);
+}
+
+int ul_fileeq_data_associated(struct ul_fileeq_data *data)
+{
+ return data->name != NULL;
+}
+
+void ul_fileeq_data_set_file(struct ul_fileeq_data *data, const char *name)
+{
+ assert(data);
+ assert(name);
+
+ DBG(DATA, ul_debugobj(data, "set file: %s", name));
+ ul_fileeq_data_init(data);
+ data->name = name;
+}
+
+size_t ul_fileeq_set_size(struct ul_fileeq *eq, uint64_t filesiz,
+ size_t readsiz, size_t memsiz)
+{
+ uint64_t nreads, maxdigs;
+ size_t digsiz;
+
+ assert(eq);
+
+ eq->filesiz = filesiz;
+
+ switch (eq->method->id) {
+ case UL_FILEEQ_MEMCMP:
+ /* align file size */
+ filesiz = (filesiz + readsiz) / readsiz * readsiz;
+ break;
+ default:
+ digsiz = eq->method->digsiz;
+ if (readsiz < digsiz)
+ readsiz = digsiz;
+ /* align file size */
+ filesiz = (filesiz + readsiz) / readsiz * readsiz;
+ /* calculate limits */
+ maxdigs = memsiz / digsiz;
+ nreads = filesiz / readsiz;
+ /* enlarge readsize for large files */
+ if (nreads > maxdigs)
+ readsiz = filesiz / maxdigs;
+ break;
+ }
+
+ eq->readsiz = readsiz;
+ eq->blocksmax = filesiz / readsiz;
+
+ DBG(EQ, ul_debugobj(eq, "set sizes: filesiz=%ju, maxblocks=%zu, readsiz=%zu",
+ eq->filesiz, eq->blocksmax, eq->readsiz));
+ return eq->blocksmax;
+}
+
+static unsigned char *get_buffer(struct ul_fileeq *eq)
+{
+ if (!eq->buf_a)
+ eq->buf_a = malloc(eq->readsiz);
+ if (!eq->buf_b)
+ eq->buf_b = malloc(eq->readsiz);
+
+ if (!eq->buf_a || !eq->buf_b)
+ return NULL;
+
+ if (eq->buf_last == eq->buf_b)
+ eq->buf_last = eq->buf_a;
+ else
+ eq->buf_last = eq->buf_b;
+
+ return eq->buf_last;
+}
+
+#define get_cached_nblocks(_d) \
+ ((_d)->nblocks ? (_d)->nblocks - 1 : 0)
+
+#define get_cached_offset(_e, _d) \
+ ((_d)->nblocks == 0 ? 0 : \
+ sizeof((_d)->intro) \
+ + (get_cached_nblocks(_d) * (_e)->readsiz))
+
+
+static int get_fd(struct ul_fileeq *eq, struct ul_fileeq_data *data, off_t *off)
+{
+ off_t o = get_cached_offset(eq, data);
+
+ assert(eq);
+ assert(data);
+
+
+ if (data->fd < 0) {
+ DBG(DATA, ul_debugobj(data, "open: %s", data->name));
+ data->fd = open(data->name, O_RDONLY);
+ if (data->fd < 0)
+ return data->fd;
+
+#if defined(POSIX_FADV_SEQUENTIAL) && defined(HAVE_POSIX_FADVISE)
+ ignore_result( posix_fadvise(data->fd, o, 0, POSIX_FADV_SEQUENTIAL) );
+#endif
+ if (o) {
+ DBG(DATA, ul_debugobj(data, "lseek off=%ju", (uintmax_t) o));
+ lseek(data->fd, o, SEEK_SET);
+ }
+ }
+
+ if (off)
+ *off = o;
+
+ return data->fd;
+}
+
+static void memcmp_reset(struct ul_fileeq *eq, struct ul_fileeq_data *data)
+{
+ /* only intro[] is cached */
+ if (data->nblocks)
+ data->nblocks = 1;
+ /* reset file possition */
+ if (data->fd >= 0)
+ lseek(data->fd, get_cached_offset(eq, data), SEEK_SET);
+ data->is_eof = 0;
+}
+
+static ssize_t read_block(struct ul_fileeq *eq, struct ul_fileeq_data *data,
+ size_t n, unsigned char **block)
+{
+ int fd;
+ off_t off = 0;
+ ssize_t rsz;
+
+ if (data->is_eof || n > eq->blocksmax)
+ return 0;
+
+ fd = get_fd(eq, data, &off);
+ if (fd < 0)
+ return fd;
+
+ DBG(DATA, ul_debugobj(data, " read block off=%ju", (uintmax_t) off));
+
+ *block = get_buffer(eq);
+ if (!*block)
+ return -ENOMEM;
+
+ rsz = read_all(data->fd, (char *) *block, eq->readsiz);
+ if (rsz < 0) {
+ DBG(DATA, ul_debugobj(data, " read failed"));
+ return rsz;
+ }
+ off += rsz;
+ data->nblocks++;
+
+ if (rsz == 0 || (uint64_t) off >= eq->filesiz) {
+ data->is_eof = 1;
+ ul_fileeq_data_close_file(data);
+ }
+
+ DBG(DATA, ul_debugobj(data, " read sz=%zu", rsz));
+ return rsz;
+}
+
+#ifdef USE_HARDLINK_CRYPTOAPI
+static ssize_t get_digest(struct ul_fileeq *eq, struct ul_fileeq_data *data,
+ size_t n, unsigned char **block)
+{
+ off_t off = 0;
+ ssize_t rsz;
+ size_t sz;
+ int fd;
+
+ if (n > eq->blocksmax)
+ return 0;
+
+ /* return already cached if alvalable */
+ if (n < get_cached_nblocks(data)) {
+ DBG(DATA, ul_debugobj(data, " digest cached"));
+ assert(data->blocks);
+ *block = data->blocks + (n * eq->method->digsiz);
+ return eq->method->digsiz;
+ }
+
+ if (data->is_eof) {
+ DBG(DATA, ul_debugobj(data, " file EOF"));
+ return 0;
+ }
+
+ /* read new block */
+ fd = get_fd(eq, data, &off);
+ if (fd < 0)
+ return fd;
+
+ DBG(DATA, ul_debugobj(data, " read digest off=%ju", (uintmax_t) off));
+
+ sz = eq->method->digsiz;
+
+ if (!data->blocks) {
+ DBG(DATA, ul_debugobj(data, " alloc cache %zu", eq->blocksmax * sz));
+ data->blocks = malloc(eq->blocksmax * sz);
+ if (!data->blocks)
+ return -ENOMEM;
+ }
+
+ assert(n <= eq->blocksmax);
+
+ rsz = sendfile(eq->fd_cip, data->fd, NULL, eq->readsiz);
+ DBG(DATA, ul_debugobj(data, " sent %zu [%zu wanted] to cipher", rsz, eq->readsiz));
+
+ if (rsz < 0)
+ return rsz;
+
+ off += rsz;
+
+ /* get block digest (note 1st block is data->intro */
+ *block = data->blocks + (n * eq->method->digsiz);
+ rsz = read_all(eq->fd_cip, (char *) *block, sz);
+
+ if (rsz > 0)
+ data->nblocks++;
+ if (rsz == 0 || (uint64_t) off >= eq->filesiz) {
+ data->is_eof = 1;
+ ul_fileeq_data_close_file(data);
+ }
+ DBG(DATA, ul_debugobj(data, " get %zuB digest", rsz));
+ return rsz;
+}
+#endif
+
+static ssize_t get_intro(struct ul_fileeq *eq, struct ul_fileeq_data *data,
+ unsigned char **block)
+{
+ if (data->nblocks == 0) {
+ int fd = get_fd(eq, data, NULL);
+ ssize_t rsz;
+
+ if (fd < 0)
+ return -1;
+ rsz = read_all(fd, (char *) data->intro, sizeof(data->intro));
+ DBG(DATA, ul_debugobj(data, " read %zu bytes intro", sizeof(data->intro)));
+ if (rsz <= 0)
+ return -1;
+ data->nblocks = 1;
+ }
+
+ DBG(DATA, ul_debugobj(data, " return intro"));
+ *block = data->intro;
+ return sizeof(data->intro);
+}
+
+static ssize_t get_cmp_data(struct ul_fileeq *eq, struct ul_fileeq_data *data,
+ size_t blockno, unsigned char **block)
+{
+ if (blockno == 0)
+ return get_intro(eq, data, block);
+
+ blockno--;
+
+ switch (eq->method->id) {
+ case UL_FILEEQ_MEMCMP:
+ return read_block(eq, data, blockno, block);
+ default:
+ break;
+ }
+#ifdef USE_HARDLINK_CRYPTOAPI
+ return get_digest(eq, data, blockno, block);
+#else
+ return -1;
+#endif
+}
+
+#define CMP(a, b) ((a) > (b) ? 1 : ((a) < (b) ? -1 : 0))
+
+int ul_fileeq(struct ul_fileeq *eq,
+ struct ul_fileeq_data *a, struct ul_fileeq_data *b)
+{
+ int cmp;
+ size_t n = 0;
+
+ DBG(EQ, ul_debugobj(eq, "--> compare %s %s", a->name, b->name));
+
+ if (eq->method->id == UL_FILEEQ_MEMCMP) {
+ memcmp_reset(eq, a);
+ memcmp_reset(eq, b);
+ }
+
+ do {
+ unsigned char *da, *db;
+ ssize_t ca, cb;
+
+ DBG(EQ, ul_debugobj(eq, "compare block #%zu", n));
+
+ ca = get_cmp_data(eq, a, n, &da);
+ if (ca < 0)
+ goto done;
+ cb = get_cmp_data(eq, b, n, &db);
+ if (cb < 0)
+ goto done;
+ if (ca != cb || ca == 0) {
+ cmp = CMP(ca, cb);
+ break;
+
+ }
+ cmp = memcmp(da, db, ca);
+ DBG(EQ, ul_debugobj(eq, "#%zu=%s", n, cmp == 0 ? "match" : "not-match"));
+ n++;
+ } while (cmp == 0);
+
+ if (cmp == 0) {
+ if (!a->is_eof || !b->is_eof)
+ goto done; /* filesize chnaged? */
+
+ DBG(EQ, ul_debugobj(eq, "<-- MATCH"));
+ return 1;
+ }
+done:
+ DBG(EQ, ul_debugobj(eq, " <-- NOT-MATCH"));
+ return 0;
+}
+
+#ifdef TEST_PROGRAM_FILEEQ
+# include <getopt.h>
+# include <err.h>
+
+int main(int argc, char *argv[])
+{
+ struct ul_fileeq eq;
+ struct ul_fileeq_data a, b, c;
+ const char *method = "sha1";
+ static const struct option longopts[] = {
+ { "method", required_argument, NULL, 'm' },
+ { "help", no_argument, NULL, 'h' },
+ { NULL, 0, NULL, 0 }
+ };
+ int ch, rc;
+ const char *file_a = NULL, *file_b = NULL, *file_c = NULL;
+ struct stat st_a, st_b, st_c;
+
+ while ((ch = getopt_long(argc, argv, "m:", longopts, NULL)) != -1) {
+ switch (ch) {
+ case 'm':
+ method = optarg;
+ break;
+ case 'h':
+ printf("usage: %s [options] <file> <file>\n"
+ " -m, --method <memcmp|sha1|crc32> compare method\n",
+ program_invocation_short_name);
+ return EXIT_FAILURE;
+ }
+ }
+
+ if (optind < argc)
+ file_a = argv[optind++];
+ if (optind < argc)
+ file_b = argv[optind++];
+ if (optind < argc)
+ file_c = argv[optind++];
+
+ if (!file_a || !file_b)
+ errx(EXIT_FAILURE, "no files specified, see --help");
+
+ if (stat(file_a, &st_a) != 0 || !S_ISREG(st_a.st_mode))
+ errx(EXIT_FAILURE, "%s: wrong file", file_a);
+ if (stat(file_b, &st_b) != 0 || !S_ISREG(st_b.st_mode))
+ errx(EXIT_FAILURE, "%s: wrong file", file_b);
+ if (file_c && (stat(file_c, &st_c) != 0 || !S_ISREG(st_c.st_mode)))
+ errx(EXIT_FAILURE, "%s: wrong file", file_c);
+
+
+ if (st_a.st_size != st_b.st_size ||
+ (file_c && st_a.st_size != st_c.st_size))
+ errx(EXIT_FAILURE, "size of the files does not match");
+
+
+ rc = ul_fileeq_init(&eq, method);
+ if (rc != 0 && strcmp(method, "memcmp") != 0) {
+ method = "memcmp";
+ rc = ul_fileeq_init(&eq, method);
+ }
+ if (rc < 0)
+ err(EXIT_FAILURE, "failed to initialize files comparior");
+
+ ul_fileeq_data_set_file(&a, file_a);
+ ul_fileeq_data_set_file(&b, file_b);
+
+ /* 3rd is optional */
+ if (file_c)
+ ul_fileeq_data_set_file(&c, file_c);
+
+ /* filesiz, readsiz, memsiz */
+ ul_fileeq_set_size(&eq, st_a.st_size, 1024*1024, 4*1024);
+
+ rc = ul_fileeq(&eq, &a, &b);
+
+ printf("1st vs. 2nd: %s\n", rc == 1 ? "MATCH" : "NOT-MATCH");
+ if (file_c) {
+ rc = ul_fileeq(&eq, &a, &c);
+ printf("1st vs. 3rd: %s\n", rc == 1 ? "MATCH" : "NOT-MATCH");
+
+ rc = ul_fileeq(&eq, &b, &c);
+ printf("2st vs. 3rd: %s\n", rc == 1 ? "MATCH" : "NOT-MATCH");
+ }
+
+ ul_fileeq_data_deinit(&a);
+ ul_fileeq_data_deinit(&b);
+
+ if (file_c)
+ ul_fileeq_data_deinit(&c);
+
+ ul_fileeq_deinit(&eq);
+ return EXIT_FAILURE;
+}
+#endif /* TEST_PROGRAM_FILEEQ */
projects / thirdparty / util-linux.git / commitdiff
