reftable: add blocksource, an abstraction for random access reads

The reftable format is usually used with files for storage. However, we abstract
away this using the blocksource data structure. This has two advantages:

* log blocks are zlib compressed, and handling them is simplified if we can
  discard byte segments from within the block layer.

* for unittests, it is useful to read and write in-memory. The blocksource
  allows us to abstract the data away from on-disk files.

Signed-off-by: Han-Wen Nienhuys <hanwen@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Makefile b/Makefile
index 98c965c61fd76b5e1ea5181a67c76c95e21b8994..621ac53d09f04ff6b89d92769fddeb5cf8986181 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -2451,6 +2451,7 @@ xdiff-objs: $(XDIFF_OBJS)
 
 REFTABLE_OBJS += reftable/basics.o
 REFTABLE_OBJS += reftable/error.o
+REFTABLE_OBJS += reftable/blocksource.o
 REFTABLE_OBJS += reftable/publicbasics.o
 
 REFTABLE_TEST_OBJS += reftable/test_framework.o

diff --git a/reftable/blocksource.c b/reftable/blocksource.c
new file mode 100644 (file)
index 0000000..0044eec
--- /dev/null
+++ b/reftable/blocksource.c
@@ -0,0 +1,148 @@
+/*
+Copyright 2020 Google LLC
+
+Use of this source code is governed by a BSD-style
+license that can be found in the LICENSE file or at
+https://developers.google.com/open-source/licenses/bsd
+*/
+
+#include "system.h"
+
+#include "basics.h"
+#include "blocksource.h"
+#include "reftable-blocksource.h"
+#include "reftable-error.h"
+
+static void strbuf_return_block(void *b, struct reftable_block *dest)
+{
+       memset(dest->data, 0xff, dest->len);
+       reftable_free(dest->data);
+}
+
+static void strbuf_close(void *b)
+{
+}
+
+static int strbuf_read_block(void *v, struct reftable_block *dest, uint64_t off,
+                            uint32_t size)
+{
+       struct strbuf *b = v;
+       assert(off + size <= b->len);
+       dest->data = reftable_calloc(size);
+       memcpy(dest->data, b->buf + off, size);
+       dest->len = size;
+       return size;
+}
+
+static uint64_t strbuf_size(void *b)
+{
+       return ((struct strbuf *)b)->len;
+}
+
+static struct reftable_block_source_vtable strbuf_vtable = {
+       .size = &strbuf_size,
+       .read_block = &strbuf_read_block,
+       .return_block = &strbuf_return_block,
+       .close = &strbuf_close,
+};
+
+void block_source_from_strbuf(struct reftable_block_source *bs,
+                             struct strbuf *buf)
+{
+       assert(!bs->ops);
+       bs->ops = &strbuf_vtable;
+       bs->arg = buf;
+}
+
+static void malloc_return_block(void *b, struct reftable_block *dest)
+{
+       memset(dest->data, 0xff, dest->len);
+       reftable_free(dest->data);
+}
+
+static struct reftable_block_source_vtable malloc_vtable = {
+       .return_block = &malloc_return_block,
+};
+
+static struct reftable_block_source malloc_block_source_instance = {
+       .ops = &malloc_vtable,
+};
+
+struct reftable_block_source malloc_block_source(void)
+{
+       return malloc_block_source_instance;
+}
+
+struct file_block_source {
+       int fd;
+       uint64_t size;
+};
+
+static uint64_t file_size(void *b)
+{
+       return ((struct file_block_source *)b)->size;
+}
+
+static void file_return_block(void *b, struct reftable_block *dest)
+{
+       memset(dest->data, 0xff, dest->len);
+       reftable_free(dest->data);
+}
+
+static void file_close(void *b)
+{
+       int fd = ((struct file_block_source *)b)->fd;
+       if (fd > 0) {
+               close(fd);
+               ((struct file_block_source *)b)->fd = 0;
+       }
+
+       reftable_free(b);
+}
+
+static int file_read_block(void *v, struct reftable_block *dest, uint64_t off,
+                          uint32_t size)
+{
+       struct file_block_source *b = v;
+       assert(off + size <= b->size);
+       dest->data = reftable_malloc(size);
+       if (pread(b->fd, dest->data, size, off) != size)
+               return -1;
+       dest->len = size;
+       return size;
+}
+
+static struct reftable_block_source_vtable file_vtable = {
+       .size = &file_size,
+       .read_block = &file_read_block,
+       .return_block = &file_return_block,
+       .close = &file_close,
+};
+
+int reftable_block_source_from_file(struct reftable_block_source *bs,
+                                   const char *name)
+{
+       struct stat st = { 0 };
+       int err = 0;
+       int fd = open(name, O_RDONLY);
+       struct file_block_source *p = NULL;
+       if (fd < 0) {
+               if (errno == ENOENT) {
+                       return REFTABLE_NOT_EXIST_ERROR;
+               }
+               return -1;
+       }
+
+       err = fstat(fd, &st);
+       if (err < 0)
+               return -1;
+
+       p = reftable_calloc(sizeof(struct file_block_source));
+       p->size = st.st_size;
+       p->fd = fd;
+
+       assert(!bs->ops);
+       bs->ops = &file_vtable;
+       bs->arg = p;
+       return 0;
+}

diff --git a/reftable/blocksource.h b/reftable/blocksource.h
new file mode 100644 (file)
index 0000000..072e272
--- /dev/null
+++ b/reftable/blocksource.h
@@ -0,0 +1,22 @@
+/*
+Copyright 2020 Google LLC
+
+Use of this source code is governed by a BSD-style
+license that can be found in the LICENSE file or at
+https://developers.google.com/open-source/licenses/bsd
+*/
+
+#ifndef BLOCKSOURCE_H
+#define BLOCKSOURCE_H
+
+#include "system.h"
+
+struct reftable_block_source;
+
+/* Create an in-memory block source for reading reftables */
+void block_source_from_strbuf(struct reftable_block_source *bs,
+                             struct strbuf *buf);
+
+struct reftable_block_source malloc_block_source(void);
+
+#endif

diff --git a/reftable/reftable-blocksource.h b/reftable/reftable-blocksource.h
new file mode 100644 (file)
index 0000000..5aa3990
--- /dev/null
+++ b/reftable/reftable-blocksource.h
@@ -0,0 +1,49 @@
+/*
+Copyright 2020 Google LLC
+
+Use of this source code is governed by a BSD-style
+license that can be found in the LICENSE file or at
+https://developers.google.com/open-source/licenses/bsd
+*/
+
+#ifndef REFTABLE_BLOCKSOURCE_H
+#define REFTABLE_BLOCKSOURCE_H
+
+#include <stdint.h>
+
+/* block_source is a generic wrapper for a seekable readable file.
+ */
+struct reftable_block_source {
+       struct reftable_block_source_vtable *ops;
+       void *arg;
+};
+
+/* a contiguous segment of bytes. It keeps track of its generating block_source
+ * so it can return itself into the pool. */
+struct reftable_block {
+       uint8_t *data;
+       int len;
+       struct reftable_block_source source;
+};
+
+/* block_source_vtable are the operations that make up block_source */
+struct reftable_block_source_vtable {
+       /* returns the size of a block source */
+       uint64_t (*size)(void *source);
+
+       /* reads a segment from the block source. It is an error to read
+          beyond the end of the block */
+       int (*read_block)(void *source, struct reftable_block *dest,
+                         uint64_t off, uint32_t size);
+       /* mark the block as read; may return the data back to malloc */
+       void (*return_block)(void *source, struct reftable_block *blockp);
+
+       /* release all resources associated with the block source */
+       void (*close)(void *source);
+};
+
+/* opens a file on the file system as a block_source */
+int reftable_block_source_from_file(struct reftable_block_source *block_src,
+                                   const char *name);
+
+#endif