lib: buffer - Add buffer_append_full_(file|istream)

Consume istream or file up to max_read_size or EOF.

diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am
index e77d252acfcb12f8d150a6bc044e3ee2da1a036e..353748f6e1b23b708d222ee316acbe1f3032c3f2 100644 (file)
--- a/src/lib/Makefile.am
+++ b/src/lib/Makefile.am
@@ -47,6 +47,7 @@ liblib_la_SOURCES = \
        bits.c \
        bsearch-insert-pos.c \
        buffer.c \
+       buffer-istream.c \
        child-wait.c \
        compat.c \
        connection.c \
@@ -372,6 +373,7 @@ test_lib_SOURCES = \
        test-bits.c \
        test-bsearch-insert-pos.c \
        test-buffer.c \
+       test-buffer-istream.c \
        test-byteorder.c \
        test-connection.c \
        test-crc32.c \

diff --git a/src/lib/buffer-istream.c b/src/lib/buffer-istream.c
new file mode 100644 (file)
index 0000000..3898fd0
--- /dev/null
+++ b/src/lib/buffer-istream.c
@@ -0,0 +1,49 @@
+/* Copyright (c) 2020 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "buffer.h"
+#include "eacces-error.h"
+#include "istream.h"
+
+enum buffer_append_result
+buffer_append_full_istream(buffer_t *buf, struct istream *is, size_t max_read_size,
+                          const char **error_r)
+{
+       const unsigned char *data;
+       size_t size;
+       ssize_t ret;
+
+       while ((ret = i_stream_read_more(is, &data, &size)) > 0) {
+               if (max_read_size == 0)
+                       return BUFFER_APPEND_READ_MAX_SIZE;
+               size = I_MIN(max_read_size, size);
+               buffer_append(buf, data, size);
+               i_stream_skip(is, size);
+               max_read_size -= size;
+       }
+
+       if (ret == 0)
+               return BUFFER_APPEND_READ_MORE;
+
+       i_assert(is->eof);
+
+       if (is->stream_errno != 0) {
+               *error_r = i_stream_get_error(is);
+               return BUFFER_APPEND_READ_ERROR;
+       }
+       return BUFFER_APPEND_OK;
+}
+
+enum buffer_append_result
+buffer_append_full_file(buffer_t *buf, const char *file, size_t max_read_size,
+                       const char **error_r)
+{
+       struct istream *is = i_stream_create_file(file, IO_BLOCK_SIZE);
+       enum buffer_append_result res =
+               buffer_append_full_istream(buf, is, max_read_size, error_r);
+       if (is->stream_errno == EACCES)
+               *error_r = eacces_error_get("open", file);
+       i_stream_unref(&is);
+       i_assert(res != BUFFER_APPEND_READ_MORE);
+       return res;
+}

diff --git a/src/lib/buffer.h b/src/lib/buffer.h
index 70b5ea7101081a83d3e3a2dff3f3f59b5e97d83f..5a9dc2fbb94faaa13d3af5ce5450ec5065921f09 100644 (file)
--- a/src/lib/buffer.h
+++ b/src/lib/buffer.h
@@ -162,4 +162,28 @@ void buffer_verify_pool(buffer_t *buf);
 */
 void buffer_truncate_rshift_bits(buffer_t *buf, size_t bits);
 
+enum buffer_append_result {
+       /* Stream reached EOF successfully */
+       BUFFER_APPEND_OK = 0,
+       /* Error was encountered */
+       BUFFER_APPEND_READ_ERROR = -1,
+       /* Stream is non-blocking, call again later */
+       BUFFER_APPEND_READ_MORE = -2,
+       /* Stream was consumed up to max_read_size */
+       BUFFER_APPEND_READ_MAX_SIZE = -3,
+};
+
+/* Attempt to fully read a stream. Since this can be a network stream, it
+   can return BUFFER_APPEND_READ_MORE, which means you need to call this
+   function again. It is caller's responsibility to keep track of
+   max_read_size in case more reading is needed. */
+enum buffer_append_result
+buffer_append_full_istream(buffer_t *buf, struct istream *is, size_t max_read_size,
+                          const char **error_r);
+
+/* Attempt to fully read a file. BUFFER_APPEND_READ_MORE is never returned. */
+enum buffer_append_result
+buffer_append_full_file(buffer_t *buf, const char *file, size_t max_read_size,
+                       const char **error_r);
+
 #endif

diff --git a/src/lib/test-buffer-istream.c b/src/lib/test-buffer-istream.c
new file mode 100644 (file)
index 0000000..688c3cd
--- /dev/null
+++ b/src/lib/test-buffer-istream.c
@@ -0,0 +1,101 @@
+/* Copyright (c) 2020 Dovecot authors, see the included COPYING file */
+
+#include "test-lib.h"
+#include "istream.h"
+#include "str.h"
+#include "write-full.h"
+
+#include <unistd.h>
+#include <fcntl.h>
+
+#define TEST_FILENAME ".test_buffer_append_full_file"
+static void test_buffer_append_full_file(void)
+{
+       const char *test_string = "this is a test string\n";
+       test_begin("buffer_append_full_file");
+       buffer_t *result = t_buffer_create(32);
+       const char *error;
+       int fd = open(TEST_FILENAME, O_WRONLY|O_CREAT, 0600);
+       i_assert(fd > -1);
+       test_assert(write_full(fd, test_string, strlen(test_string)) == 0);
+       i_close_fd(&fd);
+
+       test_assert(buffer_append_full_file(result, TEST_FILENAME, SIZE_MAX,
+                                           &error) == BUFFER_APPEND_OK);
+       test_assert_strcmp(str_c(result), test_string);
+
+       /* test max_read_size */
+       for (size_t max = 0; max < strlen(test_string)-1; max++) {
+               buffer_set_used_size(result, 0);
+               test_assert(buffer_append_full_file(result, TEST_FILENAME,
+                                       max, &error) == BUFFER_APPEND_READ_MAX_SIZE);
+               test_assert(result->used == max &&
+                           memcmp(result->data, test_string, max) == 0);
+       }
+
+       fd = open(TEST_FILENAME, O_WRONLY|O_TRUNC);
+       i_assert(fd > -1);
+       /* write it enough many times */
+       for (size_t i = 0; i < IO_BLOCK_SIZE; i += strlen(test_string)) {
+               test_assert(write_full(fd, test_string, strlen(test_string)) == 0);
+       }
+       i_close_fd(&fd);
+       buffer_set_used_size(result, 0);
+       test_assert(buffer_append_full_file(result, TEST_FILENAME,
+                                           SIZE_MAX, &error) == BUFFER_APPEND_OK);
+       for (size_t i = 0; i < result->used; i += strlen(test_string)) {
+               const char *data = result->data;
+               data += i;
+               test_assert(memcmp(data, test_string, strlen(test_string)) == 0);
+       }
+       buffer_set_used_size(result, 0);
+       test_assert(chmod(TEST_FILENAME, 0) == 0);
+       error = NULL;
+       test_assert(buffer_append_full_file(result, TEST_FILENAME, SIZE_MAX,
+                                           &error) == BUFFER_APPEND_READ_ERROR);
+       test_assert(error != NULL && *error != '\0');
+       buffer_set_used_size(result, 0);
+       test_assert(chmod(TEST_FILENAME, 0700) == 0);
+       /* test permission problems */
+       i_unlink(TEST_FILENAME);
+       test_assert(buffer_append_full_file(result, TEST_FILENAME, SIZE_MAX,
+                                           &error) == BUFFER_APPEND_READ_ERROR);
+       test_assert_strcmp(str_c(result), "");
+       test_end();
+}
+
+static void test_buffer_append_full_istream(void)
+{
+       int fds[2];
+       const char *error;
+       test_begin("buffer_append_full_istream");
+       buffer_t *result = t_buffer_create(32);
+       test_assert(pipe(fds) == 0);
+       fd_set_nonblock(fds[0], TRUE);
+       fd_set_nonblock(fds[1], TRUE);
+
+       struct istream *is = i_stream_create_fd(fds[0], (size_t)-1);
+       /* test just the READ_MORE stuff */
+
+       test_assert(write_full(fds[1], "some data ", 10) == 0);
+
+       test_assert(buffer_append_full_istream(result, is, SIZE_MAX, &error) ==
+                   BUFFER_APPEND_READ_MORE);
+       test_assert(write_full(fds[1], "final read", 10) == 0);
+       i_close_fd(&fds[1]);
+
+       test_assert(buffer_append_full_istream(result, is, SIZE_MAX, &error) ==
+                   BUFFER_APPEND_OK);
+       test_assert_strcmp(str_c(result), "some data final read");
+       i_stream_unref(&is);
+       i_close_fd(&fds[0]);
+
+       test_end();
+}
+
+void test_buffer_append_full(void)
+{
+       test_buffer_append_full_file();
+       test_buffer_append_full_istream();
+}
+

diff --git a/src/lib/test-lib.inc b/src/lib/test-lib.inc
index 0f29e58fd4db16861077e2bf41141c3b391e8db7..186fd017435c4bb215672199f01e8821ad37ce2a 100644 (file)
--- a/src/lib/test-lib.inc
+++ b/src/lib/test-lib.inc
@@ -11,6 +11,7 @@ TEST(test_base64)
 TEST(test_bits)
 TEST(test_bsearch_insert_pos)
 TEST(test_buffer)
+TEST(test_buffer_append_full)
 TEST(test_byteorder)
 TEST(test_connection)
 TEST(test_crc32)