The idea is to use istreams for larger values.
static bool dsync_brain_recv_mailbox_attribute(struct dsync_brain *brain)
{
const struct dsync_mailbox_attribute *attr;
+ struct istream *input;
enum dsync_ibc_recv_ret ret;
if ((ret = dsync_ibc_recv_mailbox_attribute(brain->ibc, &attr)) == 0)
}
if (dsync_mailbox_import_attribute(brain->box_importer, attr) < 0)
brain->failed = TRUE;
+ input = attr->value_stream;
+ if (input != NULL)
+ i_stream_unref(&input);
return TRUE;
}
{ .name = "mailbox_attribute",
.chr = 'A',
.required_keys = "type key",
- .optional_keys = "value deleted last_change modseq"
+ .optional_keys = "value stream deleted last_change modseq"
},
{ .name = "mail_change",
.chr = 'C',
pool_t ret_pool;
struct dsync_deserializer_decoder *cur_decoder;
- struct istream *mail_output, *mail_input;
+ struct istream *value_output, *value_input;
struct dsync_mail *cur_mail;
- char mail_output_last;
+ struct dsync_mailbox_attribute *cur_attr;
+ char value_output_last;
unsigned int version_received:1;
unsigned int handshake_received:1;
static int dsync_ibc_stream_read_mail_stream(struct dsync_ibc_stream *ibc)
{
do {
- i_stream_skip(ibc->mail_input,
- i_stream_get_data_size(ibc->mail_input));
- } while (i_stream_read(ibc->mail_input) > 0);
- if (ibc->mail_input->eof) {
- if (ibc->mail_input->stream_errno != 0) {
- errno = ibc->mail_input->stream_errno;
+ i_stream_skip(ibc->value_input,
+ i_stream_get_data_size(ibc->value_input));
+ } while (i_stream_read(ibc->value_input) > 0);
+ if (ibc->value_input->eof) {
+ if (ibc->value_input->stream_errno != 0) {
+ errno = ibc->value_input->stream_errno;
i_error("dsync(%s): read() failed: %m", ibc->name);
dsync_ibc_stream_stop(ibc);
return -1;
}
/* finished reading the mail stream */
- i_assert(ibc->mail_input->eof);
- i_stream_seek(ibc->mail_input, 0);
+ i_assert(ibc->value_input->eof);
+ i_stream_seek(ibc->value_input, 0);
ibc->has_pending_data = TRUE;
- ibc->mail_input = NULL;
+ ibc->value_input = NULL;
return 1;
}
return 0;
static void dsync_ibc_stream_input(struct dsync_ibc_stream *ibc)
{
- if (ibc->mail_input != NULL) {
+ if (ibc->value_input != NULL) {
if (dsync_ibc_stream_read_mail_stream(ibc) == 0)
return;
}
o_stream_uncork(ibc->output);
}
-static int dsync_ibc_stream_send_mail_stream(struct dsync_ibc_stream *ibc)
+static int dsync_ibc_stream_send_value_stream(struct dsync_ibc_stream *ibc)
{
const unsigned char *data;
unsigned char add;
size_t i, size;
int ret;
- while ((ret = i_stream_read_data(ibc->mail_output,
+ while ((ret = i_stream_read_data(ibc->value_output,
&data, &size, 0)) > 0) {
add = '\0';
for (i = 0; i < size; i++) {
if (data[i] == '.' &&
- ((i == 0 && ibc->mail_output_last == '\n') ||
+ ((i == 0 && ibc->value_output_last == '\n') ||
(i > 0 && data[i-1] == '\n'))) {
/* escape the dot */
add = '.';
if (i > 0) {
o_stream_nsend(ibc->output, data, i);
- ibc->mail_output_last = data[i-1];
- i_stream_skip(ibc->mail_output, i);
+ ibc->value_output_last = data[i-1];
+ i_stream_skip(ibc->value_output, i);
}
if (o_stream_get_buffer_used_size(ibc->output) >= 4096) {
if (add != '\0') {
o_stream_nsend(ibc->output, &add, 1);
- ibc->mail_output_last = add;
+ ibc->value_output_last = add;
}
}
i_assert(ret == -1);
- if (ibc->mail_output->stream_errno != 0) {
+ if (ibc->value_output->stream_errno != 0) {
i_error("dsync(%s): read(%s) failed: %m",
- ibc->name, i_stream_get_name(ibc->mail_output));
+ ibc->name, i_stream_get_name(ibc->value_output));
dsync_ibc_stream_stop(ibc);
return -1;
}
/* finished sending the stream. use "CRLF." instead of "LF." just in
case we're sending binary data that ends with CR. */
o_stream_nsend_str(ibc->output, "\r\n.\r\n");
- i_stream_unref(&ibc->mail_output);
+ i_stream_unref(&ibc->value_output);
return 1;
}
o_stream_cork(ibc->output);
if ((ret = o_stream_flush(output)) < 0)
ret = 1;
- else if (ibc->mail_output != NULL) {
- if (dsync_ibc_stream_send_mail_stream(ibc) < 0)
+ else if (ibc->value_output != NULL) {
+ if (dsync_ibc_stream_send_value_stream(ibc) < 0)
ret = 1;
}
timeout_reset(ibc->to);
if (ibc->cur_decoder != NULL)
dsync_deserializer_decode_finish(&ibc->cur_decoder);
- if (ibc->mail_output != NULL)
- i_stream_unref(&ibc->mail_output);
+ if (ibc->value_output != NULL)
+ i_stream_unref(&ibc->value_output);
else {
/* notify remote that we're closing. this is mainly to avoid
"read() failed: EOF" errors on failing dsyncs */
dsync_ibc_stream_send_string(struct dsync_ibc_stream *ibc,
const string_t *str)
{
- i_assert(ibc->mail_output == NULL);
+ i_assert(ibc->value_output == NULL);
o_stream_nsend(ibc->output, str_data(str), str_len(str));
}
+static int seekable_fd_callback(const char **path_r, void *context)
+{
+ struct dsync_ibc_stream *ibc = context;
+ string_t *path;
+ int fd;
+
+ path = t_str_new(128);
+ str_append(path, ibc->temp_path_prefix);
+ fd = safe_mkstemp(path, 0600, (uid_t)-1, (gid_t)-1);
+ if (fd == -1) {
+ i_error("safe_mkstemp(%s) failed: %m", str_c(path));
+ return -1;
+ }
+
+ /* we just want the fd, unlink it */
+ if (unlink(str_c(path)) < 0) {
+ /* shouldn't happen.. */
+ i_error("unlink(%s) failed: %m", str_c(path));
+ i_close_fd(&fd);
+ return -1;
+ }
+
+ *path_r = str_c(path);
+ return fd;
+}
+
+static struct istream *
+dsync_ibc_stream_input_stream(struct dsync_ibc_stream *ibc)
+{
+ struct istream *inputs[2];
+
+ inputs[0] = i_stream_create_dot(ibc->input, FALSE);
+ inputs[1] = NULL;
+ ibc->value_input = i_stream_create_seekable(inputs, MAIL_READ_FULL_BLOCK_SIZE,
+ seekable_fd_callback, ibc);
+ i_stream_unref(&inputs[0]);
+
+ return ibc->value_input;
+}
+
static int
dsync_ibc_check_missing_deserializers(struct dsync_ibc_stream *ibc)
{
const char *line, *error;
unsigned int i;
- i_assert(ibc->mail_input == NULL);
+ i_assert(ibc->value_input == NULL);
timeout_reset(ibc->to);
{
struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
- i_assert(ibc->mail_output == NULL);
+ i_assert(ibc->value_output == NULL);
o_stream_nsend_str(ibc->output, END_OF_LIST_LINE"\n");
}
dsync_serializer_encode_add(encoder, "key", attr->key);
if (attr->value != NULL)
dsync_serializer_encode_add(encoder, "value", attr->value);
+ else if (attr->value_stream != NULL)
+ dsync_serializer_encode_add(encoder, "stream", "");
if (attr->deleted)
dsync_serializer_encode_add(encoder, "deleted", "");
dsync_serializer_encode_finish(&encoder, str);
dsync_ibc_stream_send_string(ibc, str);
+
+ if (attr->value_stream != NULL) {
+ ibc->value_output_last = '\0';
+ ibc->value_output = attr->value_stream;
+ i_stream_ref(ibc->value_output);
+ (void)dsync_ibc_stream_send_value_stream(ibc);
+ }
}
static enum dsync_ibc_recv_ret
if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES)
return DSYNC_IBC_RECV_RET_FINISHED;
+ if (ibc->cur_attr != NULL) {
+ /* finished reading the stream, return the mail now */
+ *attr_r = ibc->cur_attr;
+ ibc->cur_attr = NULL;
+ return DSYNC_IBC_RECV_RET_OK;
+ }
+
p_clear(pool);
attr = p_new(pool, struct dsync_mailbox_attribute, 1);
value = dsync_deserializer_decode_get(decoder, "key");
attr->key = p_strdup(pool, value);
- if (dsync_deserializer_decode_try(decoder, "value", &value))
+ if (dsync_deserializer_decode_try(decoder, "stream", &value)) {
+ attr->value_stream = dsync_ibc_stream_input_stream(ibc);
+ if (dsync_ibc_stream_read_mail_stream(ibc) <= 0) {
+ ibc->cur_attr = attr;
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ /* already finished reading the stream */
+ i_assert(ibc->value_input == NULL);
+ } else if (dsync_deserializer_decode_try(decoder, "value", &value))
attr->value = p_strdup(pool, value);
if (dsync_deserializer_decode_try(decoder, "deleted", &value))
attr->deleted = TRUE;
struct dsync_serializer_encoder *encoder;
string_t *str = t_str_new(128);
- i_assert(ibc->mail_output == NULL);
+ i_assert(ibc->value_output == NULL);
str_append_c(str, items[ITEM_MAIL].chr);
encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_MAIL]);
dsync_ibc_stream_send_string(ibc, str);
if (mail->input != NULL) {
- ibc->mail_output_last = '\0';
- ibc->mail_output = mail->input;
- i_stream_ref(ibc->mail_output);
- (void)dsync_ibc_stream_send_mail_stream(ibc);
- }
-}
-
-static int seekable_fd_callback(const char **path_r, void *context)
-{
- struct dsync_ibc_stream *ibc = context;
- string_t *path;
- int fd;
-
- path = t_str_new(128);
- str_append(path, ibc->temp_path_prefix);
- fd = safe_mkstemp(path, 0600, (uid_t)-1, (gid_t)-1);
- if (fd == -1) {
- i_error("safe_mkstemp(%s) failed: %m", str_c(path));
- return -1;
- }
-
- /* we just want the fd, unlink it */
- if (unlink(str_c(path)) < 0) {
- /* shouldn't happen.. */
- i_error("unlink(%s) failed: %m", str_c(path));
- i_close_fd(&fd);
- return -1;
+ ibc->value_output_last = '\0';
+ ibc->value_output = mail->input;
+ i_stream_ref(ibc->value_output);
+ (void)dsync_ibc_stream_send_value_stream(ibc);
}
-
- *path_r = str_c(path);
- return fd;
}
static enum dsync_ibc_recv_ret
pool_t pool = ibc->ret_pool;
struct dsync_deserializer_decoder *decoder;
struct dsync_mail *mail;
- struct istream *inputs[2];
const char *value;
enum dsync_ibc_recv_ret ret;
- if (ibc->mail_input != NULL) {
+ if (ibc->value_input != NULL) {
/* wait until the mail's stream has been read */
return DSYNC_IBC_RECV_RET_TRYAGAIN;
}
return DSYNC_IBC_RECV_RET_TRYAGAIN;
}
if (dsync_deserializer_decode_try(decoder, "stream", &value)) {
- inputs[0] = i_stream_create_dot(ibc->input, FALSE);
- inputs[1] = NULL;
- mail->input = i_stream_create_seekable(inputs,
- MAIL_READ_FULL_BLOCK_SIZE, seekable_fd_callback, ibc);
- i_stream_unref(&inputs[0]);
-
- ibc->mail_input = mail->input;
+ mail->input = dsync_ibc_stream_input_stream(ibc);
if (dsync_ibc_stream_read_mail_stream(ibc) <= 0) {
ibc->cur_mail = mail;
return DSYNC_IBC_RECV_RET_TRYAGAIN;
}
/* already finished reading the stream */
- i_assert(ibc->mail_input == NULL);
+ i_assert(ibc->value_input == NULL);
}
*mail_r = mail;
{
struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
- if (ibc->mail_output != NULL) {
- i_stream_unref(&ibc->mail_output);
+ if (ibc->value_output != NULL) {
+ i_stream_unref(&ibc->value_output);
dsync_ibc_stream_stop(ibc);
}
}
struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
size_t bytes;
- if (ibc->mail_output != NULL)
+ if (ibc->value_output != NULL)
return TRUE;
bytes = o_stream_get_buffer_used_size(ibc->output);
#include "lib.h"
#include "array.h"
#include "hash.h"
+#include "istream.h"
#include "mail-index-modseq.h"
#include "mail-storage-private.h"
#include "mail-search-build.h"
unsigned int change_idx;
uint32_t highest_changed_uid;
- ARRAY(struct dsync_mailbox_attribute) attr_changes;
- unsigned int attr_change_idx;
+ struct mailbox_attribute_iter *attr_iter;
+ struct hash_iterate_context *attr_change_iter;
+ enum mail_attribute_type attr_type;
+ struct dsync_mailbox_attribute attr;
struct dsync_mail_change change;
struct dsync_mail dsync_mail;
}
static void
-dsync_mailbox_export_attr_changes(struct dsync_mailbox_exporter *exporter,
- enum mail_attribute_type type)
+dsync_mailbox_export_attr_init(struct dsync_mailbox_exporter *exporter,
+ enum mail_attribute_type type)
{
- HASH_TABLE_TYPE(dsync_attr_change) attr_changes;
- struct mailbox_attribute_iter *iter;
- struct dsync_mailbox_attribute lookup_attr, *attr;
- struct dsync_mailbox_attribute *attr_change;
- const char *key;
- struct mail_attribute_value value;
- bool export_all_attrs;
-
- export_all_attrs = exporter->return_all_mails ||
- exporter->last_common_uid == 0;
- attr_changes = dsync_transaction_log_scan_get_attr_hash(exporter->log_scan);
- lookup_attr.type = type;
-
- iter = mailbox_attribute_iter_init(exporter->box, type, "");
- while ((key = mailbox_attribute_iter_next(iter)) != NULL) {
- lookup_attr.key = key;
- attr_change = hash_table_lookup(attr_changes, &lookup_attr);
- if (attr_change == NULL && !export_all_attrs)
- continue;
-
- if (mailbox_attribute_get(exporter->trans, type, key, &value) < 0) {
- exporter->error = p_strdup_printf(exporter->pool,
- "Mailbox attribute %s lookup failed: %s", key,
- mailbox_get_last_error(exporter->box, NULL));
- break;
- }
- if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) {
- /* readonly attributes can't be changed,
- no point in exporting them */
- continue;
- }
-
- attr = array_append_space(&exporter->attr_changes);
- attr->type = type;
- attr->value = p_strdup(exporter->pool, value.value);
- attr->last_change = value.last_change;
- if (attr_change != NULL) {
- i_assert(!attr_change->exported);
- attr_change->exported = TRUE;
- attr->key = attr_change->key;
- attr->deleted = attr_change->deleted &&
- attr->value == NULL;
- attr->modseq = attr_change->modseq;
- } else {
- attr->key = p_strdup(exporter->pool, key);
- }
- }
- if (mailbox_attribute_iter_deinit(&iter) < 0) {
- exporter->error = p_strdup_printf(exporter->pool,
- "Mailbox attribute iteration failed: %s",
- mailbox_get_last_error(exporter->box, NULL));
- }
-}
-
-static void
-dsync_mailbox_export_nonexistent_attrs(struct dsync_mailbox_exporter *exporter)
-{
- HASH_TABLE_TYPE(dsync_attr_change) attr_changes;
- struct hash_iterate_context *iter;
- struct dsync_mailbox_attribute *attr;
- struct mail_attribute_value value;
-
- attr_changes = dsync_transaction_log_scan_get_attr_hash(exporter->log_scan);
-
- iter = hash_table_iterate_init(attr_changes);
- while (hash_table_iterate(iter, attr_changes, &attr, &attr)) {
- if (attr->exported || !attr->deleted)
- continue;
-
- /* lookup the value mainly to get its last_change value. */
- if (mailbox_attribute_get(exporter->trans, attr->type,
- attr->key, &value) < 0) {
- exporter->error = p_strdup_printf(exporter->pool,
- "Mailbox attribute %s lookup failed: %s", attr->key,
- mailbox_get_last_error(exporter->box, NULL));
- break;
- }
- if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0)
- continue;
-
- attr->last_change = value.last_change;
- if (value.value != NULL) {
- attr->value = p_strdup(exporter->pool, value.value);
- attr->deleted = FALSE;
- }
- attr->exported = TRUE;
- array_append(&exporter->attr_changes, attr, 1);
- }
- hash_table_iterate_deinit(&iter);
+ exporter->attr_iter =
+ mailbox_attribute_iter_init(exporter->box, type, "");
+ exporter->attr_type = type;
}
static void
/* get the changes sorted by UID */
dsync_mailbox_export_sort_changes(exporter);
- p_array_init(&exporter->attr_changes, pool, 16);
- dsync_mailbox_export_attr_changes(exporter, MAIL_ATTRIBUTE_TYPE_PRIVATE);
- dsync_mailbox_export_attr_changes(exporter, MAIL_ATTRIBUTE_TYPE_SHARED);
- dsync_mailbox_export_nonexistent_attrs(exporter);
+ dsync_mailbox_export_attr_init(exporter, MAIL_ATTRIBUTE_TYPE_PRIVATE);
return exporter;
}
+static int
+dsync_mailbox_export_iter_next_nonexistent_attr(struct dsync_mailbox_exporter *exporter)
+{
+ HASH_TABLE_TYPE(dsync_attr_change) attr_changes;
+ struct dsync_mailbox_attribute *attr;
+ struct mail_attribute_value value;
+
+ attr_changes = dsync_transaction_log_scan_get_attr_hash(exporter->log_scan);
+
+ while (hash_table_iterate(exporter->attr_change_iter, attr_changes,
+ &attr, &attr)) {
+ if (attr->exported || !attr->deleted)
+ continue;
+
+ /* lookup the value mainly to get its last_change value. */
+ if (mailbox_attribute_get_stream(exporter->trans, attr->type,
+ attr->key, &value) < 0) {
+ exporter->error = p_strdup_printf(exporter->pool,
+ "Mailbox attribute %s lookup failed: %s", attr->key,
+ mailbox_get_last_error(exporter->box, NULL));
+ break;
+ }
+ if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) {
+ if (value.value_stream != NULL)
+ i_stream_unref(&value.value_stream);
+ continue;
+ }
+
+ attr->last_change = value.last_change;
+ if (value.value != NULL || value.value_stream != NULL) {
+ attr->value = p_strdup(exporter->pool, value.value);
+ attr->value_stream = value.value_stream;
+ attr->deleted = FALSE;
+ }
+ attr->exported = TRUE;
+ exporter->attr = *attr;
+ return 1;
+ }
+ hash_table_iterate_deinit(&exporter->attr_change_iter);
+ return 0;
+}
+
+static int
+dsync_mailbox_export_iter_next_attr(struct dsync_mailbox_exporter *exporter)
+{
+ HASH_TABLE_TYPE(dsync_attr_change) attr_changes;
+ struct dsync_mailbox_attribute lookup_attr, *attr;
+ struct dsync_mailbox_attribute *attr_change;
+ const char *key;
+ struct mail_attribute_value value;
+ bool export_all_attrs;
+
+ export_all_attrs = exporter->return_all_mails ||
+ exporter->last_common_uid == 0;
+ attr_changes = dsync_transaction_log_scan_get_attr_hash(exporter->log_scan);
+ lookup_attr.type = exporter->attr_type;
+
+ /* note that the order of processing may be important for some
+ attributes. for example sieve can't set a script active until it's
+ first been created */
+ while ((key = mailbox_attribute_iter_next(exporter->attr_iter)) != NULL) {
+ lookup_attr.key = key;
+ attr_change = hash_table_lookup(attr_changes, &lookup_attr);
+ if (attr_change == NULL && !export_all_attrs)
+ continue;
+
+ if (mailbox_attribute_get_stream(exporter->trans,
+ exporter->attr_type, key,
+ &value) < 0) {
+ exporter->error = p_strdup_printf(exporter->pool,
+ "Mailbox attribute %s lookup failed: %s", key,
+ mailbox_get_last_error(exporter->box, NULL));
+ return -1;
+ }
+ if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) {
+ /* readonly attributes can't be changed,
+ no point in exporting them */
+ if (value.value_stream != NULL)
+ i_stream_unref(&value.value_stream);
+ continue;
+ }
+ if (value.value == NULL && value.value_stream == NULL &&
+ (attr_change == NULL || !attr_change->deleted)) {
+ /* the attribute was just deleted?
+ skip for this sync. */
+ continue;
+ }
+
+ attr = &exporter->attr;
+ memset(attr, 0, sizeof(*attr));
+ attr->type = exporter->attr_type;
+ attr->value = p_strdup(exporter->pool, value.value);
+ attr->value_stream = value.value_stream;
+ attr->last_change = value.last_change;
+ if (attr_change != NULL) {
+ i_assert(!attr_change->exported);
+ attr_change->exported = TRUE;
+ attr->key = attr_change->key;
+ attr->deleted = attr_change->deleted &&
+ !DSYNC_ATTR_HAS_VALUE(attr);
+ attr->modseq = attr_change->modseq;
+ } else {
+ attr->key = p_strdup(exporter->pool, key);
+ }
+ return 1;
+ }
+ if (mailbox_attribute_iter_deinit(&exporter->attr_iter) < 0) {
+ exporter->error = p_strdup_printf(exporter->pool,
+ "Mailbox attribute iteration failed: %s",
+ mailbox_get_last_error(exporter->box, NULL));
+ return -1;
+ }
+ if (exporter->attr_type == MAIL_ATTRIBUTE_TYPE_PRIVATE) {
+ /* export shared attributes */
+ dsync_mailbox_export_attr_init(exporter,
+ MAIL_ATTRIBUTE_TYPE_SHARED);
+ return dsync_mailbox_export_iter_next_attr(exporter);
+ }
+ exporter->attr_change_iter = hash_table_iterate_init(attr_changes);
+ return dsync_mailbox_export_iter_next_nonexistent_attr(exporter);
+}
+
const struct dsync_mailbox_attribute *
dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter)
{
- const struct dsync_mailbox_attribute *changes;
- unsigned int count;
-
if (exporter->error != NULL)
return NULL;
- changes = array_get(&exporter->attr_changes, &count);
- if (exporter->attr_change_idx == count)
- return NULL;
- return &changes[exporter->attr_change_idx++];
+ if (exporter->attr.value_stream != NULL)
+ i_stream_unref(&exporter->attr.value_stream);
+
+ if (exporter->attr_iter != NULL) {
+ if (dsync_mailbox_export_iter_next_attr(exporter) <= 0)
+ return NULL;
+ } else {
+ if (dsync_mailbox_export_iter_next_nonexistent_attr(exporter) <= 0)
+ return NULL;
+ }
+ return &exporter->attr;
}
const struct dsync_mail_change *
dsync_mailbox_export_body_search_deinit(exporter);
(void)mailbox_transaction_commit(&exporter->trans);
+ if (exporter->attr.value_stream != NULL)
+ i_stream_unref(&exporter->attr.value_stream);
hash_table_destroy(&exporter->export_guids);
*error_r = t_strdup(exporter->error);
static int
dsync_mailbox_import_lookup_attr(struct dsync_mailbox_importer *importer,
enum mail_attribute_type type, const char *key,
- const struct dsync_mailbox_attribute **attr_r)
+ struct dsync_mailbox_attribute **attr_r)
{
struct dsync_mailbox_attribute lookup_attr, *attr;
const struct dsync_mailbox_attribute *attr_change;
*attr_r = NULL;
- if (mailbox_attribute_get(importer->trans, type, key, &value) < 0) {
+ if (mailbox_attribute_get_stream(importer->trans, type, key, &value) < 0) {
i_error("Mailbox %s: Failed to get attribute %s: %s",
mailbox_get_vname(importer->box), key,
mailbox_get_last_error(importer->box, NULL));
attr_change = hash_table_lookup(importer->local_attr_changes,
&lookup_attr);
- if (attr_change == NULL && value.value == NULL) {
+ if (attr_change == NULL &&
+ value.value == NULL && value.value_stream == NULL) {
/* we have no knowledge of this attribute */
return 0;
}
attr->type = type;
attr->key = key;
attr->value = value.value;
+ attr->value_stream = value.value_stream;
attr->last_change = value.last_change;
if (attr_change != NULL) {
- attr->deleted = attr_change->deleted && attr->value == NULL;
+ attr->deleted = attr_change->deleted &&
+ !DSYNC_ATTR_HAS_VALUE(attr);
attr->modseq = attr_change->modseq;
}
*attr_r = attr;
return 0;
}
+static int
+dsync_istreams_cmp(struct istream *input1, struct istream *input2, int *cmp_r)
+{
+ const unsigned char *data1, *data2;
+ size_t size1, size2, size;
+
+ for (;;) {
+ (void)i_stream_read_data(input1, &data1, &size1, 0);
+ (void)i_stream_read_data(input2, &data2, &size2, 0);
+
+ if (size1 == 0 || size2 == 0)
+ break;
+ size = I_MIN(size1, size2);
+ *cmp_r = memcmp(data1, data2, size);
+ if (*cmp_r != 0)
+ return 0;
+ i_stream_skip(input1, size);
+ i_stream_skip(input2, size);
+ }
+ if (input1->stream_errno != 0) {
+ errno = input1->stream_errno;
+ i_error("read(%s) failed: %m", i_stream_get_name(input1));
+ return -1;
+ }
+ if (input2->stream_errno != 0) {
+ errno = input2->stream_errno;
+ i_error("read(%s) failed: %m", i_stream_get_name(input2));
+ return -1;
+ }
+ if (size1 == 0 && size2 == 0)
+ *cmp_r = 0;
+ else
+ *cmp_r = size1 == 0 ? -1 : 1;
+ return 0;
+}
+
+static int
+dsync_attributes_cmp_values(const struct dsync_mailbox_attribute *attr1,
+ const struct dsync_mailbox_attribute *attr2,
+ int *cmp_r)
+{
+ struct istream *input1, *input2;
+ int ret;
+
+ if (attr1->value != NULL && attr2->value != NULL) {
+ *cmp_r = strcmp(attr1->value, attr2->value);
+ return 0;
+ }
+ /* at least one of them is a stream. make both of them streams. */
+ input1 = attr1->value_stream != NULL ? attr1->value_stream :
+ i_stream_create_from_data(attr1->value, strlen(attr1->value));
+ input2 = attr2->value_stream != NULL ? attr2->value_stream :
+ i_stream_create_from_data(attr2->value, strlen(attr2->value));
+ i_stream_seek(input1, 0);
+ i_stream_seek(input2, 0);
+ ret = dsync_istreams_cmp(input1, input2, cmp_r);
+ if (attr1->value_stream == NULL)
+ i_stream_unref(&input1);
+ if (attr2->value_stream == NULL)
+ i_stream_unref(&input2);
+ return ret;
+}
+
+static int
+dsync_attributes_cmp(const struct dsync_mailbox_attribute *attr,
+ const struct dsync_mailbox_attribute *local_attr,
+ int *cmp_r)
+{
+ if (DSYNC_ATTR_HAS_VALUE(attr) &&
+ !DSYNC_ATTR_HAS_VALUE(local_attr)) {
+ /* remote has a value and local doesn't -> use it */
+ return 1;
+ } else if (!DSYNC_ATTR_HAS_VALUE(attr) &&
+ DSYNC_ATTR_HAS_VALUE(local_attr)) {
+ /* remote doesn't have a value, bt local does -> skip */
+ return -1;
+ }
+
+ return dsync_attributes_cmp_values(attr, local_attr, cmp_r);
+}
+
int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,
const struct dsync_mailbox_attribute *attr)
{
- const struct dsync_mailbox_attribute *local_attr;
+ struct dsync_mailbox_attribute *local_attr;
struct mail_attribute_value value;
- int ret;
+ int ret, cmp;
+ bool ignore = FALSE;
- i_assert(attr->value != NULL || attr->deleted);
+ i_assert(DSYNC_ATTR_HAS_VALUE(attr) || attr->deleted);
if (dsync_mailbox_import_lookup_attr(importer, attr->type,
attr->key, &local_attr) < 0)
return -1;
+ if (attr->deleted &&
+ (local_attr == NULL || !DSYNC_ATTR_HAS_VALUE(local_attr))) {
+ /* attribute doesn't exist on either side -> ignore */
+ return 0;
+ }
if (local_attr == NULL) {
/* we haven't seen this locally -> use whatever remote has */
- } else if (null_strcmp(attr->value, local_attr->value) == 0) {
- /* the values are identical anyway -> we can skip them.
- FIXME: but should timestamp/modseq still be updated?.. */
- return 0;
} else if (local_attr->modseq <= importer->last_common_modseq &&
attr->modseq > importer->last_common_modseq &&
importer->last_common_modseq > 0) {
importer->last_common_modseq > 0) {
/* we're doing incremental syncing, and we can see that the
attribute was changed locally, but not remotely -> ignore */
- return 0;
+ ignore = TRUE;
} else if (attr->last_change > local_attr->last_change) {
/* remote has a newer timestamp -> use it */
} else if (attr->last_change < local_attr->last_change) {
/* remote has an older timestamp -> ignore */
- return 0;
+ ignore = TRUE;
} else {
/* the timestamps are the same. now we're down to guessing
- the right answer. first try to use modseqs, but if even they
- are the same, fallback to just picking one based on the
+ the right answer, unless the values are actually equal,
+ so check that first. next try to use modseqs, but if even
+ they are the same, fallback to just picking one based on the
value. */
+ if (dsync_attributes_cmp(attr, local_attr, &cmp) < 0) {
+ importer->failed = TRUE;
+ return -1;
+ }
+ if (cmp == 0) {
+ /* identical scripts */
+ return 0;
+ }
+
if (attr->modseq > local_attr->modseq) {
/* remote has a higher modseq -> use it */
} else if (attr->modseq < local_attr->modseq) {
/* remote has an older modseq -> ignore */
- return 0;
- } else if (attr->value != NULL && local_attr->value == NULL) {
- /* remote has a value and local doesn't -> use it */
- } else if (attr->value == NULL && local_attr->value != NULL) {
- /* remote doesn't have a value, bt local does -> skip */
- return 0;
+ ignore = TRUE;
} else {
- /* now we have absolutely no reasonable guesses left.
- just pick one of them that are used. */
- if (strcmp(attr->value, local_attr->value) < 0)
- return 0;
+ if (cmp < 0)
+ ignore = TRUE;
}
}
+ if (ignore) {
+ if (local_attr->value_stream != NULL)
+ i_stream_unref(&local_attr->value_stream);
+ return 0;
+ }
memset(&value, 0, sizeof(value));
value.value = attr->value;
+ value.value_stream = attr->value_stream;
value.last_change = attr->last_change;
ret = mailbox_attribute_set(importer->trans, attr->type,
attr->key, &value);
mailbox_get_last_error(importer->box, NULL));
importer->failed = TRUE;
}
+ if (local_attr != NULL && local_attr->value_stream != NULL)
+ i_stream_unref(&local_attr->value_stream);
return ret;
}
dest_r->type = src->type;
dest_r->key = p_strdup(pool, src->key);
dest_r->value = p_strdup(pool, src->value);
+ if (src->value_stream != NULL) {
+ dest_r->value_stream = src->value_stream;
+ i_stream_ref(dest_r->value_stream);
+ }
dest_r->deleted = src->deleted;
dest_r->last_change = src->last_change;
struct dsync_mailbox_attribute {
enum mail_attribute_type type;
const char *key;
- const char *value; /* NULL = not looked up yet / deleted */
+ /* if both values are NULL = not looked up yet / deleted */
+ const char *value;
+ struct istream *value_stream;
time_t last_change; /* 0 = unknown */
uint64_t modseq; /* 0 = unknown */
bool deleted; /* attribute is known to have been deleted */
bool exported; /* internally used by exporting */
};
+#define DSYNC_ATTR_HAS_VALUE(attr) \
+ ((attr)->value != NULL || (attr)->value_stream != NULL)
void dsync_mailbox_attribute_dup(pool_t pool,
const struct dsync_mailbox_attribute *src,
struct dict_transaction_context *dtrans;
const char *mailbox_prefix;
bool pvt = type == MAIL_ATTRIBUTE_TYPE_PRIVATE;
+ int ret = 0;
if (strncmp(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT,
strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT)) == 0) {
T_BEGIN {
const char *prefixed_key =
key_get_prefixed(type, mailbox_prefix, key);
+ const char *value_str;
- if (value->value != NULL) {
- dict_set(dtrans, prefixed_key, value->value);
+ if (mailbox_attribute_value_to_string(t->box->storage, value,
+ &value_str) < 0) {
+ ret = -1;
+ } else if (value_str != NULL) {
+ dict_set(dtrans, prefixed_key, value_str);
mail_index_attribute_set(t->itrans, pvt, key);
} else {
dict_unset(dtrans, prefixed_key);
mail_index_attribute_unset(t->itrans, pvt, key);
}
} T_END;
- return 0;
+ return ret;
}
int index_storage_attribute_get(struct mailbox_transaction_context *t,
#include "ioloop.h"
#include "array.h"
#include "llist.h"
+#include "str.h"
#include "unichar.h"
#include "istream.h"
#include "eacces-error.h"
return t->box->v.attribute_set(t, type, key, &value);
}
+int mailbox_attribute_value_to_string(struct mail_storage *storage,
+ const struct mail_attribute_value *value,
+ const char **str_r)
+{
+ string_t *str;
+ const unsigned char *data;
+ size_t size;
+
+ if (value->value_stream == NULL) {
+ *str_r = value->value;
+ return 0;
+ }
+ str = t_str_new(128);
+ i_stream_seek(value->value_stream, 0);
+ while (i_stream_read_data(value->value_stream, &data, &size, 0) > 0) {
+ if (memchr(data, '\0', size) != NULL) {
+ mail_storage_set_error(storage, MAIL_ERROR_PARAMS,
+ "Attribute string value has NULs");
+ return -1;
+ }
+ str_append_n(str, data, size);
+ i_stream_skip(value->value_stream, size);
+ }
+ if (value->value_stream->stream_errno != 0) {
+ mail_storage_set_critical(storage, "read(%s) failed: %m",
+ i_stream_get_name(value->value_stream));
+ return -1;
+ }
+ i_assert(value->value_stream->eof);
+ *str_r = str_c(str);
+ return 0;
+}
+
int mailbox_attribute_get(struct mailbox_transaction_context *t,
enum mail_attribute_type type, const char *key,
struct mail_attribute_value *value_r)
{
- return t->box->v.attribute_get(t, type, key, value_r);
+ int ret;
+
+ memset(value_r, 0, sizeof(*value_r));
+ if ((ret = t->box->v.attribute_get(t, type, key, value_r)) <= 0)
+ return ret;
+ i_assert(value_r->value != NULL);
+ return 1;
+}
+
+int mailbox_attribute_get_stream(struct mailbox_transaction_context *t,
+ enum mail_attribute_type type, const char *key,
+ struct mail_attribute_value *value_r)
+{
+ int ret;
+
+ memset(value_r, 0, sizeof(*value_r));
+ value_r->flags |= MAIL_ATTRIBUTE_VALUE_FLAG_INT_STREAMS;
+ if ((ret = t->box->v.attribute_get(t, type, key, value_r)) <= 0)
+ return ret;
+ i_assert(value_r->value != NULL || value_r->value_stream != NULL);
+ return 1;
}
struct mailbox_attribute_iter *
MAIL_ATTRIBUTE_TYPE_SHARED
};
enum mail_attribute_value_flags {
- MAIL_ATTRIBUTE_VALUE_FLAG_READONLY = 0x01
+ MAIL_ATTRIBUTE_VALUE_FLAG_READONLY = 0x01,
+ MAIL_ATTRIBUTE_VALUE_FLAG_INT_STREAMS = 0x02
};
struct mail_attribute_value {
- /* The actual value */
+ /* mailbox_attribute_set() can set either value or value_stream.
+ mailbox_attribute_get() returns only values, but
+ mailbox_attribute_get_stream() may return either value or
+ value_stream. The caller must unreference the returned streams. */
const char *value;
+ struct istream *value_stream;
+
/* Last time the attribute was changed (0 = unknown). This may be
returned even for values that don't exist anymore. */
time_t last_change;
int mailbox_attribute_get(struct mailbox_transaction_context *t,
enum mail_attribute_type type, const char *key,
struct mail_attribute_value *value_r);
+/* Same as mailbox_attribute_get(), but the returned value may be either an
+ input stream or a string. */
+int mailbox_attribute_get_stream(struct mailbox_transaction_context *t,
+ enum mail_attribute_type type, const char *key,
+ struct mail_attribute_value *value_r);
/* Iterate through mailbox attributes of the given type. The prefix can be used
to restrict what attributes are returned. */
acl_attribute_update_acl(struct mailbox_transaction_context *t, const char *key,
const struct mail_attribute_value *value)
{
- const char *id, *const *rights, *error;
+ const char *value_str, *id, *const *rights, *error;
struct acl_rights_update update;
/* for now allow only admin (=dsync) to update ACLs this way.
return -1;
}
+ if (mailbox_attribute_value_to_string(t->box->storage, value,
+ &value_str) < 0)
+ return -1;
+
memset(&update, 0, sizeof(update));
update.modify_mode = ACL_MODIFY_MODE_REPLACE;
update.neg_modify_mode = ACL_MODIFY_MODE_REPLACE;
update.last_change = value->last_change;
id = key + strlen(MAILBOX_ATTRIBUTE_PREFIX_ACL);
- rights = value->value == NULL ? NULL : t_strsplit(value->value, " ");
+ rights = value_str == NULL ? NULL : t_strsplit(value_str, " ");
if (acl_rights_update_import(&update, id, rights, &error) < 0) {
mail_storage_set_error(t->box->storage, MAIL_ERROR_PARAMS, error);
return -1;
projects / thirdparty / dovecot / core.git / commitdiff
