From: Timo Sirainen <tss@iki.fi>
Date: Wed, 6 Nov 2013 21:00:58 +0000 (+0200)
Subject: lib-storage: Moved more of dbox attachments code to generic code.
X-Git-Tag: 2.2.8~41
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=52de839a8249bff5eace53dc1401b28baa0c124b;p=thirdparty%2Fdovecot%2Fcore.git

lib-storage: Moved more of dbox attachments code to generic code.
---

diff --git a/src/lib-storage/index/dbox-common/dbox-attachment.c b/src/lib-storage/index/dbox-common/dbox-attachment.c
index 35a835f61f..caa3ee5ca1 100644
--- a/src/lib-storage/index/dbox-common/dbox-attachment.c
+++ b/src/lib-storage/index/dbox-common/dbox-attachment.c
@@ -3,155 +3,32 @@
 #include "lib.h"
 #include "istream.h"
 #include "str.h"
-#include "fs-api.h"
-#include "istream-fs-file.h"
-#include "istream-attachment-connector.h"
 #include "dbox-file.h"
 #include "dbox-save.h"
 #include "dbox-attachment.h"
 
-enum dbox_attachment_decode_option {
-	DBOX_ATTACHMENT_DECODE_OPTION_NONE = '-',
-	DBOX_ATTACHMENT_DECODE_OPTION_BASE64 = 'B',
-	DBOX_ATTACHMENT_DECODE_OPTION_CRLF = 'C'
-};
-
 void dbox_attachment_save_write_metadata(struct mail_save_context *ctx,
 					 string_t *str)
 {
 	const ARRAY_TYPE(mail_attachment_extref) *extrefs;
-	const struct mail_attachment_extref *extref;
-	bool add_space = FALSE;
-	unsigned int startpos;
 
 	extrefs = index_attachment_save_get_extrefs(ctx);
 	if (extrefs == NULL || array_count(extrefs) == 0)
 		return;
 
 	str_append_c(str, DBOX_METADATA_EXT_REF);
-	array_foreach(extrefs, extref) {
-		if (!add_space)
-			add_space = TRUE;
-		else
-			str_append_c(str, ' ');
-		str_printfa(str, "%"PRIuUOFF_T" %"PRIuUOFF_T" ",
-			    extref->start_offset, extref->size);
-
-		startpos = str_len(str);
-		if (extref->base64_have_crlf)
-			str_append_c(str, DBOX_ATTACHMENT_DECODE_OPTION_CRLF);
-		if (extref->base64_blocks_per_line > 0) {
-			str_printfa(str, "%c%u",
-				    DBOX_ATTACHMENT_DECODE_OPTION_BASE64,
-				    extref->base64_blocks_per_line * 4);
-		}
-		if (startpos == str_len(str)) {
-			/* make it clear there are no options */
-			str_append_c(str, DBOX_ATTACHMENT_DECODE_OPTION_NONE);
-		}
-		str_append_c(str, ' ');
-		str_append(str, extref->path);
-	}
+	index_attachment_append_extrefs(str, extrefs);
 	str_append_c(str, '\n');
 }
 
-static bool
-parse_extref_decode_options(const char *str,
-			    struct mail_attachment_extref *extref)
-{
-	unsigned int num;
-
-	if (*str == DBOX_ATTACHMENT_DECODE_OPTION_NONE)
-		return str[1] == '\0';
-
-	while (*str != '\0') {
-		switch (*str) {
-		case DBOX_ATTACHMENT_DECODE_OPTION_BASE64:
-			str++; num = 0;
-			while (*str >= '0' && *str <= '9') {
-				num = num*10 + (*str-'0');
-				str++;
-			}
-			if (num == 0 || num % 4 != 0)
-				return FALSE;
-
-			extref->base64_blocks_per_line = num/4;
-			break;
-		case DBOX_ATTACHMENT_DECODE_OPTION_CRLF:
-			extref->base64_have_crlf = TRUE;
-			str++;
-			break;
-		default:
-			return FALSE;
-		}
-	}
-	return TRUE;
-}
-
-static bool
-dbox_attachment_parse_extref_real(const char *line, pool_t pool,
-				  ARRAY_TYPE(mail_attachment_extref) *extrefs)
-{
-	struct mail_attachment_extref extref;
-	const char *const *args;
-	unsigned int i, len;
-	uoff_t last_voffset;
-
-	args = t_strsplit(line, " ");
-	len = str_array_length(args);
-	if ((len % 4) != 0)
-		return FALSE;
-
-	last_voffset = 0;
-	for (i = 0; args[i] != NULL; i += 4) {
-		const char *start_offset_str = args[i+0];
-		const char *size_str = args[i+1];
-		const char *decode_options = args[i+2];
-		const char *path = args[i+3];
-
-		memset(&extref, 0, sizeof(extref));
-		if (str_to_uoff(start_offset_str, &extref.start_offset) < 0 ||
-		    str_to_uoff(size_str, &extref.size) < 0 ||
-		    extref.start_offset < last_voffset ||
-		    !parse_extref_decode_options(decode_options, &extref))
-			return FALSE;
-
-		last_voffset += extref.size +
-			(extref.start_offset - last_voffset);
-
-		extref.path = p_strdup(pool, path);
-		array_append(extrefs, &extref, 1);
-	}
-	return TRUE;
-}
-
-bool dbox_attachment_parse_extref(const char *line, pool_t pool,
-				  ARRAY_TYPE(mail_attachment_extref) *extrefs)
-{
-	bool ret;
-
-	T_BEGIN {
-		ret = dbox_attachment_parse_extref_real(line, pool, extrefs);
-	} T_END;
-	return ret;
-}
-
 static int
 dbox_attachment_file_get_stream_from(struct dbox_file *file,
 				     const char *ext_refs,
 				     struct istream **stream,
 				     const char **error_r)
 {
-	ARRAY_TYPE(mail_attachment_extref) extrefs_arr;
-	const struct mail_attachment_extref *extref;
-	struct istream_attachment_connector *conn;
-	struct fs_file *fsfile;
-	struct istream *input;
-	const char *path, *path_suffix;
+	const char *path_suffix;
 	uoff_t msg_size;
-	int ret;
-
-	*error_r = NULL;
 
 	if (*file->storage->attachment_dir == '\0') {
 		mail_storage_set_critical(&file->storage->storage,
@@ -160,37 +37,13 @@ dbox_attachment_file_get_stream_from(struct dbox_file *file,
 		return -1;
 	}
 
-	t_array_init(&extrefs_arr, 16);
-	if (!dbox_attachment_parse_extref_real(ext_refs, pool_datastack_create(),
-					       &extrefs_arr)) {
-		*error_r = "Broken ext-refs string";
-		return 0;
-	}
 	msg_size = dbox_file_get_plaintext_size(file);
-	conn = istream_attachment_connector_begin(*stream, msg_size);
-
 	path_suffix = file->storage->v.get_attachment_path_suffix(file);
-	array_foreach(&extrefs_arr, extref) {
-		path = t_strdup_printf("%s/%s%s", file->storage->attachment_dir,
-				       extref->path, path_suffix);
-		fsfile = fs_file_init(file->storage->attachment_fs, path,
-				      FS_OPEN_MODE_READONLY);
-		input = i_stream_create_fs_file(&fsfile, IO_BLOCK_SIZE);
-
-		ret = istream_attachment_connector_add(conn, input,
-					extref->start_offset, extref->size,
-					extref->base64_blocks_per_line,
-					extref->base64_have_crlf, error_r);
-		i_stream_unref(&input);
-		if (ret < 0) {
-			istream_attachment_connector_abort(&conn);
-			return 0;
-		}
-	}
-
-	input = istream_attachment_connector_finish(&conn);
-	i_stream_unref(stream);
-	*stream = input;
+	if (index_attachment_stream_get(file->storage->attachment_fs,
+					file->storage->attachment_dir,
+					path_suffix, stream, msg_size,
+					ext_refs, error_r) < 0)
+		return 0;
 	return 1;
 }
 
diff --git a/src/lib-storage/index/dbox-common/dbox-attachment.h b/src/lib-storage/index/dbox-common/dbox-attachment.h
index 943bdf0ca1..a90ba541e2 100644
--- a/src/lib-storage/index/dbox-common/dbox-attachment.h
+++ b/src/lib-storage/index/dbox-common/dbox-attachment.h
@@ -8,10 +8,6 @@ struct dbox_file;
 void dbox_attachment_save_write_metadata(struct mail_save_context *ctx,
 					 string_t *str);
 
-/* Parse DBOX_METADATA_EXT_REF line to given array. Names are allocated
-   from the given pool. */
-bool dbox_attachment_parse_extref(const char *line, pool_t pool,
-				  ARRAY_TYPE(mail_attachment_extref) *extrefs);
 /* Build a single message body stream out of the current message and all of its
    attachments. */
 int dbox_attachment_file_get_stream(struct dbox_file *file,
diff --git a/src/lib-storage/index/dbox-multi/mdbox-purge.c b/src/lib-storage/index/dbox-multi/mdbox-purge.c
index 3c5d9d2a1b..41decd67ac 100644
--- a/src/lib-storage/index/dbox-multi/mdbox-purge.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-purge.c
@@ -147,13 +147,13 @@ mdbox_metadata_get_extrefs(struct dbox_file *file, pool_t ext_refs_pool,
 			/* end of metadata */
 			break;
 		}
-		if (*line == DBOX_METADATA_EXT_REF) {
-			if (!dbox_attachment_parse_extref(line+1, ext_refs_pool,
-							  extrefs)) {
+		if (*line == DBOX_METADATA_EXT_REF) T_BEGIN {
+			if (!index_attachment_parse_extrefs(line+1, ext_refs_pool,
+							    extrefs)) {
 				i_warning("%s: Ignoring corrupted extref: %s",
 					  file->cur_path, line);
 			}
-		}
+		} T_END;
 	}
 	i_stream_set_max_buffer_size(file->input, buf_size);
 
diff --git a/src/lib-storage/index/dbox-single/sdbox-copy.c b/src/lib-storage/index/dbox-single/sdbox-copy.c
index e3fa5540bd..7a1ed3c354 100644
--- a/src/lib-storage/index/dbox-single/sdbox-copy.c
+++ b/src/lib-storage/index/dbox-single/sdbox-copy.c
@@ -40,7 +40,7 @@ sdbox_file_copy_attachments(struct sdbox_file *src_file,
 
 	pool = pool_alloconly_create("sdbox attachments copy", 1024);
 	p_array_init(&extrefs, pool, 16);
-	if (!dbox_attachment_parse_extref(extrefs_line, pool, &extrefs)) {
+	if (!index_attachment_parse_extrefs(extrefs_line, pool, &extrefs)) {
 		mail_storage_set_critical(&dest_storage->storage,
 			"Can't copy %s with corrupted extref metadata: %s",
 			src_file->file.cur_path, extrefs_line);
diff --git a/src/lib-storage/index/dbox-single/sdbox-file.c b/src/lib-storage/index/dbox-single/sdbox-file.c
index b020c60725..a9ba411f57 100644
--- a/src/lib-storage/index/dbox-single/sdbox-file.c
+++ b/src/lib-storage/index/dbox-single/sdbox-file.c
@@ -436,7 +436,7 @@ int sdbox_file_unlink_with_attachments(struct sdbox_file *sfile)
 
 	pool = pool_alloconly_create("sdbox attachments unlink", 1024);
 	p_array_init(&extrefs, pool, 16);
-	if (!dbox_attachment_parse_extref(extrefs_line, pool, &extrefs)) {
+	if (!index_attachment_parse_extrefs(extrefs_line, pool, &extrefs)) {
 		i_warning("%s: Ignoring corrupted extref: %s",
 			  sfile->file.cur_path, extrefs_line);
 		array_clear(&extrefs);
diff --git a/src/lib-storage/index/dbox-single/sdbox-sync.c b/src/lib-storage/index/dbox-single/sdbox-sync.c
index 9ed7b9f007..ab6009e704 100644
--- a/src/lib-storage/index/dbox-single/sdbox-sync.c
+++ b/src/lib-storage/index/dbox-single/sdbox-sync.c
@@ -160,8 +160,9 @@ static void dbox_sync_expunge_files(struct sdbox_sync_context *ctx)
 	/* NOTE: Index is no longer locked. Multiple processes may be unlinking
 	   the files at the same time. */
 	ctx->mbox->box.tmp_sync_view = ctx->sync_view;
-	array_foreach(&ctx->expunged_uids, uidp)
+	array_foreach(&ctx->expunged_uids, uidp) T_BEGIN {
 		dbox_sync_file_expunge(ctx, *uidp);
+	} T_END;
 	if (ctx->mbox->box.v.sync_notify != NULL)
 		ctx->mbox->box.v.sync_notify(&ctx->mbox->box, 0, 0);
 	ctx->mbox->box.tmp_sync_view = NULL;
diff --git a/src/lib-storage/index/index-attachment.c b/src/lib-storage/index/index-attachment.c
index 29ca262d6d..3950381897 100644
--- a/src/lib-storage/index/index-attachment.c
+++ b/src/lib-storage/index/index-attachment.c
@@ -10,11 +10,20 @@
 #include "str.h"
 #include "message-parser.h"
 #include "rfc822-parser.h"
+#include "fs-api.h"
+#include "istream-fs-file.h"
+#include "istream-attachment-connector.h"
 #include "istream-attachment-extractor.h"
 #include "mail-user.h"
 #include "index-mail.h"
 #include "index-attachment.h"
 
+enum mail_attachment_decode_option {
+	MAIL_ATTACHMENT_DECODE_OPTION_NONE = '-',
+	MAIL_ATTACHMENT_DECODE_OPTION_BASE64 = 'B',
+	MAIL_ATTACHMENT_DECODE_OPTION_CRLF = 'C'
+};
+
 struct mail_save_attachment {
 	pool_t pool;
 	struct fs *fs;
@@ -104,7 +113,7 @@ index_attachment_open_ostream(struct istream_attachment_info *info,
 			       digest[2], digest[3], digest,
 			       guid_128_to_string(guid_128));
 	attach->cur_file = fs_file_init(attach->fs, path,
-					FS_OPEN_MODE_CREATE | flags);
+					FS_OPEN_MODE_REPLACE | flags);
 
 	extref = array_append_space(&attach->extrefs);
 	extref->start_offset = info->start_offset;
@@ -286,3 +295,150 @@ int index_attachment_delete(struct mail_storage *storage,
 	} T_END;
 	return ret;
 }
+
+void index_attachment_append_extrefs(string_t *str,
+	const ARRAY_TYPE(mail_attachment_extref) *extrefs)
+{
+	const struct mail_attachment_extref *extref;
+	bool add_space = FALSE;
+	unsigned int startpos;
+
+	array_foreach(extrefs, extref) {
+		if (!add_space)
+			add_space = TRUE;
+		else
+			str_append_c(str, ' ');
+		str_printfa(str, "%"PRIuUOFF_T" %"PRIuUOFF_T" ",
+			    extref->start_offset, extref->size);
+
+		startpos = str_len(str);
+		if (extref->base64_have_crlf)
+			str_append_c(str, MAIL_ATTACHMENT_DECODE_OPTION_CRLF);
+		if (extref->base64_blocks_per_line > 0) {
+			str_printfa(str, "%c%u",
+				    MAIL_ATTACHMENT_DECODE_OPTION_BASE64,
+				    extref->base64_blocks_per_line * 4);
+		}
+		if (startpos == str_len(str)) {
+			/* make it clear there are no options */
+			str_append_c(str, MAIL_ATTACHMENT_DECODE_OPTION_NONE);
+		}
+		str_append_c(str, ' ');
+		str_append(str, extref->path);
+	}
+}
+
+static bool
+parse_extref_decode_options(const char *str,
+			    struct mail_attachment_extref *extref)
+{
+	unsigned int num;
+
+	if (*str == MAIL_ATTACHMENT_DECODE_OPTION_NONE)
+		return str[1] == '\0';
+
+	while (*str != '\0') {
+		switch (*str) {
+		case MAIL_ATTACHMENT_DECODE_OPTION_BASE64:
+			str++; num = 0;
+			while (*str >= '0' && *str <= '9') {
+				num = num*10 + (*str-'0');
+				str++;
+			}
+			if (num == 0 || num % 4 != 0)
+				return FALSE;
+
+			extref->base64_blocks_per_line = num/4;
+			break;
+		case MAIL_ATTACHMENT_DECODE_OPTION_CRLF:
+			extref->base64_have_crlf = TRUE;
+			str++;
+			break;
+		default:
+			return FALSE;
+		}
+	}
+	return TRUE;
+}
+
+bool index_attachment_parse_extrefs(const char *line, pool_t pool,
+				    ARRAY_TYPE(mail_attachment_extref) *extrefs)
+{
+	struct mail_attachment_extref extref;
+	const char *const *args;
+	unsigned int i, len;
+	uoff_t last_voffset;
+
+	args = t_strsplit(line, " ");
+	len = str_array_length(args);
+	if ((len % 4) != 0)
+		return FALSE;
+
+	last_voffset = 0;
+	for (i = 0; args[i] != NULL; i += 4) {
+		const char *start_offset_str = args[i+0];
+		const char *size_str = args[i+1];
+		const char *decode_options = args[i+2];
+		const char *path = args[i+3];
+
+		memset(&extref, 0, sizeof(extref));
+		if (str_to_uoff(start_offset_str, &extref.start_offset) < 0 ||
+		    str_to_uoff(size_str, &extref.size) < 0 ||
+		    extref.start_offset < last_voffset ||
+		    !parse_extref_decode_options(decode_options, &extref))
+			return FALSE;
+
+		last_voffset += extref.size +
+			(extref.start_offset - last_voffset);
+
+		extref.path = p_strdup(pool, path);
+		array_append(extrefs, &extref, 1);
+	}
+	return TRUE;
+}
+
+int index_attachment_stream_get(struct fs *fs, const char *attachment_dir,
+				const char *path_suffix,
+				struct istream **stream, uoff_t full_size,
+				const char *ext_refs, const char **error_r)
+{
+	ARRAY_TYPE(mail_attachment_extref) extrefs_arr;
+	const struct mail_attachment_extref *extref;
+	struct istream_attachment_connector *conn;
+	struct istream *input;
+	struct fs_file *file;
+	const char *path;
+	int ret;
+
+	*error_r = NULL;
+
+	t_array_init(&extrefs_arr, 16);
+	if (!index_attachment_parse_extrefs(ext_refs, pool_datastack_create(),
+					    &extrefs_arr)) {
+		*error_r = "Broken ext-refs string";
+		return -1;
+	}
+	conn = istream_attachment_connector_begin(*stream, full_size);
+
+	array_foreach(&extrefs_arr, extref) {
+		path = t_strdup_printf("%s/%s%s", attachment_dir,
+				       extref->path, path_suffix);
+		file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY);
+		input = i_stream_create_fs_file(&file, IO_BLOCK_SIZE);
+
+		ret = istream_attachment_connector_add(conn, input,
+					extref->start_offset, extref->size,
+					extref->base64_blocks_per_line,
+					extref->base64_have_crlf, error_r);
+		i_stream_unref(&input);
+		if (ret < 0) {
+			istream_attachment_connector_abort(&conn);
+			return -1;
+		}
+	}
+
+	input = istream_attachment_connector_finish(&conn);
+	i_stream_unref(stream);
+	*stream = input;
+	return 0;
+}
diff --git a/src/lib-storage/index/index-attachment.h b/src/lib-storage/index/index-attachment.h
index 92a4521f1a..a52f158f6b 100644
--- a/src/lib-storage/index/index-attachment.h
+++ b/src/lib-storage/index/index-attachment.h
@@ -37,4 +37,16 @@ index_attachment_save_get_extrefs(struct mail_save_context *ctx);
 int index_attachment_delete(struct mail_storage *storage,
 			    struct fs *fs, const char *name);
 
+void index_attachment_append_extrefs(string_t *str,
+	const ARRAY_TYPE(mail_attachment_extref) *extrefs);
+/* Parse extrefs value to given array. Names are allocated from the
+   given pool. */
+bool index_attachment_parse_extrefs(const char *line, pool_t pool,
+				    ARRAY_TYPE(mail_attachment_extref) *extrefs);
+
+int index_attachment_stream_get(struct fs *fs, const char *attachment_dir,
+				const char *path_suffix,
+				struct istream **stream, uoff_t full_size,
+				const char *ext_refs, const char **error_r);
+
 #endif