lib-storage: Fixed mailbox_get_expunges() to return GUIDs with all mailbox formats.

Also added mailbox_get_expunged_uids() for use cases that don't really need
GUIDs.

diff --git a/src/lib-storage/index/index-fetch.c b/src/lib-storage/index/index-fetch.c
index 77801ecf33978069fc9a2a20509dbc20790116eb..e8e4c2bbed7d7a7ba718f7cb493f83c681cfd8ce 100644 (file)
--- a/src/lib-storage/index/index-fetch.c
+++ b/src/lib-storage/index/index-fetch.c
@@ -37,51 +37,40 @@ void index_storage_get_uid_range(struct mailbox *box,
 }
 
 static void
-add_expunges(ARRAY_TYPE(mailbox_expunge_rec) *expunges,
-            const struct mail_transaction_expunge *src, size_t src_size,
-            const ARRAY_TYPE(seq_range) *uids_filter)
+add_expunges(ARRAY_TYPE(seq_range) *expunged_uids, uint32_t min_uid,
+            const struct mail_transaction_expunge *src, size_t src_size)
 {
        const struct mail_transaction_expunge *end;
-       struct mailbox_expunge_rec *expunge;
-       uint32_t uid;
 
        end = src + src_size / sizeof(*src);
        for (; src != end; src++) {
-               for (uid = src->uid1; uid <= src->uid2; uid++) {
-                       if (seq_range_exists(uids_filter, uid)) {
-                               expunge = array_append_space(expunges);
-                               expunge->uid = uid;
-                       }
+               if (src->uid2 >= min_uid) {
+                       seq_range_array_add_range(expunged_uids,
+                                                 src->uid1, src->uid2);
                }
        }
 }
 
 static void
-add_guid_expunges(ARRAY_TYPE(mailbox_expunge_rec) *expunges,
+add_guid_expunges(ARRAY_TYPE(seq_range) *expunged_uids, uint32_t min_uid,
                  const struct mail_transaction_expunge_guid *src,
-                 size_t src_size, const ARRAY_TYPE(seq_range) *uids_filter)
+                 size_t src_size)
 {
        const struct mail_transaction_expunge_guid *end;
-       struct mailbox_expunge_rec *expunge;
 
        end = src + src_size / sizeof(*src);
        for (; src != end; src++) {
-               if (seq_range_exists(uids_filter, src->uid)) {
-                       expunge = array_append_space(expunges);
-                       expunge->uid = src->uid;
-                       memcpy(expunge->guid_128, src->guid_128,
-                              sizeof(expunge->guid_128));
-               }
+               if (src->uid >= min_uid)
+                       seq_range_array_add(expunged_uids, 0, src->uid);
        }
 }
 
-bool index_storage_get_expunges(struct mailbox *box, uint64_t prev_modseq,
-                               const ARRAY_TYPE(seq_range) *uids_filter,
-                               ARRAY_TYPE(mailbox_expunge_rec) *expunges)
+static int
+index_storage_get_expunges_init(struct mailbox *box, uint64_t prev_modseq,
+                               struct mail_transaction_log_view **log_view_r,
+                               uint32_t *tail_seq_r)
 {
        struct mail_transaction_log_view *log_view;
-       const struct mail_transaction_header *thdr;
-       const void *tdata;
        uint32_t log_seq, tail_seq = 0;
        uoff_t log_offset;
        bool reset;
@@ -96,7 +85,7 @@ bool index_storage_get_expunges(struct mailbox *box, uint64_t prev_modseq,
            (log_seq == box->view->log_file_head_seq &&
             log_offset >= box->view->log_file_head_offset)) {
                /* we haven't seen this high expunges at all */
-               return TRUE;
+               return 1;
        }
 
        log_view = mail_transaction_log_view_open(box->index->log);
@@ -115,9 +104,84 @@ bool index_storage_get_expunges(struct mailbox *box, uint64_t prev_modseq,
        }
        if (ret <= 0) {
                mail_transaction_log_view_close(&log_view);
-               return FALSE;
+               return -1;
+       }
+
+       *log_view_r = log_view;
+       *tail_seq_r = tail_seq;
+       return 0;
+}
+
+static void
+index_storage_get_expunged_guids(struct mail_transaction_log_view *log_view,
+                                ARRAY_TYPE(seq_range) *expunged_uids,
+                                ARRAY_TYPE(mailbox_expunge_rec) *expunges)
+{
+       const struct mail_transaction_header *thdr;
+       const void *tdata;
+       const struct mail_transaction_expunge_guid *rec, *end;
+       struct mailbox_expunge_rec *expunge;
+       struct seq_range_iter iter;
+       unsigned int n;
+       uint32_t uid;
+
+       while (mail_transaction_log_view_next(log_view, &thdr, &tdata) > 0) {
+               if ((thdr->type & MAIL_TRANSACTION_TYPE_MASK) !=
+                   MAIL_TRANSACTION_EXPUNGE_GUID)
+                       continue;
+
+               rec = tdata;
+               end = rec + thdr->size / sizeof(*rec);
+               for (; rec != end; rec++) {
+                       if (!seq_range_exists(expunged_uids, rec->uid))
+                               continue;
+                       seq_range_array_remove(expunged_uids, rec->uid);
+
+                       expunge = array_append_space(expunges);
+                       expunge->uid = rec->uid;
+                       memcpy(expunge->guid_128, rec->guid_128,
+                              sizeof(expunge->guid_128));
+               }
+       }
+
+       /* everything left in expunged_uids didn't get a GUID */
+       seq_range_array_iter_init(&iter, expunged_uids); n = 0;
+       while (seq_range_array_iter_nth(&iter, n++, &uid)) {
+               expunge = array_append_space(expunges);
+               expunge->uid = uid;
        }
+}
+
+bool index_storage_get_expunges(struct mailbox *box, uint64_t prev_modseq,
+                               const ARRAY_TYPE(seq_range) *uids_filter,
+                               ARRAY_TYPE(seq_range) *expunged_uids,
+                               ARRAY_TYPE(mailbox_expunge_rec) *expunges)
+{
+       struct mail_transaction_log_view *log_view;
+       ARRAY_TYPE(seq_range) tmp_expunged_uids = ARRAY_INIT;
+       const struct mail_transaction_header *thdr;
+       const struct seq_range *range;
+       const void *tdata;
+       uint32_t min_uid, tail_seq;
+       int ret;
+
+       i_assert(array_count(uids_filter) > 0);
+       i_assert(expunged_uids == NULL || expunges == NULL);
 
+       ret = index_storage_get_expunges_init(box, prev_modseq,
+                                             &log_view, &tail_seq);
+       if (ret != 0)
+               return ret > 0;
+
+       range = array_idx(uids_filter, 0);
+       min_uid = range->seq1;
+
+       /* first get UIDs of all actual expunges */
+       if (expunged_uids == NULL) {
+               i_array_init(&tmp_expunged_uids, 64);
+               expunged_uids = &tmp_expunged_uids;
+       }
+       mail_transaction_log_view_mark(log_view);
        while ((ret = mail_transaction_log_view_next(log_view,
                                                     &thdr, &tdata)) > 0) {
                if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
@@ -126,14 +190,24 @@ bool index_storage_get_expunges(struct mailbox *box, uint64_t prev_modseq,
                }
                switch (thdr->type & MAIL_TRANSACTION_TYPE_MASK) {
                case MAIL_TRANSACTION_EXPUNGE:
-                       add_expunges(expunges, tdata, thdr->size, uids_filter);
+                       add_expunges(expunged_uids, min_uid, tdata, thdr->size);
                        break;
                case MAIL_TRANSACTION_EXPUNGE_GUID:
-                       add_guid_expunges(expunges, tdata, thdr->size,
-                                         uids_filter);
+                       add_guid_expunges(expunged_uids, min_uid,
+                                         tdata, thdr->size);
                        break;
                }
        }
+       mail_transaction_log_view_rewind(log_view);
+
+       /* drop UIDs that don't match the filter */
+       seq_range_array_intersect(expunged_uids, uids_filter);
+
+       if (expunges != NULL) {
+               index_storage_get_expunged_guids(log_view, expunged_uids,
+                                                expunges);
+               array_free(&tmp_expunged_uids);
+       }
 
        mail_transaction_log_view_close(&log_view);
        return ret < 0 || tail_seq != 0 ? FALSE : TRUE;

diff --git a/src/lib-storage/index/index-storage.h b/src/lib-storage/index/index-storage.h
index e548f617831f291afb0af4af89fd1dbfb7c263ee..be538f5e48cb612213bd1319ae7fb7773425a4ba 100644 (file)
--- a/src/lib-storage/index/index-storage.h
+++ b/src/lib-storage/index/index-storage.h
@@ -137,6 +137,7 @@ void index_storage_get_uid_range(struct mailbox *box,
                                 ARRAY_TYPE(seq_range) *uids);
 bool index_storage_get_expunges(struct mailbox *box, uint64_t prev_modseq,
                                const ARRAY_TYPE(seq_range) *uids_filter,
+                               ARRAY_TYPE(seq_range) *expunged_uids,
                                ARRAY_TYPE(mailbox_expunge_rec) *expunges);
 
 struct mailbox_header_lookup_ctx *

diff --git a/src/lib-storage/index/test-index-fetch.c b/src/lib-storage/index/test-index-fetch.c
index c88f414f890f7c6509660d79eaa5ead610e8bc74..060a5e9309c8be82ee693dd1cbd94382c3637880 100644 (file)
--- a/src/lib-storage/index/test-index-fetch.c
+++ b/src/lib-storage/index/test-index-fetch.c
@@ -85,6 +85,15 @@ int mail_transaction_log_view_next(struct mail_transaction_log_view *view ATTR_U
        return 1;
 }
 
+void mail_transaction_log_view_mark(struct mail_transaction_log_view *view ATTR_UNUSED)
+{
+}
+
+void mail_transaction_log_view_rewind(struct mail_transaction_log_view *view ATTR_UNUSED)
+{
+       expunge_idx = 0;
+}
+
 static void test_index_storage_get_expunges(void)
 {
        struct mailbox *box;
@@ -115,7 +124,7 @@ static void test_index_storage_get_expunges(void)
        modseq = 98ULL << 32;
        for (i = 0; i < 2; i++) {
                test_assert(index_storage_get_expunges(box, modseq, &uids_filter,
-                                                      &expunges) == i);
+                                                      NULL, &expunges) == i);
 
                exp = array_get(&expunges, &count);
                test_assert(count == 5);

diff --git a/src/lib-storage/mail-storage-private.h b/src/lib-storage/mail-storage-private.h
index b5b46f25f8b3814c5fc9deb7ab3907dab7c10dca..f2d7babaab24c45a2ffd6932cac4928b6213bbbf 100644 (file)
--- a/src/lib-storage/mail-storage-private.h
+++ b/src/lib-storage/mail-storage-private.h
@@ -173,6 +173,7 @@ struct mailbox_vfuncs {
                              ARRAY_TYPE(seq_range) *uids);
        bool (*get_expunges)(struct mailbox *box, uint64_t prev_modseq,
                             const ARRAY_TYPE(seq_range) *uids_filter,
+                            ARRAY_TYPE(seq_range) *expunged_uids,
                             ARRAY_TYPE(mailbox_expunge_rec) *expunges);
        bool (*get_virtual_uid)(struct mailbox *box,
                                const char *backend_mailbox,

diff --git a/src/lib-storage/mail-storage.c b/src/lib-storage/mail-storage.c
index b6a460972b2fa762cfbf95ea781657313667eba3..c7e7773604d54b1e35fde62235fd12af8c62f244 100644 (file)
--- a/src/lib-storage/mail-storage.c
+++ b/src/lib-storage/mail-storage.c
@@ -1079,7 +1079,15 @@ bool mailbox_get_expunges(struct mailbox *box, uint64_t prev_modseq,
                          ARRAY_TYPE(mailbox_expunge_rec) *expunges)
 {
        return box->v.get_expunges(box, prev_modseq,
-                                  uids_filter, expunges);
+                                  uids_filter, NULL, expunges);
+}
+
+bool mailbox_get_expunged_uids(struct mailbox *box, uint64_t prev_modseq,
+                              const ARRAY_TYPE(seq_range) *uids_filter,
+                              ARRAY_TYPE(seq_range) *expunged_uids)
+{
+       return box->v.get_expunges(box, prev_modseq,
+                                  uids_filter, expunged_uids, NULL);
 }
 
 bool mailbox_get_virtual_uid(struct mailbox *box, const char *backend_mailbox,

diff --git a/src/lib-storage/mail-storage.h b/src/lib-storage/mail-storage.h
index 40412d7e57b71fc8d8370fd8d01fe93ffc79dd5f..3b383fe99a45dcec22ea5bb405b0c69f7e51df70 100644 (file)
--- a/src/lib-storage/mail-storage.h
+++ b/src/lib-storage/mail-storage.h
@@ -464,6 +464,11 @@ void mailbox_get_uid_range(struct mailbox *box,
 bool mailbox_get_expunges(struct mailbox *box, uint64_t prev_modseq,
                          const ARRAY_TYPE(seq_range) *uids_filter,
                          ARRAY_TYPE(mailbox_expunge_rec) *expunges);
+/* Same as mailbox_get_expunges(), but return only list of UIDs. Not caring
+   about GUIDs is slightly faster. */
+bool mailbox_get_expunged_uids(struct mailbox *box, uint64_t prev_modseq,
+                              const ARRAY_TYPE(seq_range) *uids_filter,
+                              ARRAY_TYPE(seq_range) *expunged_uids);
 /* If box is a virtual mailbox, look up UID for the given backend message.
    Returns TRUE if found, FALSE if not. */
 bool mailbox_get_virtual_uid(struct mailbox *box, const char *backend_mailbox,

diff --git a/src/lib-storage/test-mailbox.c b/src/lib-storage/test-mailbox.c
index 3850a0b6361526acff63c5ddf2d0208cc962d0ae..9b031631fb476cba7237af6f6f070d71ae8f2beb 100644 (file)
--- a/src/lib-storage/test-mailbox.c
+++ b/src/lib-storage/test-mailbox.c
@@ -207,6 +207,7 @@ static bool
 test_mailbox_get_expunged_uids(struct mailbox *box ATTR_UNUSED,
                               uint64_t prev_modseq ATTR_UNUSED,
                               const ARRAY_TYPE(seq_range) *uids_filter ATTR_UNUSED,
+                              ARRAY_TYPE(seq_range) *expunged_uids ATTR_UNUSED,
                               ARRAY_TYPE(mailbox_expunge_rec) *expunges ATTR_UNUSED)
 {
        return FALSE;