mailbox.c: add BASECID to v20 index record

use a "virtual" annotation to pass this value via replication

cunit/annotate.testc

@@ -1146,7 +1146,6 @@ static void test_msg_copy(void)
     buf_printf(&path2, "%s/data/uuid/%c/%c/%s/cyrus.annotations", DBDIR,
                u2[0], u2[1], u2);
     CU_ASSERT_EQUAL(fexists(buf_cstring(&path1)), 0);
-    CU_ASSERT_EQUAL(fexists(buf_cstring(&path2)), 0);
     mailbox_close(&mailbox1);
 
     /* copy MBOXNAME1,1 -> MBOXNAME2,1 */

cunit/mailbox.testc

@@ -119,6 +119,7 @@ static void test_aligned_record_offsets(void)
      * not need to care about that.  Instead, keep this list sorted
      * alphabetically by the OFFSET_... name, for ease of maintenance.
      */
+    CU_ASSERT_EQUAL(0, OFFSET_BASECID       % alignof(r.basecid));
     CU_ASSERT_EQUAL(0, OFFSET_CACHE_CRC     % alignof(r.cache_crc));
     CU_ASSERT_EQUAL(0, OFFSET_CACHE_OFFSET  % alignof(XXX_CACHE32_TYPE));
     CU_ASSERT_EQUAL(0, OFFSET_CACHE_VERSION % alignof(XXX_CACHE32_TYPE));
@@ -243,6 +244,7 @@ static void test_unique_record_offsets(void)
         /* Keep this sorted alphabetically by the OFFSET_... name, for ease of
          * maintenance.  We'll qsort it into the order the test needs shortly.
          */
+        OFFSET(OFFSET_BASECID,          sizeof(r.basecid)),
         OFFSET(OFFSET_CACHE_CRC,        sizeof(r.cache_crc)),
         OFFSET(OFFSET_CACHE_OFFSET,     sizeof(XXX_CACHE32_TYPE)),
         OFFSET(OFFSET_CACHE_VERSION,    sizeof(XXX_CACHE32_TYPE)),

imap/annotate.c

@@ -2071,6 +2071,7 @@ static const annotate_entrydesc_t message_builtin_entries[] =
     },
     {
         /* we use 'basethrid' to support split threads */
+        /* prior to version 20, there was no storage for basethrid, so it became an annotation */
         IMAP_ANNOT_NS "basethrid",
         ATTRIB_TYPE_STRING,
         BACKEND_ONLY,

imap/mailbox.c

@@ -2125,6 +2125,7 @@ static int mailbox_buf_to_index_record(const char *buf, int version,
     record->gmtime = ntohll(*((bit64 *)(buf+OFFSET_GMTIME)));
     record->last_updated = ntohll(*((bit64 *)(buf+OFFSET_LAST_UPDATED)));
     record->savedate = ntohll(*((bit64 *)(buf+OFFSET_SAVEDATE)));
+    record->basecid = ntohll(*(bit64 *)(buf+OFFSET_BASECID));
 
     offset_cache_crc  = OFFSET_CACHE_CRC;
     offset_record_crc = OFFSET_RECORD_CRC;
@@ -2207,18 +2208,20 @@ static int _store_change(struct mailbox *mailbox, struct index_record *record, i
         free(c_env);
     }
 
-    annotate_state_t *astate = NULL;
-    int r = mailbox_get_annotate_state(mailbox, record->uid, &astate);
-    if (r) return r;
+    if (mailbox->i.minor_version < 20) {
+        annotate_state_t *astate = NULL;
+        int r = mailbox_get_annotate_state(mailbox, record->uid, &astate);
+        if (r) return r;
 
-    struct buf annotval = BUF_INITIALIZER;
-    if (record->cid && record->basecid && record->basecid != record->cid)
-        buf_printf(&annotval, CONV_FMT, record->basecid);
+        struct buf annotval = BUF_INITIALIZER;
+        if (record->cid && record->basecid && record->basecid != record->cid)
+            buf_printf(&annotval, CONV_FMT, record->basecid);
 
-    r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "basethrid", "", &annotval);
-    buf_free(&annotval);
+        r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "basethrid", "", &annotval);
+        buf_free(&annotval);
 
-    if (r) return r;
+        if (r) return r;
+    }
 
     return 0;
 }
@@ -3306,6 +3309,7 @@ static bit32 mailbox_index_record_to_buf(struct index_record *record,
     *((bit64 *)(buf+OFFSET_GMTIME)) = htonll(record->gmtime);
     *((bit64 *)(buf+OFFSET_LAST_UPDATED)) = htonll(record->last_updated);
     *((bit64 *)(buf+OFFSET_SAVEDATE)) = htonll(record->savedate);
+    *((bit64 *)(buf+OFFSET_BASECID)) = htonll(record->basecid);
 
     offset_cache_crc  = OFFSET_CACHE_CRC;
     offset_record_crc = OFFSET_RECORD_CRC;
@@ -3457,6 +3461,9 @@ static int mailbox_is_virtannot(struct mailbox *mailbox, const char *entry)
         // internaldate.nsec was introduced in v20
         if (!strcmp(entry, IMAP_ANNOT_NS "internaldate.nsec")) return 1;
 
+        // basethrid was introduced as virtual in v20
+        else if (!strcmp(entry, IMAP_ANNOT_NS "basethrid")) return 1;
+
         GCC_FALLTHROUGH
 
     case 19:
@@ -3506,6 +3513,12 @@ static uint32_t crc_virtannot(struct mailbox *mailbox,
             buf_reset(&buf);
         }
 
+        if (record->basecid) {
+            buf_printf(&buf, CONV_FMT, record->basecid);
+            crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "basethrid", "", &buf);
+            buf_reset(&buf);
+        }
+
         GCC_FALLTHROUGH
 
     case 19:
@@ -5102,8 +5115,8 @@ static int mailbox_repack_setup(struct mailbox *mailbox, int version,
         break;
     case 20:
         repack->newmailbox.i.start_offset = 160;
-        /* version 20 grew size and time fields to 64-bits */
-        repack->newmailbox.i.record_size = 136;
+        /* version 20 grew size and time fields to 64-bits, and added basecid */
+        repack->newmailbox.i.record_size = 144;
         break;
     default:
         fatal("index version not supported", EX_SOFTWARE);
@@ -5523,6 +5536,17 @@ static int mailbox_index_repack(struct mailbox *mailbox, int version)
             buf_reset(&buf);
             r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "internaldate.nsec", "", &buf);
             if (r) goto done;
+
+            /* extract BaseCID */
+            buf_reset(&buf);
+            mailbox_annotation_lookup(mailbox, record->uid, IMAP_ANNOT_NS "basethrid", "", &buf);
+            if (buf.len == 16) {
+                const char *p = buf_cstring(&buf);
+                parsehex(p, &p, 16, &copyrecord.basecid);
+            }
+            buf_reset(&buf);
+            r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "basethrid", "", &buf);
+            if (r) goto done;
         }
         if (mailbox->i.minor_version >= 20 && repack->newmailbox.i.minor_version < 20) {
             if (record->internaldate.tv_nsec != UTIME_OMIT) {
@@ -5531,6 +5555,13 @@ static int mailbox_index_repack(struct mailbox *mailbox, int version)
                 r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "internaldate.nsec", "", &buf);
                 if (r) goto done;
             }
+
+            if (record->basecid) {
+                buf_reset(&buf);
+                buf_printf(&buf, CONV_FMT, record->basecid);
+                r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "basethrid", "", &buf);
+                if (r) goto done;
+            }
         }
 
         /* read in the old cache record */
@@ -8197,6 +8228,19 @@ EXPORTED int mailbox_reconstruct(const char *name, int flags, struct mailbox **m
                 if (r) goto close;
             }
         }
+
+        if (mailbox->i.minor_version >= 20) {
+            buf_reset(&buf);
+            mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "basethrid", "", &buf);
+            if (!buf.len) mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "basethrid", NULL, &buf);
+            if (buf.len) {
+                syslog(LOG_NOTICE, "removing stale basethrid for %u", record.uid);
+                printf("removing stale basethrid for %u\n", record.uid);
+                buf_reset(&buf);
+                r = mailbox_annotation_write(mailbox, record.uid, IMAP_ANNOT_NS "basethrid", "", &buf);
+                if (r) goto close;
+            }
+        }
     }
 
     /* make sure appends will be allowed, fixing last_uid to be at least

imap/mailbox.h

@@ -164,15 +164,15 @@ struct index_record {
     uint16_t cache_version;
     struct message_guid guid;
     modseq_t modseq;
-    bit64 cid;
     modseq_t createdmodseq;
+    bit64 cid;
+    bit64 basecid;
     bit32 cache_crc;
 
     /* metadata */
     uint32_t recno;
     unsigned silentupdate:1;
     unsigned ignorelimits:1;
-    bit64 basecid;
     struct cacherecord crec;
 };
 
@@ -411,8 +411,9 @@ struct mailbox_iter;
 #define OFFSET_GMTIME         104 /* grew to 64-bit in v20 */
 #define OFFSET_LAST_UPDATED   112 /* grew to 64-bit in v20 */
 #define OFFSET_SAVEDATE       120 /* added in v15 */
-#define OFFSET_CACHE_CRC      128 /* CRC32 of cache record */
-#define OFFSET_RECORD_CRC     132
+#define OFFSET_BASECID        128 /* base conversation id, added in v20 */
+#define OFFSET_CACHE_CRC      136 /* CRC32 of cache record */
+#define OFFSET_RECORD_CRC     140
 
 #define PRE20_OFFSET_INTERNALDATE    4
 #define PRE20_OFFSET_SENTDATE        8

imap/mbexamine.c

@@ -329,7 +329,10 @@ static int do_examine(struct findall_data *data, void *rock)
                     printf(" MODSEQ:" MODSEQ_FMT, record->modseq);
 
                     if (mailbox->i.minor_version >= 13) {
-                        printf("  THRID: " CONV_FMT, record->cid);
+                        printf("  CID: " CONV_FMT, record->cid);
+
+                        if (mailbox->i.minor_version >= 20)
+                            printf("  BASECID: " CONV_FMT, record->basecid);
                     }
                 }
             }

imap/message.c

@@ -4048,6 +4048,10 @@ EXPORTED int message_update_conversations(struct conversations_state *state,
     /* mark that it's split so basecid gets saved */
     if (record->basecid != record->cid)
         record->internal_flags |= FLAG_INTERNAL_SPLITCONVERSATION;
+    else {
+        /* otherwise, we DO NOT want/expect basecid to be set */
+        record->basecid = NULLCONVERSATION;
+    }
 
 out:
     message_unref(&msg);

imap/sync_support.c

@@ -1462,6 +1462,16 @@ void encode_annotations(struct dlist *parent,
             dlist_setnum64(aa, "VALUE", record->internaldate.tv_nsec);
         }
 
+        if (record->basecid) {
+            if (!annots)
+                annots = dlist_newlist(parent, "ANNOTATIONS");
+            aa = dlist_newkvlist(annots, NULL);
+            dlist_setatom(aa, "ENTRY", IMAP_ANNOT_NS "basethrid");
+            dlist_setatom(aa, "USERID", "");
+            dlist_setnum64(aa, "MODSEQ", 0);
+            dlist_sethex64(aa, "VALUE", record->basecid);
+        }
+
         GCC_FALLTHROUGH
 
     case 19:
@@ -1598,6 +1608,8 @@ int decode_annotations(/*const*/struct dlist *annots,
                 parsehex(p, &p, 16, &record->basecid);
                 /* XXX - check on p? */
 
+                if (mailbox->i.minor_version >= 20) break;
+
                 /* "basethrid" is special, since it is written during mailbox
                  * appends and rewrites, using whatever modseq the index_record
                  * has at this moment. This might differ from the modseq we

perl/imap/Cyrus/IndexFile.pm

@@ -651,6 +651,7 @@ SKIPPED VERSION 11 - Fastmail internal only
  13: GmTime                time_t 8
  14: LastUpdated           time_t 8
  15: SaveDate              time_t 8
+ 11: BaseCID               hex    8
  16: CacheCRC              int32  4
  17: RecordCRC             int32  4
 
@@ -1307,7 +1308,7 @@ QuotaExpungedUsed     int64  8
 ChangesEpoch          int32  4
 HeaderCrc             int32  4
 EOF
-    RecordSize => 136, # defined in file too, check it!
+    RecordSize => 144, # defined in file too, check it!
     _make_fields('Record', <<EOF),
 Uid                   int32  4
 CacheOffset           int32  4
@@ -1325,6 +1326,7 @@ CreatedModseq         int64  8
 GmTime                time64 8
 LastUpdated           time64 8
 SaveDate              time64 8
+BaseCID               hex    8
 CacheCrc              int32  4
 RecordCrc             int32  4
 EOF