append.c: use internaldate of existing message when appending a dupli…

…cate

cassandane/Cassandane/Cyrus/Append.pm

@@ -0,0 +1,88 @@
+#!/usr/bin/perl
+#
+#  Copyright (c) 2011-2023 FastMail Pty Ltd. All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#
+#  2. Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#
+#  3. The name "Fastmail Pty Ltd" must not be used to
+#     endorse or promote products derived from this software without
+#     prior written permission. For permission or any legal
+#     details, please contact
+#      FastMail Pty Ltd
+#      PO Box 234
+#      Collins St West 8007
+#      Victoria
+#      Australia
+#
+#  4. Redistributions of any form whatsoever must retain the following
+#     acknowledgment:
+#     "This product includes software developed by Fastmail Pty. Ltd."
+#
+#  FASTMAIL PTY LTD DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+#  INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY  AND FITNESS, IN NO
+#  EVENT SHALL OPERA SOFTWARE AUSTRALIA BE LIABLE FOR ANY SPECIAL, INDIRECT
+#  OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
+#  USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+#  TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+#  OF THIS SOFTWARE.
+#
+
+package Cassandane::Cyrus::Append;
+use strict;
+use warnings;
+use DateTime;
+use JSON;
+use JSON::XS;
+use Mail::JMAPTalk 0.13;
+use Data::Dumper;
+use Storable 'dclone';
+use File::Basename;
+use IO::File;
+
+use lib '.';
+use base qw(Cassandane::Cyrus::TestCase);
+use Cassandane::Util::Log;
+
+use charnames ':full';
+
+sub new
+{
+    my ($class, @args) = @_;
+
+    my $config = Cassandane::Config->default()->clone();
+
+    $config->set(conversations => 'yes');
+
+    my $self = $class->SUPER::new({
+        config => $config,
+        services => [ 'imap' ]
+    }, @args);
+
+    return $self;
+}
+
+sub set_up
+{
+    my ($self) = @_;
+    $self->SUPER::set_up();
+}
+
+sub tear_down
+{
+    my ($self) = @_;
+    $self->SUPER::tear_down();
+}
+
+use Cassandane::Tiny::Loader 'tiny-tests/Append';
+
+1;

cassandane/tiny-tests/Append/append_duplicate_diff_internaldate

@@ -0,0 +1,43 @@
+#!perl
+use Cassandane::Tiny;
+
+sub test_append_duplicate_diff_internaldate
+{
+    my ($self) = @_;
+
+    my $imaptalk = $self->{store}->get_client();
+
+    xlog $self, "Append duplicate messages with different internaldates";
+    my $mimeMessage = <<'EOF';
+From: <from@local>
+To: to@local
+Subject: test
+Date: Mon, 13 Apr 2020 15:34:03 +0200
+MIME-Version: 1.0
+Content-Type: text/plain
+
+test
+EOF
+    $mimeMessage =~ s/\r?\n/\r\n/gs;
+
+
+    $imaptalk->append('INBOX', $mimeMessage) || die $@;
+    $imaptalk->append('INBOX',
+                      '17-Aug-2023 15:13:54 +0200', $mimeMessage) || die $@;
+
+    $imaptalk->create('foo');
+    $imaptalk->append('foo',
+                      '26-Jan-2025 15:13:54 -0500', $mimeMessage) || die $@;
+
+    xlog $self, "Verify that all messages have the same internaldates";
+    $imaptalk->examine('INBOX');
+    my $res = $imaptalk->fetch('1:*', 'INTERNALDATE');
+    my $internaldate = $res->{1}->{internaldate};
+    $self->assert_str_equals($internaldate, $res->{2}->{internaldate});
+
+    $imaptalk->examine('foo');
+    $res = $imaptalk->fetch('1:*', 'INTERNALDATE');
+    $self->assert_str_equals($internaldate, $res->{1}->{internaldate});
+}
+
+1;

imap/append.c

@@ -886,20 +886,41 @@ static int append_apply_flags(struct appendstate *as,
 struct findstage_cb_rock {
     const char *partition;
     const char *guid;
+    struct timespec *internaldate;
     char *fname;
+    int nolink;
 };
 
 static int findstage_cb(const conv_guidrec_t *rec, void *vrock)
 {
     struct findstage_cb_rock *rock = vrock;
     mbentry_t *mbentry = NULL;
+    int r, ret = 0;
 
     if (rec->part) return 0;
+
+    if (rock->internaldate &&
+        !(rec->internal_flags & FLAG_INTERNAL_EXPUNGED)) {
+        r = conv_guidrec_mbentry(rec, &mbentry);
+        if (r) return 0;
+
+        if (mbtype_isa(mbentry->mbtype) == MBTYPE_EMAIL) {
+            // found a non-expunged duplicate email; use its internaldate
+            TIMESPEC_FROM_NANOSEC(rock->internaldate, rec->internaldate);
+            if (rock->nolink) {
+                ret = CYRUSDB_DONE;
+                goto done;
+            }
+        }
+    }
+
     // no point copying from archive, spool is on data
-    if (rec->internal_flags & FLAG_INTERNAL_ARCHIVED) return 0;
+    if (rec->internal_flags & FLAG_INTERNAL_ARCHIVED) goto done;
 
-    int r = conv_guidrec_mbentry(rec, &mbentry);
-    if (r) return 0;
+    if (!mbentry) {
+        r = conv_guidrec_mbentry(rec, &mbentry);
+        if (r) return 0;
+    }
 
     if (!strcmp(rock->partition, mbentry->partition)) {
         struct stat sbuf;
@@ -910,8 +931,10 @@ static int findstage_cb(const conv_guidrec_t *rec, void *vrock)
             if (file) {
                 struct body *body = NULL;
                 r = message_parse_file(file, NULL, NULL, &body, msgpath);
-                if (!r && !strcmp(rock->guid, message_guid_encode(&body->guid)))
+                if (!r && !strcmp(rock->guid, message_guid_encode(&body->guid))) {
                     rock->fname = xstrdup(msgpath);
+                    ret = CYRUSDB_DONE;
+                }
                 if (body) {
                     message_free_body(body);
                     free(body);
@@ -921,9 +944,10 @@ static int findstage_cb(const conv_guidrec_t *rec, void *vrock)
         }
     }
 
+ done:
     mboxlist_entry_free(&mbentry);
 
-    return rock->fname ? CYRUSDB_DONE : 0;
+    return ret;
 }
 
 /*
@@ -974,30 +998,35 @@ EXPORTED int append_fromstage_full(struct appendstate *as, struct body **body,
     mboxlist_findstage(mailbox_name(mailbox), stagefile, sizeof(stagefile));
     strlcat(stagefile, stage->fname, sizeof(stagefile));
 
-    if (!nolink) {
+    uint32_t mbtype = mbtype_isa(mailbox_mbtype(mailbox));
+    if (!nolink || mbtype == MBTYPE_EMAIL) {
         /* attempt to find an existing message with the same guid
-           and use it as the stagefile */
+           to use its internaldate, and optionally use it as the stagefile */
         struct conversations_state *cstate = mailbox_get_cstate(mailbox);
 
         if (cstate) {
-            char *guid = xstrdup(message_guid_encode(&(*body)->guid));
-            struct findstage_cb_rock rock = { mailbox_partition(mailbox), guid, NULL };
+            char guid[2*MESSAGE_GUID_SIZE+1];
+            struct findstage_cb_rock rock = {
+                mailbox_partition(mailbox),
+                strcpy(guid, message_guid_encode(&(*body)->guid)),
+                mbtype == MBTYPE_EMAIL ? internaldate : NULL,
+                NULL/*fname*/,
+                nolink
+            };
 
             // ignore errors, it's OK for this to fail
             conversations_guid_foreach(cstate, guid, findstage_cb, &rock);
 
-            // if we found a file, remember it
+            // if we found a file, use it
             if (rock.fname) {
-                syslog(LOG_NOTICE, "found existing file %s for %s; linking", guid, rock.fname);
+                syslog(LOG_NOTICE, "found existing file %s for %s; linking",
+                       guid, rock.fname);
                 linkfile = rock.fname;
+                goto havefile;
             }
-
-            free(guid);
         }
     }
 
-    if (linkfile) goto havefile;
-
     for (i = 0 ; i < stage->parts.count ; i++) {
         /* ok, we've successfully created the file */
         if (!strcmp(stagefile, stage->parts.data[i])) {