From edb56f3ece9ff58488c7723015093f610d7aebae Mon Sep 17 00:00:00 2001 From: Robert Stepanek Date: Thu, 23 Nov 2023 14:16:48 +0100 Subject: [PATCH 1/6] jmap_expire: remove jmap_expire from Cyrus Signed-off-by: Robert Stepanek --- .gitignore | 1 - Makefile.am | 7 - changes/next/cyr_expire_jmap | 16 - doc/examples/cyrus_conf/normal-master.conf | 5 - doc/examples/cyrus_conf/normal-replica.conf | 5 - doc/examples/cyrus_conf/normal.conf | 5 - doc/examples/cyrus_conf/small.conf | 5 - .../manpages/systemcommands/jmap_expire.rst | 138 ------ imap/jmap_expire.c | 447 ------------------ 9 files changed, 629 deletions(-) delete mode 100644 changes/next/cyr_expire_jmap delete mode 100644 docsrc/imap/reference/manpages/systemcommands/jmap_expire.rst delete mode 100644 imap/jmap_expire.c diff --git a/.gitignore b/.gitignore index 6be28392755..426e04dc7ce 100644 --- a/.gitignore +++ b/.gitignore @@ -108,7 +108,6 @@ imap/imapd imap/ipurge imap/jmap_err.c imap/jmap_err.h -imap/jmap_expire imap/lmtp_err.c imap/lmtp_err.h imap/lmtpd diff --git a/Makefile.am b/Makefile.am index 0968835116b..00b832b4dfb 100644 --- a/Makefile.am +++ b/Makefile.am @@ -314,10 +314,6 @@ endif # SERVER endif # SIEVE -if JMAP -sbin_PROGRAMS += imap/jmap_expire -endif # JMAP - EXTRA_DIST = \ COPYING \ README.md \ @@ -965,9 +961,6 @@ imap_idled_LDADD = $(LD_UTILITY_ADD) imap_calalarmd_SOURCES = imap/calalarmd.c imap/mutex_fake.c imap_calalarmd_LDADD = $(LD_SERVER_ADD) -imap_jmap_expire_SOURCES = imap/cli_fatal.c imap/jmap_expire.c imap/mutex_fake.c -imap_jmap_expire_LDADD = $(LD_UTILITY_ADD) - imap_imapd_SOURCES = \ imap/imap_proxy.c \ imap/imap_proxy.h \ diff --git a/changes/next/cyr_expire_jmap b/changes/next/cyr_expire_jmap deleted file mode 100644 index 3a594ccb739..00000000000 --- a/changes/next/cyr_expire_jmap +++ /dev/null @@ -1,16 +0,0 @@ -Description: - -Adds the jmap_expire executable to expire stale JMAP notifications. - -Config changes: - -None. - -Upgrade instructions: - -Installations using JMAP calendars should regularly run jmap_expire, -similar to how they run cyr_expire. - -GitHub issue: - -If theres a github issue number for this, put it here. diff --git a/doc/examples/cyrus_conf/normal-master.conf b/doc/examples/cyrus_conf/normal-master.conf index 3ba81dcff44..3db8c13a5ab 100644 --- a/doc/examples/cyrus_conf/normal-master.conf +++ b/doc/examples/cyrus_conf/normal-master.conf @@ -52,11 +52,6 @@ EVENTS { # this is only necessary if caching TLS sessions tlsprune cmd="tls_prune" at=0400 - - # this is only necessary if CalDAV or JMAP are enabled - # expire JMAP notifications after 7 days and - # unlink expired JMAP notifications after 1 day - jmapprune cmd="jmap_expire -E 7d -X 1d" at 0500 } DAEMON { diff --git a/doc/examples/cyrus_conf/normal-replica.conf b/doc/examples/cyrus_conf/normal-replica.conf index 00993968c86..1ea933ed252 100644 --- a/doc/examples/cyrus_conf/normal-replica.conf +++ b/doc/examples/cyrus_conf/normal-replica.conf @@ -52,11 +52,6 @@ EVENTS { # this is only necessary if caching TLS sessions tlsprune cmd="tls_prune" at=0400 - - # this is only necessary if CalDAV or JMAP are enabled - # expire JMAP notifications after 7 days and - # unlink expired JMAP notifications after 1 day - jmapprune cmd="jmap_expire -E 7d -X 1d" at 0500 } DAEMON { diff --git a/doc/examples/cyrus_conf/normal.conf b/doc/examples/cyrus_conf/normal.conf index 599367860c9..9af28cbecf4 100644 --- a/doc/examples/cyrus_conf/normal.conf +++ b/doc/examples/cyrus_conf/normal.conf @@ -47,11 +47,6 @@ EVENTS { # this is only necessary if caching TLS sessions tlsprune cmd="tls_prune" at=0400 - - # this is only necessary if CalDAV or JMAP are enabled - # expire JMAP notifications after 7 days and - # unlink expired JMAP notifications after 1 day - jmapprune cmd="jmap_expire -E 7d -X 1d" at 0500 } DAEMON { diff --git a/doc/examples/cyrus_conf/small.conf b/doc/examples/cyrus_conf/small.conf index 0fc970dc740..d3d8b1e569f 100644 --- a/doc/examples/cyrus_conf/small.conf +++ b/doc/examples/cyrus_conf/small.conf @@ -31,11 +31,6 @@ EVENTS { # this is only necessary if caching TLS sessions tlsprune cmd="tls_prune" at=0400 - - # this is only necessary if CalDAV or JMAP are enabled - # expire JMAP notifications after 7 days and - # unlink expired JMAP notifications after 1 day - jmapprune cmd="jmap_expire -E 7d -X 1d" at 0500 } DAEMON { diff --git a/docsrc/imap/reference/manpages/systemcommands/jmap_expire.rst b/docsrc/imap/reference/manpages/systemcommands/jmap_expire.rst deleted file mode 100644 index 4d515db5612..00000000000 --- a/docsrc/imap/reference/manpages/systemcommands/jmap_expire.rst +++ /dev/null @@ -1,138 +0,0 @@ -.. cyrusman:: jmap_expire(8) - -.. author: Robert Stepanek (Fastmail) - -.. _imap-reference-manpages-systemcommands-jmap_expire: - -=============== -**jmap_expire** -=============== - -Expire stale JMAP data such as calendar event notifications. - -Synopsis -======== - -.. parsed-literal:: - - **jmap_expire** [ **-C** *config-file* ] - [ **-E** *notif-expire-duration* ] - [ **-X** *notif-unlink-duration* ] - [ **-h** or **--help**] - [ **-l** or **--lock** *lock-duration*] - [ **-u** *username* ] - [ **-v** ] - -Description -=========== - -**jmap_expire** is used to run a number of regular maintenance tasks for -JMAP mailboxes and databases, specifically: - -- expires and unlinks JMAP CalendarEventNotification objects - -**jmap_expire** requires at least one of the **-E -X** optionss. - -Option arguments that denote durations may be specified using any -combination of zero or positive integers with the following units: -seconds `s`, minutes `m`, hours `h`, days `d` (a 24 hour day without leap second). -For example, the value `1d3m2s` denotes a duration of one day, three -minutes and 2 seconds. The default unit is seconds. - -Options -======= - -.. program:: jmap_expire - -.. option:: -C config-file - - |cli-dash-c-text| - -.. option:: -E duration, --notif-expire=duration - - Flags JMAP notifications in the *jmapnotificationfolder* mailbox as - expired if their creation time is older than *duration*. - - The duration must be zero or positive. Setting this flag causes the - JMAP notification to not be visible anymore to JMAP methods, but preserves - them on the file system. This is required to allow Cyrus replica to - synchronize their state. Also see the **--notif-unlink** option. - -.. option:: -X duration, --notif-unlink=duration - - Removes already previously expired JMAP notifications in the - *jmapnotificationfolder* mailbox if their last modification time - is older than *duration*. This unlinks the notification from the file - system and is not reversible. - - The duration must be zero or positive and should be higher than the - synchronization interval of any Cyrus replica. - -.. option:: -h, --help - - Show help information how to use this program. - -.. option:: -l --lock=duration - - Attempt to lock a mailbox at most *duration* seconds for each - operation (e.g expire, unlink). Without this option, each - mailbox is exclusively locked as long as it takes to complete - the requested operation. For large mailboxes, this may get other - processes stuck waiting for the mailbox. With this option, the - mailbox typically is unlocked after the given duration, but this - can not be guaranteed for mailboxes with hundreds of thousands - of entries. - - The duration must be at least one second and at most one hour. - - Any remaining work for the operation and mailbox is left until - the next execution of jmap_expire. - -.. option:: -u username, --user=username - - Only process JMAP data belonging to this user, e.g. "someone@example.com". - Multiple occurrences of this option cause **jmap_expire** to operate on - all given user names. - -.. option:: -v, --verbose - - Enable verbose output. Multiple occurrences of this option increase - the verbosity level, up to 3 times. - -Examples -======== - -.. parsed-literal:: - - **jmap_expire** **-E** *7d* **-X** *1d* - -.. - - Expire JMAP notifications for all users where the notification - was created one week ago or earlier. Expunge all notifications - that were expired until yesterday. - -.. parsed-literal:: - - **jmap_expire** **--notif-expire**=*1d* **--lock=2s** **-u** *me@example.com* - -.. - - Expire JMAP notifications in the *jmapnotificationfolder* mailbox - of user *me@example.com*. Release the mailbox lock after at most - 2 seconds and leave any remaining work for future runs of **jmap_expire**. - -History -======= - -This was introduced in Cyrus version 3.7. - -Files -===== - -/etc/imapd.conf - -See Also -======== - -:cyrusman:`imapd.conf(5)`, :cyrusman:`master(8)` diff --git a/imap/jmap_expire.c b/imap/jmap_expire.c deleted file mode 100644 index 1ecff82c674..00000000000 --- a/imap/jmap_expire.c +++ /dev/null @@ -1,447 +0,0 @@ -/* jmap_expire.c -- Program to clean up stale JMAP data - * - * Copyright (c) 1994-2017 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include - -#include -#include -#include -#include -#include - -#include "global.h" - -/* generated headers are not necessarily in current directory */ -#include "imap/imap_err.h" - -#define SECS_IN_A_MIN 60 -#define SECS_IN_AN_HR (60 * SECS_IN_A_MIN) -#define SECS_IN_A_DAY (24 * SECS_IN_AN_HR) - -#define LOCK_MINSECS 1 -#define LOCK_MAXSECS SECS_IN_AN_HR - -/* global state */ -static int verbose = 0; -static const char *progname = NULL; -static struct namespace expire_namespace; /* current namespace */ - -struct args { - const char *altconfig; - int expire_seconds; - int unlink_seconds; - int lock_seconds; - strarray_t userids; -}; - -#define VERBOSE_INFO 1 -#define VERBOSE_TRACE 2 -#define VERBOSE_DEBUG 3 - -/* verbosep - a wrapper to print if the 'verbose' option is - turned on. */ -__attribute__((format(printf, 1, 2))) -static inline void verbosep(const char *fmt, ...) -{ - va_list params; - - if (!verbose) return; - - va_start(params, fmt); - vfprintf(stderr, fmt, params); - va_end(params); - fputc('\n', stderr); -} - -struct jmapnotif_rock { - const struct args *args; - time_t expire_before; - time_t unlink_before; - unsigned lock_millis; - unsigned nexpired; -}; - -static unsigned expire_jmapnotifs_cb(struct mailbox *mailbox, - const struct index_record *record, - void *vrock) -{ - struct jmapnotif_rock *rock = vrock; - - /* we're expiring messages by sent date */ - if (record->gmtime < rock->expire_before) { - if (verbose >= VERBOSE_TRACE) { - verbosep("%s: expiring uid %d", mailbox_name(mailbox), record->uid); - } - rock->nexpired++; - return 1; - } - - return 0; -} - -static void add_millis(struct timespec *t, unsigned ms) -{ - t->tv_sec += ms / 1000L; - t->tv_nsec += (ms % 1000L) * 1000000L; - if (t->tv_nsec >= 1000000000L) { - t->tv_sec++; - t->tv_nsec %= 1000000000L; - } -} - -static void print_lock(const char *mboxname, - struct timespec start, - struct timespec until) -{ - struct timespec now; - if (clock_gettime(CLOCK_MONOTONIC, &now) == 0) { - verbosep("%s: lock info: start=" TIME_T_FMT ".%.3ld until=" TIME_T_FMT ".%03ld now=" TIME_T_FMT ".%.3ld", - mboxname, - start.tv_sec, start.tv_nsec / 1000000L, - until.tv_sec, until.tv_nsec / 1000000L, - now.tv_sec, now.tv_nsec / 1000000L); - } -} - -static int expire_jmapnotifs(const mbentry_t *mbentry, void *vrock) -{ - signals_poll(); - - if (mbentry->mbtype & MBTYPE_DELETED) - return 0; - - if (!mboxname_isjmapnotificationsmailbox(mbentry->name, mbentry->mbtype)) - return 0; - - struct jmapnotif_rock *rock = vrock; - const struct args *args = rock->args; - struct mailbox *mailbox = NULL; - unsigned lock_millis = args->lock_seconds * 1000L; - - /* First, unlink previously expired notifications */ - - if (args->unlink_seconds >= 0) { - unsigned nunlinked = 0; - struct timespec start = {0}; - struct timespec until = {0}; - - int r = mailbox_open_iwl(mbentry->name, &mailbox); - if (r) { - verbosep("%s: can not open mailbox: %s", - mbentry->name, error_message(r)); - goto done; - } - - if (verbose >= VERBOSE_TRACE) { - verbosep("%s: expunging mailbox", mbentry->name); - } - - struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, 0); - if (lock_millis) { - clock_gettime(CLOCK_MONOTONIC, &start); - until = start; - add_millis(&until, lock_millis); - mailbox_iter_timer(iter, until, 1000); - } - r = mailbox_expunge_cleanup(mailbox, iter, - rock->unlink_before, &nunlinked); - if (r) { - verbosep("%s: failed to unlink notifications %s", - mbentry->name, error_message(r)); - } - mailbox_iter_done(&iter); - - mailbox_close(&mailbox); - - if (lock_millis && verbose >= VERBOSE_TRACE) { - print_lock(mbentry->name, start, until); - } - - verbosep("%s: unlinked %u notifications", mbentry->name, nunlinked); - - libcyrus_run_delayed(); // TODO(rsto): this should support lock_millis - } - - /* Next, expire stale notifications */ - - if (args->expire_seconds >= 0) { - rock->nexpired = 0; - struct timespec start = {0}; - struct timespec until = {0}; - - int r = mailbox_open_iwl(mbentry->name, &mailbox); - if (r) { - verbosep("%s: can not open mailbox: %s", - mbentry->name, error_message(r)); - goto done; - } - - if (verbose >= VERBOSE_TRACE) { - verbosep("%s: expiring mailbox", mbentry->name); - } - - struct mailbox_iter *iter = - mailbox_iter_init(mailbox, 0, ITER_SKIP_EXPUNGED); - if (lock_millis) { - clock_gettime(CLOCK_MONOTONIC, &start); - until = start; - add_millis(&until, lock_millis); - mailbox_iter_timer(iter, until, 1000); - } - r = mailbox_expunge(mailbox, iter, expire_jmapnotifs_cb, - rock, NULL, EVENT_MESSAGE_EXPIRE); - mailbox_iter_done(&iter); - if (r) { - verbosep("%s: failed to expire notifications: %s", - mbentry->name, error_message(r)); - } - - mailbox_close(&mailbox); - - if (lock_millis && verbose >= VERBOSE_TRACE) { - print_lock(mbentry->name, start, until); - } - - verbosep("%s: expired %u notifications", mbentry->name, rock->nexpired); - } - -done: - mailbox_close(&mailbox); - return 0; -} - -static void do_jmapnotifs(const struct args *args) -{ - struct jmapnotif_rock rock = {.args = args}; - - if (args->expire_seconds >= 0) { - rock.expire_before = time(0) - args->expire_seconds; - } - - if (args->unlink_seconds >= 0) { - rock.unlink_before = time(0) - args->unlink_seconds; - } - - if (strarray_size(&args->userids)) { - const char *folder = config_getstring(IMAPOPT_JMAPNOTIFICATIONFOLDER); - if (!folder) { - verbosep("no jmapnotificationfolder config found in imapd.conf"); - return; - } - - for (int i = 0; i < strarray_size(&args->userids); i++) { - const char *userid = strarray_nth(&args->userids, i); - mbname_t *mbname = mbname_from_userid(userid); - mbname_push_boxes(mbname, folder); - mbentry_t *mbentry = NULL; - int r = mboxlist_lookup_allow_all(mbname_intname(mbname), &mbentry, NULL); - if (!r) { - expire_jmapnotifs(mbentry, &rock); - mboxlist_entry_free(&mbentry); - } - else if (r == IMAP_MAILBOX_NONEXISTENT) { - if (verbose >= VERBOSE_TRACE) { - verbosep("%s: ignoring inexistent mailbox", - mbname_intname(mbname)); - } - } - else { - verbosep("%s: can not open mailbox: %s", - mbname_intname(mbname), error_message(r)); - } - mbname_free(&mbname); - } - } - else { - mboxlist_allmbox(NULL, expire_jmapnotifs, &rock, MBOXTREE_SKIP_ROOT); - } -} - -static void usage(void) -{ - fprintf(stderr, "Usage: %s [OPTIONS]\n", progname); - fprintf(stderr, "Expire stale JMAP data\n"); - fprintf(stderr, "\n"); - fprintf(stderr, - "Mandatory arguments (at least one required):\n" - "-E --notif-expire= expire notifications older than duration\n" - "-X --notif-unlink= unlink notifications older than duration\n" - "\n" - "Optional arguments:\n" - "-C use instead of config from imapd.conf\n" - "-l, --lock= lock mailboxes at most duration seconds\n" - "-u --user= process userid\n" - "-v, --verbose enable verbose output\n"); - fprintf(stderr, "\n"); - - exit(EX_USAGE); -} - -static int parse_args(int argc, char *argv[], struct args *args) -{ - int opt; - - /* keep this in alphabetical order */ - static const char *const short_options = "C:E:X:l:hu:v"; - - static const struct option long_options[] = { - /* n.b. no long option for -C */ - {"notif-expire", required_argument, NULL, 'E'}, - {"notif-unlink", required_argument, NULL, 'X'}, - {"lock", required_argument, NULL, 'l'}, - {"user", required_argument, NULL, 'u'}, - {"verbose", no_argument, NULL, 'v'}, - {0, 0, 0, 0}, - }; - - struct buf buf = BUF_INITIALIZER; - int dur; - - while (-1 != - (opt = getopt_long(argc, argv, short_options, long_options, NULL))) { - switch (opt) { - case 'C': - args->altconfig = optarg; - break; - - case 'E': - if (config_parseduration(optarg, 's', &dur) < 0 || dur < 0) - usage(); - args->expire_seconds = dur; - break; - - case 'X': - if (config_parseduration(optarg, 's', &dur) < 0 || dur < 0) - usage(); - args->unlink_seconds = dur; - break; - - case 'l': - if (config_parseduration(optarg, 's', &dur) < 0 || dur < 0) - usage(); - if (dur < LOCK_MINSECS || dur > LOCK_MAXSECS) - usage(); - args->lock_seconds = dur; - break; - - case 'u': - buf_setcstr(&buf, optarg); - buf_trim(&buf); - if (!buf_len(&buf)) { - fprintf(stderr, "Invalid userid: %s\n", optarg); - usage(); - } - strarray_append(&args->userids, buf_cstring(&buf)); - break; - - case 'v': - verbose++; - break; - - case 'h': - default: - usage(); - break; - } - } - - if (args->expire_seconds == -1 && - args->unlink_seconds == -1) { - fprintf(stderr, "Missing mandatory arguments\n\n"); - usage(); - return -EINVAL; - } - - buf_free(&buf); - - return 0; -} - -static void init(const char *progname, struct args *args) -{ - signals_add_handlers(0); - - cyrus_init(args->altconfig, progname, 0, 0); - global_sasl_init(1, 0, NULL); -} - -static void fini(struct args *args) -{ - strarray_fini(&args->userids); -} - -static void shut_down(int code) __attribute__((noreturn)); -static void shut_down(int code) -{ - in_shutdown = 1; - - exit(code); -} - -int main(int argc, char *argv[]) -{ - int exitcode = 0; - struct args args = {0}; - args.expire_seconds = -1; - args.unlink_seconds = -1; - - progname = basename(argv[0]); - if (parse_args(argc, argv, &args) != 0) exit(EXIT_FAILURE); - - init(progname, &args); - - /* Set namespace -- force standard (internal) */ - int r = mboxname_init_namespace(&expire_namespace, 1); - if (r) { - verbosep("can not initialize namespace: %s", error_message(r)); - exitcode = EX_CONFIG; - goto done; - } - - mboxevent_setnamespace(&expire_namespace); - - do_jmapnotifs(&args); - -done: - fini(&args); - shut_down(exitcode); -} From 0fd56adcbff4f6ba8a349440289500c1c204f72a Mon Sep 17 00:00:00 2001 From: Robert Stepanek Date: Fri, 24 Nov 2023 15:09:21 +0100 Subject: [PATCH 2/6] jmap_notif: automatically prune calendar event notifications Mark all JMAP CalendarEventNotification objects but the last 200 items as expunged, each time a new one is created. The maximum count can be set in the jmap_max_calendareventnotifs config option. Signed-off-by: Robert Stepanek --- cassandane/Cassandane/Cyrus/TestCase.pm | 5 + .../calendareventnotification-prune | 128 ++++++++++++++++++ changes/next/prune_calendareventnotifs | 15 ++ imap/jmap_notif.c | 45 +++++- lib/imapoptions | 6 + 5 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 cassandane/tiny-tests/JMAPCalendars/calendareventnotification-prune create mode 100644 changes/next/prune_calendareventnotifs diff --git a/cassandane/Cassandane/Cyrus/TestCase.pm b/cassandane/Cassandane/Cyrus/TestCase.pm index 18d6813b3f4..8a3b9419fe1 100644 --- a/cassandane/Cassandane/Cyrus/TestCase.pm +++ b/cassandane/Cassandane/Cyrus/TestCase.pm @@ -458,6 +458,11 @@ magic(NoCheckSyslog => sub { my $self = shift; $self->{no_check_syslog} = 1; }); +magic(JmapMaxCalendarEventNotifs => sub { + my $conf = shift; + # set to some small number + $conf->config_set('jmap_max_calendareventnotifs' => 10); +}); # Run any magic handlers indicated by the test name or attributes sub _run_magic diff --git a/cassandane/tiny-tests/JMAPCalendars/calendareventnotification-prune b/cassandane/tiny-tests/JMAPCalendars/calendareventnotification-prune new file mode 100644 index 00000000000..80317d32769 --- /dev/null +++ b/cassandane/tiny-tests/JMAPCalendars/calendareventnotification-prune @@ -0,0 +1,128 @@ +#!perl +use Cassandane::Tiny; + +sub test_calendareventnotification_prune + :min_version_3_9 :needs_component_jmap :JmapMaxCalendarEventNotifs +{ + my ($self) = @_; + my $jmap = $self->{jmap}; + + my $jmap_max_calendareventnotifs = $self->{instance}->{ + config}->get('jmap_max_calendareventnotifs'); + $self->assert_not_null($jmap_max_calendareventnotifs); + + my ($manJmap) = $self->create_user('manifold'); + $manJmap->DefaultUsing([ + 'urn:ietf:params:jmap:core', + 'urn:ietf:params:jmap:calendars', + 'urn:ietf:params:jmap:principals', + 'https://cyrusimap.org/ns/jmap/calendars', + ]); + + xlog $self, "Share calendar"; + my $res = $jmap->CallMethods([ + ['Calendar/set', { + update => { + Default => { + shareWith => { + manifold => { + mayReadFreeBusy => JSON::true, + mayReadItems => JSON::true, + mayUpdatePrivate => JSON::true, + mayWriteOwn => JSON::true, + mayAdmin => JSON::false + }, + }, + }, + }, + }, 'R1'], + ]); + $self->assert(exists $res->[0][1]{updated}{Default}); + + xlog $self, "Create event notification"; + my $res = $jmap->CallMethods([ + ['CalendarEvent/set', { + create => { + event1 => { + title => 'event1', + calendarIds => { + Default => JSON::true, + }, + start => '2011-01-01T04:05:06', + duration => 'PT1H', + }, + }, + }, 'R1'], + ]); + $self->assert_not_null($res->[0][1]{created}{event1}); + + xlog $self, "Get event notification"; + $res = $manJmap->CallMethods([ + ['CalendarEventNotification/get', { + accountId => 'cassandane', + }, 'R1'], + ]); + $self->assert_num_equals(1, scalar @{$res->[0][1]{list}}); + my $notif1Id = $res->[0][1]{list}[0]{id}; + $self->assert_not_null($notif1Id); + + xlog $self, "Create maximum count of allowed notifications"; + for my $i (2 .. $jmap_max_calendareventnotifs) { + my $res = $jmap->CallMethods([ + ['CalendarEvent/set', { + create => { + "event$i" => { + title => "event$i", + calendarIds => { + Default => JSON::true, + }, + start => '2011-01-01T04:05:06', + duration => 'PT1H', + }, + }, + }, 'R1'], + ]); + $self->assert_not_null($res->[0][1]{created}{"event$i"}); + } + + xlog $self, "Get event notifications"; + $res = $manJmap->CallMethods([ + ['CalendarEventNotification/get', { + accountId => 'cassandane', + properties => ['id'], + }, 'R1'], + ]); + $self->assert_num_equals($jmap_max_calendareventnotifs, scalar @{$res->[0][1]{list}}); + + xlog $self, "Assert first event notification exists"; + $self->assert_equals(1, scalar grep { $_->{id} eq $notif1Id } @{$res->[0][1]{list}}); + + xlog $self, "Create one more event notification"; + my $res = $jmap->CallMethods([ + ['CalendarEvent/set', { + create => { + eventX => { + title => 'eventX', + calendarIds => { + Default => JSON::true, + }, + start => '2011-01-01T04:05:06', + duration => 'PT1H', + }, + }, + }, 'R1'], + ]); + $self->assert_not_null($res->[0][1]{created}{eventX}); + + xlog $self, "Get event notifications"; + $res = $manJmap->CallMethods([ + ['CalendarEventNotification/get', { + accountId => 'cassandane', + properties => ['id'], + }, 'R1'], + ]); + $self->assert_num_equals($jmap_max_calendareventnotifs, scalar @{$res->[0][1]{list}}); + + xlog $self, "Assert first event notification does not exist"; + $self->assert_equals(0, scalar grep { $_->{id} eq $notif1Id } @{$res->[0][1]{list}}); +} diff --git a/changes/next/prune_calendareventnotifs b/changes/next/prune_calendareventnotifs new file mode 100644 index 00000000000..9ec8d6d7ca4 --- /dev/null +++ b/changes/next/prune_calendareventnotifs @@ -0,0 +1,15 @@ +Description: + +Prune JMAP CalendarEventNotification objects each time a new one is created. + + +Config changes: + +jmap_max_calendareventnotifs + + +Upgrade instructions: + +The default maximum count of CalendarEventNotifications is set to 200 +per account. Installations that need any other count or want to not +prune notifications must update the jmap_max_calendareventnotifs config. diff --git a/imap/jmap_notif.c b/imap/jmap_notif.c index 324c78dadf1..2ae92740113 100644 --- a/imap/jmap_notif.c +++ b/imap/jmap_notif.c @@ -114,6 +114,27 @@ HIDDEN char *jmap_caleventnotif_format_fromheader(const char *userid) return notfrom; } +static void prune_notifications(struct mailbox *notifmbox, + size_t notifications_max) +{ + struct mailbox_iter *iter = mailbox_iter_init(notifmbox, 0, + ITER_STEP_BACKWARD|ITER_SKIP_UNLINKED| + ITER_SKIP_EXPUNGED|ITER_SKIP_DELETED); + + // Remove all but the last notifications_max messages. + size_t n_notifications = 0; + message_t *msg; + while ((msg = (message_t *) mailbox_iter_step(iter))) { + if (++n_notifications >= notifications_max) { + struct index_record record = *msg_record(msg); + record.internal_flags |= FLAG_INTERNAL_EXPUNGED; + mailbox_rewrite_index_record(notifmbox, &record); + } + } + + mailbox_iter_done(&iter); +} + static int append_eventnotif(const char *from, const char *authuserid, const struct auth_state *authstate, @@ -128,12 +149,31 @@ static int append_eventnotif(const char *from, struct buf buf = BUF_INITIALIZER; const char *type = json_string_value(json_object_get(jnotif, "type")); const char *ical_uid = json_string_value(json_object_get(jnotif, "calendarEventId")); + int notifications_max = config_getint(IMAPOPT_JMAP_MAX_CALENDAREVENTNOTIFS); + if (notifications_max < 0) notifications_max = 0; + // Prune notifications each time we create a new one. + if (notifications_max) { + prune_notifications(notifmbox, (size_t) notifications_max); + } + + // Expunge all former notifications for destroyed events. if (!strcmp(type, "destroyed")) { - /* Expunge all former event notifications for this UID */ - struct mailbox_iter *iter = mailbox_iter_init(notifmbox, 0, 0); + struct mailbox_iter *iter = mailbox_iter_init(notifmbox, 0, + ITER_STEP_BACKWARD|ITER_SKIP_UNLINKED| + ITER_SKIP_EXPUNGED|ITER_SKIP_DELETED); + size_t n_notifications = 0; + message_t *msg; while ((msg = (message_t *) mailbox_iter_step(iter))) { + + // Notifications exceeding notifications_max must + // have been pruned already, no need to check. + if (notifications_max && + (++n_notifications >= (size_t) notifications_max)) { + break; + } + buf_reset(&buf); if (message_get_subject(msg, &buf) || strcmp(JMAP_NOTIF_CALENDAREVENT, buf_cstring(&buf))) { @@ -165,6 +205,7 @@ static int append_eventnotif(const char *from, } buf_reset(&buf); + // Append new notification. FILE *fp = append_newstage(mailbox_name(notifmbox), created, strhash(ical_uid), &stage); if (!fp) { diff --git a/lib/imapoptions b/lib/imapoptions index 7d6a568ab70..61a11a8aa0f 100644 --- a/lib/imapoptions +++ b/lib/imapoptions @@ -1293,6 +1293,12 @@ Blank lines and lines beginning with ``#'' are ignored. For backward compatibility, if no unit is specified, kibibytes is assumed. */ +{ "jmap_max_calendareventnotifs", 200, INT, "UNRELEASED" } +/* The maximum count of CalendarEventNotification objects to keep per account. + Any notifications exceeding this count are expunged to make room for new + ones. Zero or any negative number disables this limit. + */ + { "jmap_max_concurrent_upload", 5, INT, "3.1.6" } /* The value to return for the maxConcurrentUpload property of the JMAP \"urn:ietf:params:jmap:core\" capabilities object. The Cyrus JMAP From 23ca1f4d2073bec7b6f549626d3d4b64322cab12 Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Thu, 14 Dec 2023 10:39:58 -0500 Subject: [PATCH 3/6] mailbox.c: don't try to xunlink() directories This squashes a syslog() IOERROR and allows Cass tests that rename mailbox trees to succeed --- imap/mailbox.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/imap/mailbox.c b/imap/mailbox.c index fb4d420b6dc..78b82bbdd22 100644 --- a/imap/mailbox.c +++ b/imap/mailbox.c @@ -6061,6 +6061,11 @@ static void mailbox_delete_files(const char *path) dirp = opendir(path); if (dirp) { while ((f = readdir(dirp))!=NULL) { + if (f->d_type == DT_DIR) { + /* xunlink() will fail on a directory and create syslog noise. + We rmdir() later in mailbox_delete_cleanup() anyways */ + continue; + } if (f->d_name[0] == '.' && (f->d_name[1] == '\0' || (f->d_name[1] == '.' && From c7df638b9b5bbc6b15ef843d017ac77784aa70c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Sat, 16 Dec 2023 12:32:30 +0100 Subject: [PATCH 4/6] tls_th-lock.c: compile without ssl.h --- imap/tls_th-lock.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/imap/tls_th-lock.c b/imap/tls_th-lock.c index a39d7525263..ad86b759eda 100644 --- a/imap/tls_th-lock.c +++ b/imap/tls_th-lock.c @@ -8,12 +8,10 @@ #include #include +#ifdef HAVE_SSL #include - #include "tls_th-lock.h" -#ifdef HAVE_SSL - /* * This entire interface is obsoleted by OpenSSL 1.1.0. * Keep it around for a while for backward compatibility though. From 5f5503a9b78b59c633e4cf92d186356e315a4a76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Sun, 17 Dec 2023 19:28:39 +0100 Subject: [PATCH 5/6] message_parse_received_date() avoid calling message_parse_string(hdr="") as the latter does hdr = strchr(hdr+1, '\n') and hdr+1 is not allocated. --- cunit/message.testc | 13 +++++++++++++ imap/message.c | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cunit/message.testc b/cunit/message.testc index a22966bf6a9..5925725f2c5 100644 --- a/cunit/message.testc +++ b/cunit/message.testc @@ -1391,4 +1391,17 @@ static void test_parse_bogus_charset_params(void) #undef TESTCASE } +/* + * Verifies that message_parse_received_date() does not read + * uninitialized data in the second call to message_parse_string() + */ +static void test_parse_received_semicolon(void) +{ + static const char msg[] = "Received: abc;\r\n\r\nd"; + struct body body; + memset(&body, 0x45, sizeof(body)); + CU_ASSERT_EQUAL(message_parse_mapped(msg, sizeof(msg)-1, &body, NULL), 0); + CU_ASSERT_STRING_EQUAL(body.received_date, ""); + message_free_body(&body); +} /* vim: set ft=c: */ diff --git a/imap/message.c b/imap/message.c index 97583000e8b..793ff21087d 100644 --- a/imap/message.c +++ b/imap/message.c @@ -2010,7 +2010,7 @@ static void message_parse_received_date(const char *hdr, char **hdrp) curp--; /* Didn't find ; - fill in hdrp so we don't look at next received header */ - if (curp == hdrbuf) { + if (curp == hdrbuf || curp[1] == '\0') { *hdrp = hdrbuf; return; } From e14ef0ad88303450dfea8e1883f766b9a0b564d1 Mon Sep 17 00:00:00 2001 From: Robert Stepanek Date: Mon, 18 Dec 2023 10:45:21 +0100 Subject: [PATCH 6/6] message.c: set received_date to NULL for empty date-time Signed-off-by: Robert Stepanek --- cunit/message.testc | 2 +- imap/message.c | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cunit/message.testc b/cunit/message.testc index 5925725f2c5..e7b60c1c24e 100644 --- a/cunit/message.testc +++ b/cunit/message.testc @@ -1401,7 +1401,7 @@ static void test_parse_received_semicolon(void) struct body body; memset(&body, 0x45, sizeof(body)); CU_ASSERT_EQUAL(message_parse_mapped(msg, sizeof(msg)-1, &body, NULL), 0); - CU_ASSERT_STRING_EQUAL(body.received_date, ""); + CU_ASSERT_PTR_NULL(body.received_date); message_free_body(&body); } /* vim: set ft=c: */ diff --git a/imap/message.c b/imap/message.c index 793ff21087d..832a2c58899 100644 --- a/imap/message.c +++ b/imap/message.c @@ -2010,11 +2010,18 @@ static void message_parse_received_date(const char *hdr, char **hdrp) curp--; /* Didn't find ; - fill in hdrp so we don't look at next received header */ - if (curp == hdrbuf || curp[1] == '\0') { + if (curp == hdrbuf) { *hdrp = hdrbuf; return; } + /* No date string after ; - treat as non-existent */ + if (curp[1] == '\0') { + free(hdrbuf); + *hdrp = NULL; + return; + } + /* Found it, copy out date string part */ curp++; message_parse_string(curp, hdrp);