Merge pull request #2 from jfsmig/Cyrus-OpenIO

OpenIO containers autocreation and accounts

Makefile.am

@@ -932,7 +932,8 @@ imap_libcyrus_imap_la_SOURCES = \
 	imap/append.h \
 	imap/backend.c \
 	imap/backend.h \
-        imap/blobstore_openio.c \
+	imap/objectstore_openio.c \
+	imap/objectstore.h \
 	imap/conversations.c \
 	imap/conversations.h \
 	imap/convert_code.c \

imap/blobstore_openio.c → imap/objectstore_openio.c

@@ -1,6 +1,7 @@
 /* blobstore_openio.h
  *
  * Copyright (c) 1994-2008 Carnegie Mellon University.  All rights reserved.
+ * Copyright (c) 2015 OpenIO, as a part of Cyrus
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -54,26 +55,76 @@
 #include "objectstore.h"
 
 static struct oio_sds_s *sds = NULL;
+static const char *namespace = NULL;
+static const char *account = NULL;
 
 static int openio_sds_lazy_init (void)
 {
     struct oio_error_s *err = NULL;
     int v, rc;
-    const char *namespace;
 
     if (sds) return 0;
 
-    namespace = config_getstring(IMAPOPT_OPENIO_NAMESPACE);
-    // XXX - this may not be the right error here, it should say "no namespace is set"
-    if (!namespace) {
-        syslog(LOG_ERR, "OIOSDS: no namespace is set in config, "
-                "set 'openio_namespace: <ns_name>'");
+    /* Turn the OIO logging ON/OFF */
+    const char *verbosity = config_getstring(IMAPOPT_OPENIO_VERBOSITY);
+    if (!verbosity) {
+        oio_log_to_syslog(); // let the default (warn)
+        oio_log_more();
+    } else {
+        int found = 0;
+        oio_log_to_syslog();
+        switch (*verbosity) {
+            case 'q':
+            case 'Q':
+                if (!strcasecmp(verbosity, "quiet"))
+                    oio_log_nothing ();
+                break;
+            case 't':
+            case 'T':
+                if (!found && strcasecmp(verbosity, "trace"))
+                    break;
+                found = 1;
+                oio_log_more();
+                /* FALLTHROUGH */
+            case 'd':
+            case 'D':
+                if (!found && strcasecmp(verbosity, "debug"))
+                    break;
+                found = 1;
+                oio_log_more();
+                /* FALLTHROUGH */
+            case 'i':
+            case 'I':
+                if (!found && strcasecmp(verbosity, "info"))
+                    break;
+                found = 1;
+                oio_log_more();
+                /* FALLTHROUGH */
+            case 'n':
+            case 'N':
+                if (!found && strcasecmp(verbosity, "notice"))
+                    break;
+                found = 1;
+                oio_log_more();
+                /* one more call to pass from ERR to WARN */
+                oio_log_more();
+        }
+    }
+
+    if (!namespace)
+        namespace = config_getstring(IMAPOPT_OPENIO_NAMESPACE);
+    if (!account)
+        account = config_getstring(IMAPOPT_OPENIO_ACCOUNT);
+
+    if (!namespace || !*namespace) {
+        syslog(LOG_ERR, "OIOSDS: no namespace configured");
         return ENOTSUP;
     }
+
     err = oio_sds_init (&sds, namespace);
     if (err) {
         syslog(LOG_ERR, "OIOSDS: NS init failure %s : (%d) %s",
-            namespace, oio_error_code(err), oio_error_message(err));
+                namespace, oio_error_code(err), oio_error_message(err));
         oio_error_pfree (&err);
         return ENOTSUP;
     }
@@ -114,18 +165,12 @@ static struct hc_url_s *
 mailbox_openio_name (struct mailbox *mailbox, const struct index_record *record)
 {
     struct hc_url_s *url;
-    const char *namespace;
     const char *filename = mailbox_record_blobname(mailbox, record) ;
 
-    namespace = config_getstring(IMAPOPT_OPENIO_NAMESPACE);
-    // XXX - this may not be the right error here, it should say "no namespace is set"
-    if (!namespace) {
-        syslog(LOG_ERR, "OIOSDS: no namespace is set in config, set 'openio_namespace: <ns_name>'");
-        return NULL;
-    }
-
     url = hc_url_empty ();
     hc_url_set (url, HCURL_NS, namespace);
+    if (account)
+        hc_url_set (url, HCURL_ACCOUNT, account);
     hc_url_set (url, HCURL_USER, mboxname_to_userid(mailbox->name));
     hc_url_set (url, HCURL_PATH, filename);
     return url;
@@ -135,23 +180,29 @@ int objectstore_put (struct mailbox *mailbox,
         const struct index_record *record, const char *fname)
 {
     struct oio_error_s *err = NULL;
-    struct hc_url_s *url;
+    struct hc_url_s *url = NULL;
     int rc, already_saved = 0;
 
-    url = mailbox_openio_name (mailbox, record);
-
     rc = openio_sds_lazy_init ();
     if (rc) return rc;
 
     rc = objectstore_is_filename_in_container (mailbox, record, &already_saved);
     if (rc) return rc;
+
+    url = mailbox_openio_name (mailbox, record);
     if (already_saved) {
         syslog(LOG_DEBUG, "OIOSDS: blob %s already uploaded for %u",
                 hc_url_get(url, HCURL_WHOLE), record->uid);
+        hc_url_pclean (&url);
         return 0;
     }
 
-    err = oio_sds_upload_from_file (sds, url, fname);
+    struct oio_source_s src = {
+        .autocreate = config_getswitch(IMAPOPT_OPENIO_AUTOCREATE),
+        .type = OIO_SRC_FILE,
+        .data.path = fname,
+    };
+    err = oio_sds_upload_from_source (sds, url, &src);
     if (!err) {
         syslog(LOG_INFO, "OIOSDS: blob %s uploaded for %u",
                 hc_url_get(url, HCURL_WHOLE), record->uid);
@@ -161,7 +212,7 @@ int objectstore_put (struct mailbox *mailbox,
                 hc_url_get(url, HCURL_WHOLE), record->uid,
                 oio_error_code(err), oio_error_message(err));
         oio_error_pfree(&err);
-        rc = EAGAIN;                      // XXX jfs : smarter error necessary
+        rc = EAGAIN;
     }
 
     hc_url_pclean (&url);
@@ -179,6 +230,7 @@ int objectstore_get (struct mailbox *mailbox,
     if (rc) return rc;
 
     url = mailbox_openio_name (mailbox, record);
+
     err = oio_sds_download_to_file (sds, url, fname);
     if (!err) {
         syslog(LOG_INFO, "OIOSDS: blob %s downloaded for %u",
@@ -189,7 +241,7 @@ int objectstore_get (struct mailbox *mailbox,
                 hc_url_get(url, HCURL_WHOLE), record->uid,
                 oio_error_code(err), oio_error_message(err));
         oio_error_pfree (&err);
-        rc = EAGAIN;                          //* XXX jfs : smarter error necessary, according to err->code
+        rc = EAGAIN;
     }
 
     hc_url_pclean (&url);
@@ -207,6 +259,7 @@ int objectstore_delete (struct mailbox *mailbox,
     if (rc) return rc;
 
     url = mailbox_openio_name (mailbox, record);
+
     err = oio_sds_delete (sds, url);
     if (!err) {
         syslog(LOG_INFO, "OIOSDS: blob %s deleted for %u", hc_url_get(url, HCURL_WHOLE), record->uid);
@@ -230,13 +283,14 @@ int objectstore_is_filename_in_container (struct mailbox *mailbox,
     struct hc_url_s *url;
     int rc, has;
 
-    rc = openio_sds_lazy_init ();
-    if (rc) return rc;
-
     assert (phas != NULL);
     *phas = 0;
 
+    rc = openio_sds_lazy_init ();
+    if (rc) return rc;
+
     url = mailbox_openio_name (mailbox, record);
+
     err = oio_sds_has (sds, url, &has);
     if (!err) {
         rc = 0;

lib/imapoptions

@@ -2109,20 +2109,42 @@ product version in the capabilities */
    will be confdir/zoneinfo.db */
 
 { "object_storage_enabled", 0, SWITCH }
-/* Is Object storage enabled for this server.  You also need to have 
-   archiving enabled and archivepartition for the mailbox.  
-   Only email files will be stored on object Storage 
-   archivepartition will be used to store any other files */
+/* Is Object storage enabled for this server.  You also need to have
+   archiving enabled and archivepartition for the mailbox.
+   Only email files will be stored on object Storage archive partition will be
+   used to store any other files */
 
 { "openio_namespace", NULL, STRING }
-/* The OpenIO namespace used to store archived email messages. */
+/* The OpenIO namespace used to store archived email messages. A namespace
+   identifies the physical platform cyrus must contact. This directive is used
+   by the OpenIO's SDK to locate its platform entry point. */
+
+{ "openio_account", NULL, STRING }
+/* The OpenIO account used to account for stored emails. Accounts are unique
+   in their namespace. They provides virtual partitions, with quotas and QoS
+   features. */
 
 { "openio_rawx_timeout", 30, INT }
 /* The OpenIO timeout to query to the RAWX services (default 30 sec). */
 
 { "openio_proxy_timeout", 5, INT }
 /* The OpenIO timeout to query to the PROXY services (default 5 sec). */
 
+{ "openio_autocreate", 0, SWITCH }
+/* Allow the OpenIO SDK to autocreate containers. Mainly destined to be turned
+   on development environments. In production, the container should have been
+   provisioned with the mailboxes. */
+
+{ "openio_verbosity", NULL, STRING }
+/* Sets the logging verbosity of the OpenIO's internal behavior. Admissible
+   values are: "warning", "notice", "info", "debug", "trace", "quiet".
+   The default verbosity is "warning". Set to "notice" for a few lines on a
+   per-client basis. Set to "info" for a few lines on a per-request basis. Set
+   to "debug" Set to "trace" to activate the underlying libcurl debug
+   output. Enabling a verbosity higher to equal than "debug" requires
+   the cyrus to be set in debug mode. The special "quiet" value disables all
+   kinds of logging at the GLib level. */
+
 /*
 .SH SEE ALSO
 .PP