[S3] Parse AWS configuration with support for profile section

libdispatch/ds3util.c

@@ -603,13 +603,18 @@ NC_getactives3profile(NCURI* uri, const char** profilep)
     int stat = NC_NOERR;
     const char* profile = NULL;
     struct AWSprofile* ap = NULL;
+    struct NCglobalstate* gs = NC_getglobalstate();
 
-    profile = ncurifragmentlookup(uri,"aws.profile");
-    if(profile == NULL)
-        profile = NC_rclookupx(uri,"AWS.PROFILE");
+    if (uri != NULL) {
+	profile = ncurifragmentlookup(uri,"aws.profile");
+	if(profile == NULL)
+		profile = NC_rclookupx(uri,"AWS.PROFILE");
+    }
 
-    if(profile == NULL)
-        profile = NC_getglobalstate()->aws.profile;
+    if(profile == NULL && gs->aws.profile != NULL) {
+        if((stat=NC_authgets3profile(gs->aws.profile,&ap))) goto done;
+	if(ap) profile = nulldup(gs->aws.profile);
+    }
 
     if(profile == NULL) {
         if((stat=NC_authgets3profile("default",&ap))) goto done;
@@ -837,13 +842,19 @@ awsparse(const char* text, NClist* profiles)
 	if(token ==  AWS_EOF) break; /* finished */
 	if(token ==  AWS_EOL) {continue;} /* blank line */
 	if(token != LBR) {stat = NCTHROW(NC_EINVAL); goto done;}
-	/* parse [profile name] */
+	/* parse [profile name] or [name] */
         token = awslex(parser);
 	if(token != AWS_WORD) {stat = NCTHROW(NC_EINVAL); goto done;}
 	assert(profile == NULL);
 	if((profile = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile)))==NULL)
 	    {stat = NC_ENOMEM; goto done;}
 	profile->name = ncbytesextract(parser->yytext);
+	if(strncmp("profile", profile->name, sizeof("profile")) == 0 ) {
+		token =  awslex(parser);
+		if(token != AWS_WORD) {stat = NCTHROW(NC_EINVAL); goto done;}
+		nullfree(profile->name);
+		profile->name = ncbytesextract(parser->yytext);
+	}
 	profile->entries = nclistnew();
         token = awslex(parser);
 	if(token != RBR) {stat = NCTHROW(NC_EINVAL); goto done;}
@@ -881,10 +892,22 @@ fprintf(stderr,">>> parse: entry=(%s,%s)\n",entry->key,entry->value);
 	        {stat = NCTHROW(NC_EINVAL); goto done;}
 	}
 
-	/* If this profile already exists, then replace old one */
+	/* If this profile already exists, then overwrite old one */
 	for(size_t i=0;i<nclistlength(profiles);i++) {
 	    struct AWSprofile* p = (struct AWSprofile*)nclistget(profiles,i);
 	    if(strcasecmp(p->name,profile->name)==0) {
+		// Keep unique parameters from previous (incomplete!?) profile
+		for (size_t j=0;j<nclistlength(p->entries);j++){
+			struct AWSentry* old = (struct AWSentry*)nclistget(p->entries,j);
+			int add = 1;
+			for (size_t z=0;z<nclistlength(profile->entries);z++){
+				struct AWSentry* new = (struct AWSentry*)nclistget(profile->entries,z);
+				add &= (strcasecmp(old->key,new->key)!=0);
+			}
+			if(add){
+			    nclistpush(profile->entries, nclistremove(p->entries,j--));
+			}
+		}
 		nclistset(profiles,i,profile);
                 profile = NULL;
 		/* reclaim old one */

unit_test/CMakeLists.txt

@@ -37,6 +37,9 @@ IF(NETCDF_BUILD_UTILITIES)
   # SDK Test
     build_bin_test(test_s3sdk ${XGETOPTSRC})
     add_sh_test(unit_test run_s3sdk)
+  #AWS Configuration test
+    build_bin_test(aws_config)
+    add_sh_test(unit_test run_aws_config)
   ENDIF()
 ENDIF()
 

unit_test/Makefile.am

@@ -43,6 +43,8 @@ if NETCDF_ENABLE_S3_TESTALL
 check_PROGRAMS += test_s3sdk
 TESTS += run_s3sdk.sh
 endif
+check_PROGRAMS += aws_config
+TESTS += run_aws_config.sh
 endif
 
 EXTRA_DIST = CMakeLists.txt run_s3sdk.sh run_reclaim_tests.sh

unit_test/aws_config.c

@@ -0,0 +1,90 @@
+/*********************************************************************
+ *   Copyright 2018, UCAR/Unidata
+ *   See netcdf/COPYRIGHT file for copying and redistribution conditions.
+ *********************************************************************/
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include "netcdf.h"
+#include "ncrc.h"
+#include "ncpathmgr.h"
+#include "ncs3sdk.h"
+#include "ncuri.h"
+#include "nc4internal.h"
+
+NCS3INFO s3info;
+void* s3client = NULL;
+
+/* Forward */
+static void cleanup(void);
+
+#define STR(p) p?p:"NULL"
+#define CHECK(code) do {stat = check(code,__func__,__LINE__); if(stat) {goto done;}} while(0)
+
+static int
+check(int code, const char* fcn, int line)
+{
+    if(code == NC_NOERR) return code;
+    fprintf(stderr,"***FAIL: (%d) %s @ %s:%d\n",code,nc_strerror(code),fcn,line);
+    abort();
+}
+
+static void
+cleanup(void)
+{
+    if(s3client)
+        NC_s3sdkclose(s3client, &s3info, 0/*deleteit*/, NULL);
+    s3client = NULL;
+    NC_s3clear(&s3info);
+}
+
+int
+main(int argc, char** argv)
+{
+    int c = 0,stat = NC_NOERR;
+
+    /* Load RC and .aws/config */
+    CHECK(nc_initialize());
+    CHECK(NC_s3sdkinitialize());
+    NCglobalstate* gs = NC_getglobalstate();
+    //Not checking, aborts if ~/.aws/config doesn't exist
+    CHECK(NC_aws_load_credentials(gs));
+
+    // Lets ensure the active profile is loaded
+    // from the configurtion files instead of an URL
+    const char* activeprofile = NULL;
+    CHECK(NC_getactives3profile(NULL, &activeprofile));
+
+    fprintf(stderr, "Active profile:%s\n", STR(activeprofile));
+
+    // ARGV contains should contain "key[=value]" to verify
+    // if key was parsed when loading the aws config and if it's
+    // value is updated in case it's redefined on the .aws/credentials
+    for(int i = 1; i < argc; i++) {
+        const char *argkey = strtok(argv[i],"=");
+        const char *argvalue = strtok(NULL,"=");
+        const char* value = NULL;
+
+        NC_s3profilelookup(activeprofile,argkey,&value);
+        fprintf(stderr, "%s\t%s -> %s\n",value?"":"*** FAIL:", argv[i],value?value:"NOT DEFINED!");
+        if ( value == NULL 
+            || (argvalue != NULL 
+                && strncmp(value, argvalue, strlen(value)))
+        ){
+                c++;
+                stat |= NC_ERCFILE;
+        }
+    }
+
+done:
+    cleanup();
+    if(stat)
+        printf("*** FAIL: a total of %d keys were not found on the profile %s\n", c, STR(activeprofile));
+    else
+        printf("***PASS\n");
+    (void)NC_s3sdkfinalize();
+    (void)nc_finalize();
+    exit(stat?:0);
+}

unit_test/run_aws_config.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+
+if test "x$srcdir" = x ; then srcdir=`pwd`; fi 
+. ../test_common.sh
+
+set -e
+
+#CMD="valgrind --leak-check=full"
+
+isolate "testdir_ut_aws_config"
+
+THISDIR=`pwd`
+cd $ISOPATH
+
+mkdir -p $THISDIR/.aws/
+
+test_cleanup() {
+    rm -rfv $THISDIR/.aws/
+}
+trap test_cleanup EXIT
+
+cat << 'EOF' > $THISDIR/.aws/config
+[uni]
+region = somewhere-1
+endpoint_url = https://example.com/bucket/prefix/1
+key = value
+extrakey = willbepropagated
+
+[profile unidata]
+region = us-east-1
+endpoint_url = https://s3.example.domain/
+dummy_key = dummy_value
+
+[profile play]
+region = us-east-1
+endpoint_url = https://endpoint.example.com/
+EOF
+
+cat << 'EOF' > $THISDIR/.aws/credentials
+[play] 
+aws_access_key_id = DummyKeys
+aws_secret_access_key = DummySecret
+
+[uni]
+region = somewhere-2
+endpoint_url = https://example.com/bucket/prefix/2
+key = value-overwritten
+EOF
+
+echo -e "Testing loading AWS configuration in ${THISDIR}/.aws/config"
+NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=unidata ${CMD} ${execdir}/aws_config  endpoint_url region dummy_key
+echo "Status: $?"
+
+NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=play ${CMD} ${execdir}/aws_config  endpoint_url region 
+echo "Status: $?"
+
+NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=uni ${CMD} ${execdir}/aws_config  endpoint_url region key 
+echo "Status: $?"
+
+NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=uni ${CMD} ${execdir}/aws_config key=value-overwritten region=somewhere-2 endpoint_url=https://example.com/bucket/prefix/2 extrakey=willbepropagated
+echo "Status: $?"
+
+# Will use profile=no
+NC_TEST_AWS_DIR=${THISDIR} ${CMD} ${execdir}/aws_config 2>&1 | grep -q 'Active profile:no'
+echo "Status: $?"
+echo -e "Finished"
+
+exit
+