Merge branch 'dev-2.3'

Kubernetes/Admin/Authentication/update-LE-cert.sh

@@ -8,13 +8,19 @@ debug=1
 # The user managing k8s
 user=metadig
 # k8s namespace that we are managing
-ns=metadig
+#k8sns=metadig
+k8sns=nginx-ingress
 
 # Save current LE cert modified time so we can see if certbot delivers
 # new certs
-host=`hostname -f`
-CA_DIR=/etc/letsencrypt/live/${host}
-certFilename=${CA_DIR}/cert.pem
+domain=`hostname -f`
+damainDir=$domain
+domain=api.test.dataone.org,${domain}
+CA_DIR=/etc/letsencrypt/live/${domainDir}
+# Use fullchain.pem, which includes the intermediate certificate, that will allow TLS
+# client authentication, for those clients that don't know about LE certs
+#certFilename=${CA_DIR}/cert.pem
+certFilename=${CA_DIR}/fullchain.pem
 privkeyFilename=${CA_DIR}/privkey.pem
 certModTime=`stat -c %Y ${certFilename}`
 
@@ -28,7 +34,8 @@ certModTime=`stat -c %Y ${certFilename}`
 # the IP that the certbot request will come from.
 ufw allow 80
 #sudo ufw allow from ${certbotIP} to any port 80
-/usr/bin/certbot renew > /var/log/letsencrypt/letsencrypt-renew.log 2>&1
+#/usr/bin/certbot renew -d ${domain} > /var/log/letsencrypt/letsencrypt-renew.log 2>&1
+/usr/bin/certbot renew -d ${domain} > /var/log/letsencrypt/letsencrypt-renew.log 2>&1
 # Close the port as soon as certbot is done
 ufw delete allow 80
 #sudo ufw delete allow from ${certbotIP} to any port 80
@@ -55,7 +62,8 @@ if (( $certModTimeNew > $certModTime )); then
   su ${user} -c "kubectl get secret ${k8sns}-tls-cert --namespace ${k8sns}"
   su ${user} -c "kubectl delete secret ${k8sns}-tls-cert --namespace ${k8sns}"
   #sudo kubectl create secret tls ${k8sns}-tls-cert --key ${CA_DIR}/privkey.pem --cert ${CA_DIR}/cert.pem --namespace ${k8sns}
-  su ${user} -c "kubectl create secret tls ${k8sns}-tls-cert --key ~${user}/tmp/privkey.pem --cert ~${user}/tmp/cert.pem --namespace ${k8sns}"
+  #su ${user} -c "kubectl create secret tls ${k8sns}-tls-cert --key ~${user}/tmp/privkey.pem --cert ~${user}/tmp/cert.pem --namespace ${k8sns}"
+  su ${user} -c "kubectl create secret tls ${k8sns}-tls-cert --key ~${user}/tmp/privkey.pem --cert ~${user}/tmp/chain.pem --namespace ${k8sns}"
   #su metadig -c "kubectl get secret metadig-tls-cert --namespace metadig"
   rm -f ~${user}/tmp/privkey.pem ~${user}/tmp/cert.pem
 
@@ -65,4 +73,4 @@ else
   if (( $debug )); then
     echo "Let's Encrypt cert not updated by certbot, Not updating k8s with new certfile "
   fi
-fi
+fi

Kubernetes/Admin/Solr/renew-LE-cert-for-solr.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Follow these steps in order to enable SSL for solr standalone server.
+# From SO: https://stackoverflow.com/questions/41592427/letsencypt-solr-ssl-jvm
+# As i have a key for the Domain already, and Solr responds on mydomain.com:8983 all that is needed is to create a Java Key Store (jks) from the existing keys on the system
+
+# Note: Use the password "metadig" when prompted by openssl
+sudo openssl pkcs12 -export -in /etc/letsencrypt/live/docker-ucsb-4.dataone.org/fullchain.pem -inkey /etc/letsencrypt/live/docker-ucsb-4.dataone.org/privkey.pem -out pkcs.p12 -name metadig
+
+# specifing the location of the Lets-Encrypt Cert (on my system /etc/letsencrypt/live/mydomain.com/)
+# Then convert the PKCS12 key to a jks, replacing password where needed.
+
+# keytool -importkeystore -deststorepass PASSWORD_STORE -destkeypass PASSWORD_KEYPASS -destkeystore keystore.jks -srckeystore pkcs.p12 -srcstoretype PKCS12 -srcstorepass STORE_PASS -alias NAME
+
+sudo keytool -importkeystore -deststorepass metadig -destkeypass metadig -destkeystore keystore.jks -srckeystore pkcs.p12 -srcstoretype PKCS12 -srcstorepass metadig -alias metadig
+sudo cp keystore.jks /opt/solr/server/etc/solr-ssl-letsencrypt.keystore.jks 
+sudo chown solr /opt/solr/server/etc/solr-ssl-letsencrypt.keystore.jks 
+sudo chgrp solr /opt/solr/server/etc/solr-ssl-letsencrypt.keystore.jks 
+
+rm -f keystore.jks
+
+# Now that the keystore has been created, Solr must be told where it is:
+
+#* on docker-ucsb-4, the ’service solr start’ (/etc/init.d/solr) reads from /etc/default/solr.in.sh
+#    * these values are currently used
+#        * SOLR_SSL_ENABLED=true
+#        * # Uncomment to set SSL-related system properties
+#        * # Be sure to update the paths to the correct keystore for your environment
+#        * SOLR_SSL_KEY_STORE=/opt/solr/server/etc/solr-ssl-letsencrypt.keystore.jks
+#        * SOLR_SSL_KEY_STORE_PASSWORD=metadig
+#        * SOLR_SSL_KEY_STORE_TYPE=JKS
+#        * SOLR_SSL_TRUST_STORE=/opt/solr/server/etc/solr-ssl-letsencrypt.keystore.jks
+#        * SOLR_SSL_TRUST_STORE_PASSWORD=metadig
+#        * SOLR_SSL_TRUST_STORE_TYPE=JKS
+#        * #SOLR_SSL_NEED_CLIENT_AUTH=false
+#        * SOLR_SSL_WANT_CLIENT_AUTH=false
+
+
+# Now restart Solr
+sudo service solr restart

Kubernetes/Admin/metadig-engine/rabbitmq.yaml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: rabbitmq
@@ -13,16 +13,17 @@ spec:
       labels:
         app: rabbitmq
     spec:
-      serviceAccountName: metadig-serviceaccount
       containers:
       - name: rabbitmq
-        image: rabbitmq:3.7
+        # 3-management loads the RabbitMQ Administrative plugin with the
+        # most recent 3.x release.
+        image: rabbitmq:3-management
         ports:
-        - name: rabbitmq 
+        - name: rabbitmq
           containerPort: 5672
         - name: rabbitmqadmin
           containerPort: 15672
-        imagePullPolicy: IfNotPresent
+        imagePullPolicy: Always
       restartPolicy: Always
       #volumes:
       #- name: metadig-ctl-claim0
@@ -47,5 +48,4 @@ spec:
   - name: rabbitmqadmin
     port: 15672
     protocol: TCP
-    targetPort: rabbitmqadmin
-
+    targetPort: rabbitmqadmin

Kubernetes/metadig-scheduler/Dockerfile

@@ -6,14 +6,14 @@ MAINTAINER [email protected]
 # Set the working directory 
 WORKDIR /var/lib/metadig
 
-COPY log4j.properties .
+#COPY log4j.properties .
 # The most recently built jar file is copied from the maven build directory to this dir by maven, so that
 # it can be copied to the image.
 COPY metadig-engine.jar metadig-engine.jar
 
 #COPY metadig.properties /etc/metadig/metadig.properties
 #COPY taskList.csv /etc/metadig/taskList.csv
-COPY log4j.properties .
+#COPY log4j.properties .
 
 #COPY run.sh run.sh
 # The 'run.sh' script copies config files that should be available from persistent volume to the standard location where the software
@@ -23,4 +23,6 @@ COPY log4j.properties .
 #CMD java -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:+UseSerialGC -cp ./metadig-engine.jar:./solr edu.ucsb.nceas.mdqengine.scheduler.JobScheduler
 
 #CMD [ "./run.sh" ]
-CMD java -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:+UseSerialGC -cp ./metadig-engine.jar: edu.ucsb.nceas.mdqengine.scheduler.JobScheduler
+# Set classpath to include /opt/local/metadig/log4j.properties, if it exists, so that logging can be changed without
+# having to rebuild the container. Note that on k8s, this dir is mapped to the persistent volume, so will be /data/metadig/log4j.properties
+CMD java -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:+UseSerialGC -cp /opt/local/metadig/config:./metadig-engine.jar: edu.ucsb.nceas.mdqengine.scheduler.JobScheduler

Kubernetes/metadig-scorer/Dockerfile

@@ -9,7 +9,7 @@ WORKDIR /var/lib/metadig
 # This file was created from the https://github.com/NCEAS/metadig-r repo
 # and contains R functions that assist in writing R based quality checks.
 COPY metadig_0.2.0.tar.gz metadig.tar.gz
-COPY log4j.properties .
+#COPY log4j.properties .
 # The most recently built jar file is copied from the maven build directory to this dir by maven, so that
 # it can be copyied to the image.
 COPY metadig-engine.jar metadig-engine.jar
@@ -24,21 +24,16 @@ RUN mkdir -p /etc/dataone/index && touch /etc/dataone/index/d1client.properties
 
 # Add R runtime and install packges required by the quality suites
 RUN apt update
-RUN apt -y install vim
-RUN apt -y install r-base
-RUN apt -y install r-cran-httr
-RUN apt -y install r-cran-xml2
-RUN apt -y install r-cran-tidyr
-RUN apt -y install r-cran-scales
-RUN apt -y install r-cran-lubridate
-RUN apt -y install r-cran-ggplot2
-RUN apt -y install r-cran-magrittr
+RUN apt -y install vim bash
+RUN apt -y install r-base r-cran-httr r-cran-xml2 r-cran-tidyr r-cran-scales r-cran-lubridate r-cran-ggplot2 r-cran-magrittr
 # Debian stretch doesn't have a pre-cooked package for readr, so install now.
-RUN Rscript --vanilla -e 'install.packages("readr", repos="https://cran.mtu.edu/")'
+RUN Rscript --vanilla -e 'install.packages("readr", repos=c(CRAN = "http://cran.rstudio.com"))'
 RUN Rscript --vanilla -e 'install.packages("metadig.tar.gz", repos=NULL)'
 
 # Run the Scorer process
 # Note: docker --build-arg only allows one argument (one token only, multiple tokens inside quotes doesn't work, so have
 #   to specify java options directly on command line.
-CMD java -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:+UseSerialGC -cp ./metadig-engine.jar:./solr edu.ucsb.nceas.mdqengine.scorer.Scorer
+# Set classpath to include /opt/local/metadig/log4j.properties, if it exists, so that logging can be changed without
+# having to rebuild the container. Note that on k8s, this dir is mapped to the persistent volume, so will be /data/metadig/log4j.properties
+CMD java -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:+UseSerialGC -cp /opt/local/metadig/config:./metadig-engine.jar:./solr edu.ucsb.nceas.mdqengine.scorer.Scorer
 

Kubernetes/metadig-worker/Dockerfile

@@ -11,7 +11,7 @@ WORKDIR /var/lib/metadig
 # This file was created from the https://github.com/NCEAS/metadig-r repo
 # and contains R functions that assist in writing R based quality checks.
 COPY metadig_0.2.0.tar.gz metadig.tar.gz
-COPY log4j.properties .
+#COPY log4j.properties .
 # The most recently built jar file is copied from the maven build directory to this dir by maven, so that
 # it can be copyied to the image.
 COPY metadig-engine.jar metadig-engine.jar
@@ -27,13 +27,17 @@ RUN mkdir -p /etc/dataone/index && touch /etc/dataone/index/d1client.properties
 # Add R runtime and install packges required by the quality suites
 COPY r-cmds.txt r-cmds.txt
 RUN apk update
+# bash is needed by the openssl install
+RUN apk add bash
 RUN apk add g++ R R-dev R-doc libc-dev openssl-dev libxml2 libxml2-dev
-RUN echo 'options(repos = c(CRAN = "https://cran.cnr.berkeley.edu/"))' >> /usr/lib/R/etc/Rprofile.site
+RUN echo 'options(repos = c(CRAN = "http://cran.rstudio.com"))' >> /usr/lib/R/etc/Rprofile.site
 RUN Rscript --vanilla r-cmds.txt
 
 # Run the Worker process
 # Note: docker --buile-arg only allows one argument (one token only, multiple tokens inside quotes doesn't work, so have
 #   to specify java options directly on command line.
-CMD java -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:+UseSerialGC -cp ./metadig-engine.jar:./solr edu.ucsb.nceas.mdqengine.Worker
+# Set classpath to include /opt/local/metadig/log4j.properties, if it exists, so that logging can be changed without
+# having to rebuild the container. Note that on k8s, this dir is mapped to the persistent volume, so will be /data/metadig/log4j.properties
+CMD java -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:+UseSerialGC -cp /opt/local/metadig/config:./metadig-engine.jar:./solr edu.ucsb.nceas.mdqengine.Worker
 #CMD java -Xms128m -Xmx256m -Dlog4j.configuration=log4j.properties -cp ./metadig-engine.jar:./solr edu.ucsb.nceas.mdqengine.Worker