Merge pull request #1438 from arooshap/mongo-eos

Changes to the MongoDB setup.

docker/mongodb/source/copy_cron.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Print a message to indicate the script is running
+echo "copy_cron.sh is running"
+
+if [ "$ENVIRONMENT" = "prod" ]; then
+  # Copy the production cron file
+  echo "Copying mongo-prod.cron"
+  cp /data/tools/mongo-prod.cron /data/tools/mongo.cron
+elif [ "$ENVIRONMENT" = "preprod" ]; then
+  # Copy the development cron file
+  echo "Copying mongo-preprod.cron"
+  cp /data/tools/mongo-preprod.cron /data/tools/mongo.cron
+elif [ "$ENVIRONMENT" = "test" ]; then
+  # Copy the test cron file
+  echo "Copying mongo-test.cron"
+  cp /data/tools/mongo-test.cron /data/tools/mongo.cron
+else
+  echo "Unsupported environment: $ENVIRONMENT"
+  exit 1
+fi
+
+# Set up the cron job
+crontab /data/tools/mongo.cron
+
+# Add another cronjob to get kerberos token once everyday
+(crontab -l ; echo "0 0 * * * /root/run.sh") | crontab -
+

docker/mongodb/source/dockerfile

@@ -46,16 +46,20 @@ COPY --from=go-builder /data/email/email /data/tools
 
 #add kinit operation
 ADD run.sh /root/run.sh
-RUN chmod +x /root/run.sh
 
 #add cronjob
-ADD mongo.cron /data/tools/mongo.cron
+
 
 #add mongotools
 COPY /mongotools /data/tools
 
-RUN crontab /data/tools/mongo.cron
+COPY copy_cron.sh /data/tools/copy_cron.sh
+COPY mongo-prod.cron /data/tools/mongo-prod.cron
+COPY mongo-preprod.cron /data/tools/mongo-preprod.cron
+COPY mongo-test.cron /data/tools/mongo-test.cron
+
 
 ENV PATH=/data/tools:$PATH
 
-CMD ./startup-$MONGODB_ID.sh; ./run.sh
+CMD ["/bin/sh", "-c", "./run.sh; /data/tools/copy_cron.sh; ./startup-$MONGODB_ID.sh;"]
+

docker/mongodb/source/mongo-preprod.cron

@@ -0,0 +1,2 @@
+0 */12 * * * export AGE_KEY="/etc/mongodb-secret/age-key.txt" && /data/tools/mongo_manage.sh backup /etc/mongodb-secret/mongo.ini
+

docker/mongodb/source/mongo-prod.cron

@@ -0,0 +1,2 @@
+0 */4 * * * export AGE_KEY="/etc/mongodb-secret/age-key.txt" && /data/tools/mongo_manage.sh backup /etc/mongodb-secret/mongo.ini
+

docker/mongodb/source/mongo-test.cron

@@ -0,0 +1,2 @@
+0 2 * * * export AGE_KEY="/etc/mongodb-secret/age-key.txt" && /data/tools/mongo_manage.sh backup /etc/mongodb-secret/mongo.ini
+

docker/mongodb/source/mongotools/alerts.sh

@@ -5,6 +5,7 @@ msg="MongoDB backup cronjob failure"
 DATE=`date`
 host=`hostname`
 job="mongodb"
-amhost="http://cms-monitoring.cern.ch:30093"
-amtool alert add mongodb_cronjob_failure alertname="$msg" job="$job" host="$host" tag=mongodb alert=amtool service=cron --end="$expire" --annotation=summary="$msg" --annotation=date="$DATE" --alertmanager.url="$amhost" action=restart
+tag="cmsweb"
+amhost="http://cms-monitoring-ha1.cern.ch:30093"
+amtool alert add mongodb_cronjob_failure alertname="$msg" tag="$tag" job="$job" host="$host" alert=amtool service=cron --end="$expire" --annotation=summary="$msg" --annotation=date="$DATE" --alertmanager.url="$amhost" action=restart
 amhost="http://cms-monitoring-ha1.cern.ch:30093"

docker/mongodb/source/mongotools/mongo_manage.sh

@@ -38,6 +38,7 @@ init(){
       PASSWORD=`cat $CONFIG | grep PASSWORD | sed -e "s,PASSWORD=,,g"`
       BACKUP_DIR=`cat $CONFIG | grep BACKUP_DIR | sed -e "s,BACKUP_DIR=,,g"`
       RS_NAME=`cat $CONFIG | grep RS_NAME | sed -e "s,RS_NAME=,,g"`
+      DB_NAMES=$(cat "$CONFIG" | grep DB_NAMES | sed -e "s,DB_NAMES=,,g")
   else
       if [ -z "$AGE_KEY" ]; then
         echo "AGE_KEY environment is not set, please generate appropriate key file"
@@ -53,6 +54,7 @@ init(){
       PASSWORD=`age -i $AGE_KEY --decrypt -o - $CONFIG | grep PASSWORD | sed -e "s,PASSWORD=,,g"`
       BACKUP_DIR=`age -i $AGE_KEY --decrypt -o - $CONFIG | grep BACKUP_DIR | sed -e "s,BACKUP_DIR=,,g"`
       RS_NAME=`age -i $AGE_KEY --decrypt -o - $CONFIG | grep RS_NAME | sed -e "s,RS_NAME=,,g"`
+      DB_NAMES=$(cat "$CONFIG" | grep DB_NAMES | sed -e "s,DB_NAMES=,,g")
   fi
   if [ -z "$USERNAME" ]; then
       echo "Unable to locate USERNAME in $CONFIG"
@@ -90,6 +92,8 @@ init(){
           exit 1
       fi
   fi
+  # Split DB_NAMES into an array
+  IFS=' ' read -ra DB_NAME_ARRAY <<< "$DB_NAMES"
   #selecting backup directory based on the deployment name
   BACKUP_DIR=$BACKUP_DIR/$MONGODB_ID
 }
@@ -102,11 +106,14 @@ backup()
     # Get the current date and time
     DATE=$(date +%Y-%m-%d_%H-%M-%S)
     # Loop through each database and run mongodump
-    for dbName in "msOutputDBPreProd" "msPileupDBPreProd" "msUnmergedDBPreProd"
+    for dbName in "${DB_NAME_ARRAY[@]}"
     do
         echo "Dumping database: $dbName"
-
-        mongodump --uri "mongodb://$USERNAME:$PASSWORD@$URI/$dbName?replicaSet=$RS_NAME" --authenticationDatabase=$AUTHDB --out="$BACKUP_DIR/$DATE" 
+        if mongodump --uri "mongodb://$USERNAME:$PASSWORD@$URI/$dbName?replicaSet=$RS_NAME" --authenticationDatabase="$AUTHDB" --out "$BACKUP_DIR/$DATE"; then
+            echo "MongoDB backup for $dbName succeeded."
+        else
+            echo "MongoDB backup for $dbName failed. Running alerts.sh..."
+            /data/tools/alerts.sh
     done
     find $BACKUP_DIR -mindepth 1 -maxdepth 1 -type d -ctime +10  | xargs rm -rf;
 }
@@ -120,14 +127,13 @@ restore()
     DATE=$(date +%Y-%m-%d_%H-%M-%S)
 
     # Loop through each database and run mongodump
-    for dbName in "msOutputDBPreProd" "msPileupDBPreProd" "msUnmergedDBPreProd"
+    for dbName in "${DB_NAME_ARRAY[@]}"
     do
          echo "Restoring database: $db_name"
 
          mongorestore --uri "mongodb://$USERNAME:$PASSWORD@$URI/$dbName?replicaSet=$RS_NAME" --authenticationDatabase=$AUTHDB  "$BACKUP_DIR/$DATE"
     done
 
-    done
 }
 
 backup_status()

docker/mongodb/source/run.sh

@@ -1,12 +1,14 @@
 #!/bin/bash
 ### This script relies on provided keytab file which will be
 ### be mounted to /etc/krb area
-if [ -f /etc/krb ]; then
+if [ -d /etc/krb ]; then
+  echo "Starting the kinit operation!"
   export keytab=/etc/krb/cmsweb.keytab
   principal=`klist -k "$keytab" | tail -1 | awk '{print $2}'`
-  kinit $principal -k -t "$keytab" 2>&1 1>& /dev/null
+  kinit $principal -k -t "$keytab" >/dev/null 2>&1
   if [ $? == 1 ]; then
     echo "Unable to perform kinit operation for cmsweb keytab."
     exit 1
   fi
 fi
+

helm/mongodb/templates/deployment.yaml

@@ -65,6 +65,8 @@ spec:
               cpu: {{.Values.db.pod.resources.requests.cpu}}
               memory: {{.Values.db.pod.resources.limits.memory}}
           env:
+            - name: "ENVIRONMENT"
+              value: {{ (toString $environment) }}
             - name: "RS_NAME"
               value: {{.Values.db.rsname}}
           {{- if  (eq (toString $environment) "prod") }}
@@ -91,7 +93,6 @@ spec:
             - name: "NODE_HOSTNAME_THREE"
               value: {{.Values.db.nodeName.test.three}}
             {{- end }}
-
             - name: "MONGODB_ID"
               value: {{.Values.db.instance0.mongoId | quote}}
             - name: "MONGODB_ADMIN_PASSWORD"
@@ -192,6 +193,8 @@ spec:
               cpu: {{.Values.db.pod.resources.requests.cpu}}
               memory: {{.Values.db.pod.resources.limits.memory}}
           env:
+            - name: "ENVIRONMENT"
+              value: {{ (toString $environment) }}
             - name: "RS_NAME"
               value: {{.Values.db.rsname}}
             - name: "MONGODB_ID"
@@ -289,6 +292,8 @@ spec:
               cpu: {{.Values.db.pod.resources.requests.cpu}}
               memory: {{.Values.db.pod.resources.limits.memory}}
           env:
+            - name: "ENVIRONMENT"
+              value: {{ (toString $environment) }}
             - name: "RS_NAME"
               value: {{.Values.db.rsname}}
             - name: "MONGODB_ID"

helm/mongodb/values.yaml

@@ -3,11 +3,11 @@ quickSetting:
   namespace: default
   env: test-rs
   image:
-    mongodb: registry.cern.ch/cmsweb/cmsmongo:HG2310a
+    mongodb: registry.cern.ch/cmsweb/cmsmongo:HG2310f-test
   pvc:
     storageClassName: cpio1
     accessMode: ReadWriteMany
-    storageSize: 1Ti
+    storageSize: 1Ti       
     mongodbMountPath: "/mnt/NFS1/cluster5/defacement-detection-test-deployment-with-helm/"
 monitoring:
   jobName: "mongodb-exporter"