diff --git a/docker/mongodb/source/copy_cron.sh b/docker/mongodb/source/copy_cron.sh new file mode 100755 index 000000000..17fd29203 --- /dev/null +++ b/docker/mongodb/source/copy_cron.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Print a message to indicate the script is running +echo "copy_cron.sh is running" + +if [ "$ENVIRONMENT" = "prod" ]; then + # Copy the production cron file + echo "Copying mongo-prod.cron" + cp /data/tools/mongo-prod.cron /data/tools/mongo.cron +elif [ "$ENVIRONMENT" = "preprod" ]; then + # Copy the development cron file + echo "Copying mongo-preprod.cron" + cp /data/tools/mongo-preprod.cron /data/tools/mongo.cron +elif [ "$ENVIRONMENT" = "test" ]; then + # Copy the test cron file + echo "Copying mongo-test.cron" + cp /data/tools/mongo-test.cron /data/tools/mongo.cron +else + echo "Unsupported environment: $ENVIRONMENT" + exit 1 +fi + +# Set up the cron job +crontab /data/tools/mongo.cron + +# Add another cronjob to get kerberos token once everyday +(crontab -l ; echo "0 0 * * * /root/run.sh") | crontab - + diff --git a/docker/mongodb/source/dockerfile b/docker/mongodb/source/dockerfile index 468d62389..00131405c 100644 --- a/docker/mongodb/source/dockerfile +++ b/docker/mongodb/source/dockerfile @@ -46,16 +46,20 @@ COPY --from=go-builder /data/email/email /data/tools #add kinit operation ADD run.sh /root/run.sh -RUN chmod +x /root/run.sh #add cronjob -ADD mongo.cron /data/tools/mongo.cron + #add mongotools COPY /mongotools /data/tools -RUN crontab /data/tools/mongo.cron +COPY copy_cron.sh /data/tools/copy_cron.sh +COPY mongo-prod.cron /data/tools/mongo-prod.cron +COPY mongo-preprod.cron /data/tools/mongo-preprod.cron +COPY mongo-test.cron /data/tools/mongo-test.cron + ENV PATH=/data/tools:$PATH -CMD ./startup-$MONGODB_ID.sh; ./run.sh +CMD ["/bin/sh", "-c", "./run.sh; /data/tools/copy_cron.sh; ./startup-$MONGODB_ID.sh;"] + diff --git a/docker/mongodb/source/mongo-preprod.cron b/docker/mongodb/source/mongo-preprod.cron new file mode 100644 index 000000000..68b390acb --- /dev/null +++ b/docker/mongodb/source/mongo-preprod.cron @@ -0,0 +1,2 @@ +0 */12 * * * export AGE_KEY="/etc/mongodb-secret/age-key.txt" && /data/tools/mongo_manage.sh backup /etc/mongodb-secret/mongo.ini + diff --git a/docker/mongodb/source/mongo-prod.cron b/docker/mongodb/source/mongo-prod.cron new file mode 100644 index 000000000..031c9c3b8 --- /dev/null +++ b/docker/mongodb/source/mongo-prod.cron @@ -0,0 +1,2 @@ +0 */4 * * * export AGE_KEY="/etc/mongodb-secret/age-key.txt" && /data/tools/mongo_manage.sh backup /etc/mongodb-secret/mongo.ini + diff --git a/docker/mongodb/source/mongo-test.cron b/docker/mongodb/source/mongo-test.cron new file mode 100644 index 000000000..2c6473adb --- /dev/null +++ b/docker/mongodb/source/mongo-test.cron @@ -0,0 +1,2 @@ +0 2 * * * export AGE_KEY="/etc/mongodb-secret/age-key.txt" && /data/tools/mongo_manage.sh backup /etc/mongodb-secret/mongo.ini + diff --git a/docker/mongodb/source/mongotools/alerts.sh b/docker/mongodb/source/mongotools/alerts.sh index f2ed49d98..2b5d0bf2b 100755 --- a/docker/mongodb/source/mongotools/alerts.sh +++ b/docker/mongodb/source/mongotools/alerts.sh @@ -5,6 +5,7 @@ msg="MongoDB backup cronjob failure" DATE=`date` host=`hostname` job="mongodb" -amhost="http://cms-monitoring.cern.ch:30093" -amtool alert add mongodb_cronjob_failure alertname="$msg" job="$job" host="$host" tag=mongodb alert=amtool service=cron --end="$expire" --annotation=summary="$msg" --annotation=date="$DATE" --alertmanager.url="$amhost" action=restart +tag="cmsweb" +amhost="http://cms-monitoring-ha1.cern.ch:30093" +amtool alert add mongodb_cronjob_failure alertname="$msg" tag="$tag" job="$job" host="$host" alert=amtool service=cron --end="$expire" --annotation=summary="$msg" --annotation=date="$DATE" --alertmanager.url="$amhost" action=restart amhost="http://cms-monitoring-ha1.cern.ch:30093" diff --git a/docker/mongodb/source/mongotools/mongo_manage.sh b/docker/mongodb/source/mongotools/mongo_manage.sh index 5ffeb2839..082cdf9fc 100755 --- a/docker/mongodb/source/mongotools/mongo_manage.sh +++ b/docker/mongodb/source/mongotools/mongo_manage.sh @@ -38,6 +38,7 @@ init(){ PASSWORD=`cat $CONFIG | grep PASSWORD | sed -e "s,PASSWORD=,,g"` BACKUP_DIR=`cat $CONFIG | grep BACKUP_DIR | sed -e "s,BACKUP_DIR=,,g"` RS_NAME=`cat $CONFIG | grep RS_NAME | sed -e "s,RS_NAME=,,g"` + DB_NAMES=$(cat "$CONFIG" | grep DB_NAMES | sed -e "s,DB_NAMES=,,g") else if [ -z "$AGE_KEY" ]; then echo "AGE_KEY environment is not set, please generate appropriate key file" @@ -53,6 +54,7 @@ init(){ PASSWORD=`age -i $AGE_KEY --decrypt -o - $CONFIG | grep PASSWORD | sed -e "s,PASSWORD=,,g"` BACKUP_DIR=`age -i $AGE_KEY --decrypt -o - $CONFIG | grep BACKUP_DIR | sed -e "s,BACKUP_DIR=,,g"` RS_NAME=`age -i $AGE_KEY --decrypt -o - $CONFIG | grep RS_NAME | sed -e "s,RS_NAME=,,g"` + DB_NAMES=$(cat "$CONFIG" | grep DB_NAMES | sed -e "s,DB_NAMES=,,g") fi if [ -z "$USERNAME" ]; then echo "Unable to locate USERNAME in $CONFIG" @@ -90,6 +92,8 @@ init(){ exit 1 fi fi + # Split DB_NAMES into an array + IFS=' ' read -ra DB_NAME_ARRAY <<< "$DB_NAMES" #selecting backup directory based on the deployment name BACKUP_DIR=$BACKUP_DIR/$MONGODB_ID } @@ -102,11 +106,14 @@ backup() # Get the current date and time DATE=$(date +%Y-%m-%d_%H-%M-%S) # Loop through each database and run mongodump - for dbName in "msOutputDBPreProd" "msPileupDBPreProd" "msUnmergedDBPreProd" + for dbName in "${DB_NAME_ARRAY[@]}" do echo "Dumping database: $dbName" - - mongodump --uri "mongodb://$USERNAME:$PASSWORD@$URI/$dbName?replicaSet=$RS_NAME" --authenticationDatabase=$AUTHDB --out="$BACKUP_DIR/$DATE" + if mongodump --uri "mongodb://$USERNAME:$PASSWORD@$URI/$dbName?replicaSet=$RS_NAME" --authenticationDatabase="$AUTHDB" --out "$BACKUP_DIR/$DATE"; then + echo "MongoDB backup for $dbName succeeded." + else + echo "MongoDB backup for $dbName failed. Running alerts.sh..." + /data/tools/alerts.sh done find $BACKUP_DIR -mindepth 1 -maxdepth 1 -type d -ctime +10 | xargs rm -rf; } @@ -120,14 +127,13 @@ restore() DATE=$(date +%Y-%m-%d_%H-%M-%S) # Loop through each database and run mongodump - for dbName in "msOutputDBPreProd" "msPileupDBPreProd" "msUnmergedDBPreProd" + for dbName in "${DB_NAME_ARRAY[@]}" do echo "Restoring database: $db_name" mongorestore --uri "mongodb://$USERNAME:$PASSWORD@$URI/$dbName?replicaSet=$RS_NAME" --authenticationDatabase=$AUTHDB "$BACKUP_DIR/$DATE" done - done } backup_status() diff --git a/docker/mongodb/source/run.sh b/docker/mongodb/source/run.sh index b81684d0a..842dd0767 100755 --- a/docker/mongodb/source/run.sh +++ b/docker/mongodb/source/run.sh @@ -1,12 +1,14 @@ #!/bin/bash ### This script relies on provided keytab file which will be ### be mounted to /etc/krb area -if [ -f /etc/krb ]; then +if [ -d /etc/krb ]; then + echo "Starting the kinit operation!" export keytab=/etc/krb/cmsweb.keytab principal=`klist -k "$keytab" | tail -1 | awk '{print $2}'` - kinit $principal -k -t "$keytab" 2>&1 1>& /dev/null + kinit $principal -k -t "$keytab" >/dev/null 2>&1 if [ $? == 1 ]; then echo "Unable to perform kinit operation for cmsweb keytab." exit 1 fi fi + diff --git a/helm/mongodb/templates/deployment.yaml b/helm/mongodb/templates/deployment.yaml index eaf885e7e..c0bff0abb 100644 --- a/helm/mongodb/templates/deployment.yaml +++ b/helm/mongodb/templates/deployment.yaml @@ -65,6 +65,8 @@ spec: cpu: {{.Values.db.pod.resources.requests.cpu}} memory: {{.Values.db.pod.resources.limits.memory}} env: + - name: "ENVIRONMENT" + value: {{ (toString $environment) }} - name: "RS_NAME" value: {{.Values.db.rsname}} {{- if (eq (toString $environment) "prod") }} @@ -91,7 +93,6 @@ spec: - name: "NODE_HOSTNAME_THREE" value: {{.Values.db.nodeName.test.three}} {{- end }} - - name: "MONGODB_ID" value: {{.Values.db.instance0.mongoId | quote}} - name: "MONGODB_ADMIN_PASSWORD" @@ -192,6 +193,8 @@ spec: cpu: {{.Values.db.pod.resources.requests.cpu}} memory: {{.Values.db.pod.resources.limits.memory}} env: + - name: "ENVIRONMENT" + value: {{ (toString $environment) }} - name: "RS_NAME" value: {{.Values.db.rsname}} - name: "MONGODB_ID" @@ -289,6 +292,8 @@ spec: cpu: {{.Values.db.pod.resources.requests.cpu}} memory: {{.Values.db.pod.resources.limits.memory}} env: + - name: "ENVIRONMENT" + value: {{ (toString $environment) }} - name: "RS_NAME" value: {{.Values.db.rsname}} - name: "MONGODB_ID" diff --git a/helm/mongodb/values.yaml b/helm/mongodb/values.yaml index 879af464a..f64c6eaba 100644 --- a/helm/mongodb/values.yaml +++ b/helm/mongodb/values.yaml @@ -3,11 +3,11 @@ quickSetting: namespace: default env: test-rs image: - mongodb: registry.cern.ch/cmsweb/cmsmongo:HG2310a + mongodb: registry.cern.ch/cmsweb/cmsmongo:HG2310f-test pvc: storageClassName: cpio1 accessMode: ReadWriteMany - storageSize: 1Ti + storageSize: 1Ti mongodbMountPath: "/mnt/NFS1/cluster5/defacement-detection-test-deployment-with-helm/" monitoring: jobName: "mongodb-exporter"