Merge pull request #6 from silinternational/develop

Release 1.2.0: Backups using Docker, Restic, and Backblaze B2

.gitignore

@@ -36,3 +36,4 @@ inc/
 
 # tfc-dump creates .json files that don't belong in version control
 *.json
+*.env

Dockerfile

@@ -0,0 +1,30 @@
+FROM alpine:3
+
+# Variables set with ARG can be overridden at image build time with
+# "--build-arg var=value".  They are not available in the running container.
+ARG restic_ver=0.15.2
+ARG tfc_ops_ver=3.5.1
+ARG tfc_ops_distrib=tfc-ops_${tfc_ops_ver}_Linux_x86_64.tar.gz
+
+# Install Restic, tfc-ops, perl, and jq
+RUN cd /tmp \
+ && wget -O /tmp/restic.bz2 \
+    https://github.com/restic/restic/releases/download/v${restic_ver}/restic_${restic_ver}_linux_amd64.bz2 \
+ && bunzip2 /tmp/restic.bz2 \
+ && chmod +x /tmp/restic \
+ && mv /tmp/restic /usr/local/bin/restic \
+ && wget https://github.com/silinternational/tfc-ops/releases/download/v${tfc_ops_ver}/${tfc_ops_distrib} \
+ && tar zxf ${tfc_ops_distrib} \
+ && rm LICENSE README.md ${tfc_ops_distrib} \
+ && mv tfc-ops /usr/local/bin \
+ && apk update \
+ && apk add --no-cache perl jq curl \
+ && rm -rf /var/cache/apk/*
+
+COPY ./tfc-backup.sh  /usr/local/bin/tfc-backup.sh
+COPY ./tfc-dump.pl    /usr/local/bin/tfc-dump.pl
+COPY application/     /data/
+
+WORKDIR /data
+
+CMD [ "/usr/local/bin/tfc-backup.sh" ]

README.md

@@ -37,3 +37,17 @@ Spaces in the variable set name are replaced with hyphens (`-`).
 ## Restrictions
 The code assumes that all of the Terraform Cloud Variable Sets are contained
 within the first result page of 20 entries.
+
+## Example use with Docker and Backblaze
+The image created by the Dockerfile will run `tfc-dump` and use Restic to back up the resulting files to a repository stored in a Backblaze B2 bucket.
+1. Copy `local.env.dist` to `local.env`.
+1. Set the values for the variables contained in `local.env`.
+1. Obtain a Terraform Cloud access token. Go to https://app.terraform.io/app/settings/tokens to create an API token.
+1. Add the access token value to `local.env`.
+1. Create a Backblaze B2 bucket. Set the `File Lifecycle` to `Keep only the last version`.
+1. Add the B2 bucket name to `RESTIC_REPOSITORY` in `local.env`.
+1. Obtain a Backblaze Application Key. Restrict its access to the B2 bucket you just created. Ensure the application key has these capabilities: deleteFiles, listBuckets, listFiles, readBuckets, readFiles, writeBuckets, writeFiles.
+1. Add the application key and secret to `local.env` as the values of `B2_ACCOUNT_ID` and `B2_ACCOUNT_KEY` respectively.
+1. Build the Docker image:  `docker build --tag tfc-backup:latest .`
+1. Initialize the Restic repository (one time only):  `docker run --env-file=local.env --env BACKUP_MODE=init tfc-backup:latest`
+1. Run the Docker image:  `docker run --env-file=local.env tfc-backup:latest`

application/backup.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env sh
+
+STATUS=0
+
+echo "tfc-backup: Backing up ${SOURCE_PATH}"
+
+start=$(date +%s)
+/usr/local/bin/restic backup --host ${RESTIC_HOST} --tag ${RESTIC_TAG} ${RESTIC_BACKUP_ARGS} ${SOURCE_PATH} || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+	echo "tfc-backup: FATAL: Backup returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+	exit $STATUS
+else
+	echo "tfc-backup: Backup completed in $(expr ${end} - ${start}) seconds."
+fi
+
+start=$(date +%s)
+/usr/local/bin/restic forget --host ${RESTIC_HOST} ${RESTIC_FORGET_ARGS} --prune || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+	echo "tfc-backup: FATAL: Backup pruning returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+	exit $STATUS
+else
+	echo "tfc-backup: Backup pruning completed in $(expr ${end} - ${start}) seconds."
+fi
+
+start=$(date +%s)
+/usr/local/bin/restic check || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+	echo "tfc-backup: FATAL: Repository check returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+	exit $STATUS
+else
+	echo "tfc-backup: Repository check completed in $(expr ${end} - ${start}) seconds."
+fi
+
+start=$(date +%s)
+/usr/local/bin/restic unlock || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+	echo "tfc-backup: FATAL: Repository unlock returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+	exit $STATUS
+else
+	echo "tfc-backup: Repository unlock completed in $(expr ${end} - ${start}) seconds."
+fi
+
+exit $STATUS

application/init.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env sh
+
+STATUS=0
+
+echo "tfc-backup: init: Started"
+
+start=$(date +%s)
+/usr/local/bin/restic init || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+	echo "tfc-backup: FATAL: Repository initialization returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+	exit $STATUS
+else
+	echo "tfc-backup: Repository initialization completed in $(expr ${end} - ${start}) seconds."
+fi
+
+echo "tfc-backup: init: Completed"
+exit $STATUS

local.env.dist

@@ -0,0 +1,27 @@
+# ATLAS_TOKEN        - Terraform Cloud access token
+# B2_ACCOUNT_ID      - Backblaze keyID
+# B2_ACCOUNT_KEY     - Backblaze applicationKey
+# BACKUP_MODE        - `init` initializes the Restic repository at `$RESTIC_REPOSITORY` (only do this once)
+#                      `backup` performs a backup
+# ORGANIZATION       - Name of the Terraform Cloud organization to be backed up
+# RESTIC_BACKUP_ARGS - additional arguments to pass to 'restic backup' command
+# RESTIC_FORGET_ARGS - additional arguments to pass to 'restic forget --prune' command
+#                      (e.g., --keep-daily 7 --keep-weekly 5  --keep-monthly 3 --keep-yearly 2)
+# RESTIC_HOST        - hostname to be used for the backup
+# RESTIC_PASSWORD    - password for the Restic repository
+# RESTIC_REPOSITORY  - Restic repository location (e.g., 'b2:bucketname:restic')
+# RESTIC_TAG         - tag to apply to the backup
+# SOURCE_PATH        - Full path to the directory to be backed up
+
+ATLAS_TOKEN=
+B2_ACCOUNT_ID=
+B2_ACCOUNT_KEY=
+BACKUP_MODE=
+ORGANIZATION=
+RESTIC_BACKUP_ARGS=
+RESTIC_FORGET_ARGS=
+RESTIC_HOST=
+RESTIC_PASSWORD=
+RESTIC_REPOSITORY=b2:backblaze-bucket-name-goes-here:restic
+RESTIC_TAG=
+SOURCE_PATH=

tfc-backup.sh

@@ -0,0 +1,75 @@
+#!/bin/sh
+
+# tfc-backup.sh - Back up Terraform Cloud workspaces and variables to a Restic repository on Backblaze B2
+#
+# Dale Newby
+# SIL International
+# July 20, 2023
+
+# Required environment variables:
+# ATLAS_TOKEN        - Terraform Cloud access token
+# B2_ACCOUNT_ID      - Backblaze keyID
+# B2_ACCOUNT_KEY     - Backblaze applicationKey
+# FSBACKUP_MODE      - `init` initializes the Restic repository at `$RESTIC_REPOSITORY` (only do this once)
+#                      `backup` performs a backup
+# ORGANIZATION       - Name of the Terraform Cloud organization to be backed up
+# RESTIC_BACKUP_ARGS - additional arguments to pass to 'restic backup' command
+# RESTIC_FORGET_ARGS - additional arguments to pass to 'restic forget --prune' command
+#                      (e.g., --keep-daily 7 --keep-weekly 5  --keep-monthly 3 --keep-yearly 2)
+# RESTIC_HOST        - hostname to be used for the backup
+# RESTIC_PASSWORD    - password for the Restic repository
+# RESTIC_REPOSITORY  - Restic repository location (e.g., 'b2:bucketname:restic')
+# RESTIC_TAG         - tag to apply to the backup
+# SOURCE_PATH        - Full path to the directory to be backed up
+
+STATUS=0
+
+case "${BACKUP_MODE}" in
+	init)
+		/data/${BACKUP_MODE}.sh || STATUS=$?
+		;;
+	backup)
+		echo "tfc-backup: backup: Started"
+		echo "tfc-backup: Exporting Terraform Cloud data to ${SOURCE_PATH}"
+
+		mkdir -p ${SOURCE_PATH} && cd ${SOURCE_PATH} && rm -rf *
+		if [ $STATUS -ne 0 ]; then
+			echo "tfc-backup: FATAL: Cannot create directory ${SOURCE_PATH}: $STATUS"
+			exit $STATUS
+		fi
+
+		start=$(date +%s)
+		/usr/local/bin/tfc-dump.pl --org ${ORGANIZATION} --all --quiet
+		end=$(date +%s)
+
+		if [ $STATUS -ne 0 ]; then
+			echo "tfc-backup: FATAL: Terraform Cloud export returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+        		exit $STATUS
+		else
+        		echo "tfc-backup: Terraform Cloud export completed in $(expr ${end} - ${start}) seconds."
+		fi
+
+		/data/${BACKUP_MODE}.sh || STATUS=$?
+		if [ $STATUS -ne 0 ]; then
+			echo "tfc-backup: FATAL: backup failed: $STATUS"
+			exit $STATUS
+		fi
+
+		cd .. &&  rm -rf ${SOURCE_PATH} || STATUS=$?
+		if [ $STATUS -ne 0 ]; then
+			echo "tfc-backup: FATAL: Cannot remove directory ${SOURCE_PATH}: $STATUS"
+			exit $STATUS
+		fi
+
+		echo "tfc-backup: backup: Completed"
+		;;
+	*)
+		echo "tfc-backup: FATAL: Unknown BACKUP_MODE: ${BACKUP_MODE}"
+		exit 1
+esac
+
+if [ $STATUS -ne 0 ]; then
+	echo "tfc-backup: Non-zero exit: $STATUS"
+fi
+
+exit $STATUS

tfc-dump.pl

@@ -2,7 +2,7 @@
 #
 # tfc-dump.pl - dump Terraform Cloud workspace and variable information
 #
-# Usage: tfc-dump.pl --org=org-name {--workspace=name | --all} [--help]
+# Usage: tfc-dump.pl --org=org-name {--workspace=name | --all} [--quiet] [--help]
 #
 # For the supplied Terraform Cloud workspace name, dump the workspace
 # and variable information in JSON format.
@@ -21,18 +21,20 @@
 use warnings;
 use Getopt::Long qw(GetOptions);
 
-my $usage = "Usage: $0 --org=org-name {--workspace=name | --all} [--help]\n";
+my $usage = "Usage: $0 --org=org-name {--workspace=name | --all} [--quiet] [--help]\n";
 my $tfc_org_name;	# Terraform Cloud organization name
 my $tfc_workspace_name;	# Terraform Cloud workspace name
 my $tfc_workspace_id;	# Terraform Cloud workspace ID
 my $all_workspaces;
+my $quiet_mode;
 my $help;
 
 Getopt::Long::Configure qw(gnu_getopt);
 GetOptions(
 	'org|o=s'       => \$tfc_org_name,
 	'workspace|w=s' => \$tfc_workspace_name,
 	'all|a'         => \$all_workspaces,
+	'quiet|q'       => \$quiet_mode,
 	'help|h'        => \$help
 ) or die $usage;
 
@@ -48,6 +50,9 @@
 my $curl_header1 = "--header \"Authorization: Bearer $ENV{ATLAS_TOKEN}\"";
 my $curl_header2 = "--header \"Content-Type: application/vnd.api+json\"";
 my $curl_headers = "$curl_header1 $curl_header2";
+if (defined($quiet_mode)) {
+	$curl_headers .= " --no-progress-meter";
+}
 my $curl_query;
 my $curl_cmd;
 my $jq_cmd;