Add workflow and update script

.github/workflows/build-grafana-backup.yml

@@ -0,0 +1,49 @@
+# Ref: https://githubhttps://github.blog/changelog/2023-06-13-github-actions-all-actions-will-run-on-node16-instead-of-node12-by-default/.com/dmwm/dbs2go/blob/master/.github/workflows/build.yml
+
+name: Build grafana backup
+
+on:
+  push:
+    tags:
+      - 'gb-*.*.*'
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ^1.20
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+
+      - name: Get git tag
+        id: get_tag
+        run: echo ::set-output name=tag::${GITHUB_REF/refs\/tags\//}
+
+      - name: Build grafana-backup image
+        run: |
+          echo Image tag: ${{ steps.get_tag.outputs.tag }}
+          curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/grafana-backup/Dockerfile
+          sed -i -e "s,ENV CMSMON_TAG=.*,ENV CMSMON_TAG=${{steps.get_tag.outputs.tag}},g" Dockerfile
+          docker build . --tag docker.pkg.github.com/dmwm/grafana-backup/grafana-backup
+          docker tag docker.pkg.github.com/dmwm/grafana-backup/grafana-backup registry.cern.ch/cmsmonitoring/grafana-backup
+
+     - name: Login to registry.cern.ch
+       uses: docker/login-action@v3
+       with:
+         registry: registry.cern.ch
+         username: ${{ secrets.CERN_LOGIN }}
+         password: ${{ secrets.CERN_TOKEN }}
+
+     - name: Publish grafana-backup image to registry.cern.ch
+       uses: docker/build-push-action@v1
+       with:
+         username: ${{ secrets.CERN_LOGIN }}
+         password: ${{ secrets.CERN_TOKEN }}
+         registry: registry.cern.ch
+         repository: cmsmonitoring/grafana-backup
+         tag_with_ref: true

src/go/grafana-backup/dashboard-exporter.go

@@ -4,34 +4,33 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
 
-	"github.com/urfave/cli/v2"
 	"archive/tar" // Import the "tar" package for tar file operations
-)
 
+	"github.com/urfave/cli/v2"
+)
 
-const baseUrl = "https://monit-grafana.cern.ch/api"
+const (
+	baseUrl       = "https://monit-grafana.cern.ch/api"
+	grafanaFolder = "./grafana"
+	tarFileName   = "grafanaBackup.tar.gz"
+)
 
 type Dashboard struct {
 	Uid   string `json:"uid"`
 	Title string `json:"title"`
 }
 
-func updatePathEnding(path string) string {
-	if !strings.HasSuffix(path, "/") {
-		path += "/"
-	}
-	return path
-}
-
 func getGrafanaAuth(fname string) (string, error) {
 	if _, err := os.Stat(fname); os.IsNotExist(err) {
-		return "", fmt.Errorf("File %s does not exist", fname)
+		return "", fmt.Errorf("file %s does not exist", fname)
 	}
 
 	file, err := os.Open(fname)
@@ -48,19 +47,168 @@ func getGrafanaAuth(fname string) (string, error) {
 
 	secretKey, ok := data["SECRET_KEY"].(string)
 	if !ok {
-		return "", fmt.Errorf("Failed to get SECRET_KEY from file")
+		return "", fmt.Errorf("failed to get SECRET_KEY from file")
 	}
 
 	headers := fmt.Sprintf("Bearer %s", secretKey)
 	return headers, nil
 }
 
-func createBackUpFiles(dashboard Dashboard, folderTitle, headers string) {
-	// Function to create backup files
+func createBackUpFiles(dashboard Dashboard, folderTitle, headers string) error {
+	allJson := []Dashboard{}
+
+	dashboardUid := dashboard.Uid
+	path := filepath.Join("grafana", "dashboards", folderTitle)
+
+	if err := os.MkdirAll(path, os.ModePerm); err != nil {
+		return err
+	}
+
+	dashboardUrl := filepath.Join(baseUrl, "dashboards/uid", dashboardUid)
+	fmt.Println("dashboardUrl", dashboardUrl)
+	req, err := http.NewRequest("GET", dashboardUrl, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", headers)
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	var dashboardData Dashboard
+	if err := json.Unmarshal(body, &dashboardData); err != nil {
+		return err
+	}
+
+	titleOfDashboard := strings.ReplaceAll(dashboardData.Title, " ", "_")
+	titleOfDashboard = strings.ReplaceAll(titleOfDashboard, ".", "_")
+	titleOfDashboard = strings.ReplaceAll(titleOfDashboard, "/", "_")
+	uidOfDashboard := dashboardData.Uid
+
+	filenameForFolder := fmt.Sprintf("%s/%s-%s.json", path, titleOfDashboard, uidOfDashboard)
+	fmt.Println("filenameForFolder", filenameForFolder)
+	if err := os.WriteFile(filenameForFolder, body, os.ModePerm); err != nil {
+		return err
+	}
+
+	fmt.Println(strings.Repeat("*", 10) + "\n")
+	allJson = append(allJson, dashboardData)
+
+	filenameForAllJson := filepath.Join(path, "all.json")
+	fmt.Println("filenameForAllJson", filenameForAllJson)
+	allJsonBytes, err := json.Marshal(allJson)
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(filenameForAllJson, allJsonBytes, os.ModePerm); err != nil {
+		return err
+	}
+
+	return nil
 }
 
-func searchFoldersFromGrafana(headers string) {
-	// Function to search folders and perform backup operations
+func searchFoldersFromGrafana(headers string) error {
+	foldersUrl := filepath.Join(baseUrl, "search?folderIds=0&orgId=11")
+
+	req, err := http.NewRequest("GET", foldersUrl, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", headers)
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("request %s, failed with reason: %s", foldersUrl, resp.Status)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	var foldersJson []Dashboard
+	if err := json.Unmarshal(body, &foldersJson); err != nil {
+		return err
+	}
+
+	for _, folder := range foldersJson {
+		folderId := folder.Uid
+		folderTitle := folder.Title
+
+		if folderTitle != "Production" && folderTitle != "Development" && folderTitle != "Playground" && folderTitle != "Backup" {
+			folderTitle = "General"
+			dashboard := folder
+			if err := createBackUpFiles(dashboard, folderTitle, headers); err != nil {
+				return err
+			}
+		} else {
+			dashboardQueryUrl := filepath.Join(baseUrl, "search?folderIds="+folderId+"&orgId=11&query=")
+
+			fmt.Println("individualFolderUrl", dashboardQueryUrl)
+
+			req, err := http.NewRequest("GET", dashboardQueryUrl, nil)
+			if err != nil {
+				return err
+			}
+			req.Header.Set("Authorization", headers)
+
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				return err
+			}
+			defer resp.Body.Close()
+
+			if resp.StatusCode != http.StatusOK {
+				return fmt.Errorf("request %s, failed with reason: %s", dashboardQueryUrl, resp.Status)
+			}
+
+			body, err := io.ReadAll(resp.Body)
+			if err != nil {
+				return err
+			}
+
+			pathToDb := filepath.Join("grafana")
+			if err := os.MkdirAll(pathToDb, os.ModePerm); err != nil {
+				return err
+			}
+
+			filenameForFolder := filepath.Join(pathToDb, folderId+"-"+folderTitle+".json")
+			fmt.Println("filenameForFolder", filenameForFolder)
+			if err := os.WriteFile(filenameForFolder, body, os.ModePerm); err != nil {
+				return err
+			}
+
+			var folderData []Dashboard
+			if err := json.Unmarshal(body, &folderData); err != nil {
+				return err
+			}
+
+			dashboardsAmount := len(folderData)
+			for index, dashboard := range folderData {
+				folderTitle := dashboard.Title
+				fmt.Printf("Index/Amount: %d/%d\n", index, dashboardsAmount)
+
+				if err := createBackUpFiles(dashboard, folderTitle, headers); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
 }
 
 func createTar(path, tarName string) error {
@@ -134,8 +282,7 @@ func getDate() (string, string, string) {
 
 func copyToFileSystem(archive, baseDir string) error {
 	day, mon, year := getDate()
-	baseDir = updatePathEnding(baseDir)
-	path := fmt.Sprintf("%s%s/%s/%s", baseDir, year, mon, day)
+	path := filepath.Join(baseDir, year, mon, day)
 
 	if err := os.MkdirAll(path, os.ModePerm); err != nil {
 		return err
@@ -159,6 +306,24 @@ func copyToFileSystem(archive, baseDir string) error {
 	return nil
 }
 
+func backupGrafana(headers, filesystemPath string) error {
+	searchFoldersFromGrafana(headers)
+
+	if err := createTar(grafanaFolder, tarFileName); err != nil {
+		return fmt.Errorf("error creating tar: %v", err)
+	}
+
+	if err := copyToFileSystem(tarFileName, filesystemPath); err != nil {
+		return fmt.Errorf("error copying to filesystem: %v", err)
+	}
+
+	if err := removeTempFiles(grafanaFolder); err != nil {
+		return fmt.Errorf("error removing temp files: %v", err)
+	}
+
+	return nil
+}
+
 func main() {
 	app := &cli.App{
 		Name:  "Grafana Backup",
@@ -181,23 +346,14 @@ func main() {
 
 			headers, err := getGrafanaAuth(tokenFile)
 			if err != nil {
-				return err
-			}
-
-			searchFoldersFromGrafana(headers)
-
-			if err := createTar("./grafana", "grafanaBackup.tar.gz"); err != nil {
-				return err
+				log.Fatalf("Error getting Grafana auth: %v", err)
 			}
 
-			if err := copyToFileSystem("grafanaBackup.tar.gz", filesystemPath); err != nil {
-				return err
-			}
-
-			if err := removeTempFiles("./grafana/"); err != nil {
-				return err
+			if err := backupGrafana(headers, filesystemPath); err != nil {
+				log.Fatalf("Error backing up Grafana: %v", err)
 			}
 
+			fmt.Println("Backup completed successfully")
 			return nil
 		},
 	}

src/go/grafana-backup/run.sh

@@ -9,7 +9,7 @@
 ##H        run.sh keytab keys/token.json /eos/cms/store/group/offcomp_monit/
 
 # help definition
-if [ "$1" == "-h" ] || [ "$1" == "-help" ] || [ "$1" == "--help" ] || [ "$1" == "help" ] || [ "$1" == "" ]; then
+if [ "$1" = "-h" ] || [ "$1" = "-help" ] || [ "$1" = "--help" ] || [ "$1" = "help" ] || [ "$1" = "" ]; then
     grep "^##H" <"$0" | sed -e "s,##H,,g"
     exit 1
 fi