Improve rucio-dataset-mon-go html (#150)

src/go/rucio-dataset-mon-go/controllers/dataset_controller.go

@@ -5,158 +5,109 @@ import (
 	"encoding/json"
 	"github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/configs"
 	"github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/models"
-	"github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/responses"
+	"github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/mongo"
+	"github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/utils"
 	"github.com/gin-gonic/gin"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
-	"go.mongodb.org/mongo-driver/mongo"
-	"go.mongodb.org/mongo-driver/mongo/options"
 	"io"
 	"log"
 	"net/http"
-	"strings"
 	"time"
 )
 
 var (
-	// connection instance for the "datasets" collection
-	datasetCollection   = configs.GetCollection(configs.DB, configs.GetEnvVar("COLLECTION_DATASETS"))
+	// connection instance for the "datasets" datasetsDb
+	datasetsDb          = mongo.GetCollection(mongo.DBClient, configs.GetEnvVar("COLLECTION_DATASETS"))
 	GlobalTotalRecCount int64
 )
 
-// MiddlewareReqHandler handles CORS and HTTP request settings for the context router
-func MiddlewareReqHandler() gin.HandlerFunc {
-	return func(c *gin.Context) {
-		//c.Writer.Header().Set("Content-Type", "application/json")
-		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
-		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
-		c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
-		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE")
-		if c.Request.Method == "OPTIONS" {
-			c.AbortWithStatus(204)
-			return
-		}
-		c.Next()
-	}
-}
-
-// errorResponse returns error response with given msg and error
-func errorResponse(c *gin.Context, msg string, err error) {
-	log.Printf("[ERROR] %s %s", msg, err)
-	c.JSON(http.StatusInternalServerError,
-		responses.ErrorResponseStruct{
-			Status:  http.StatusInternalServerError,
-			Message: msg,
-			Data:    map[string]string{"data": err.Error()},
-		})
-}
-
-// getRequestBody parse datatable request
+// getRequestBody parse datatables request
 func getRequestBody(c *gin.Context) models.DataTableRequest {
 	// === ~~~~~~ Decode incoming DataTable request json ~~~~~~ ===
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
 	body, err := io.ReadAll(c.Request.Body)
 	if err != nil {
-		errorResponse(c, "Request body read failed", err)
+		utils.ErrorResponse(c, "Request body read failed", err)
 	}
 	log.Printf("Request Body:\n%#v\n", string(body))
 	var dataTableRequest models.DataTableRequest
 	if err := json.Unmarshal(body, &dataTableRequest); err != nil {
-		errorResponse(c, "Unmarshal request body failed", err)
+		utils.ErrorResponse(c, "Unmarshal request body failed", err)
 	}
 	return dataTableRequest
 }
 
-// getTotalRecCount total document count in the collection
-func getTotalRecCount(ctx context.Context, c *gin.Context) int64 {
-	if GlobalTotalRecCount == 0 {
-		countTotal, err := datasetCollection.CountDocuments(ctx, bson.M{})
-		if err != nil {
-			errorResponse(c, "TotalRecCount query failed", err)
-		}
-		GlobalTotalRecCount = countTotal
-	}
-	log.Printf("[INFO] Total Count %d", GlobalTotalRecCount)
-	return GlobalTotalRecCount
-}
-
-// getFilteredRecCount filtered document count in the collection
-func getFilteredRecCount(ctx context.Context, c *gin.Context, findQuery bson.M) int64 {
-	filteredRecCount, err := datasetCollection.CountDocuments(ctx, findQuery)
-	if err != nil {
-		errorResponse(c, "FilteredRecCount query failed", err)
-	}
-	return filteredRecCount
-}
-
-// convertOrderEnumToMongoInt converts DataTable enums ("asc" and "desc") to Mongo sorting integer definitions (1,-1)
-func convertOrderEnumToMongoInt(dir string) int {
-	switch strings.ToLower(dir) {
-	case "asc":
-		return 1
-	case "desc":
-		return -1
-	default:
-		return 0
-	}
-}
-
-// generateSortOrder creates sort order for the Mongo query by iterating over DataTable json request
-func generateSortOrder(dataTableRequest models.DataTableRequest) *options.FindOptions {
+// getSortBson creates sort object which support multiple column sort
+func getSortBson(dataTableRequest models.DataTableRequest) bson.D {
 	orders := dataTableRequest.Orders
 	columns := dataTableRequest.Columns
-	sortOpts := options.Find()
-	sortOpts.Limit = &dataTableRequest.Length
-	sortOpts.Skip = &dataTableRequest.Start
+	sortBson := bson.D{}
 	for _, order := range orders {
-		intSortDirection := convertOrderEnumToMongoInt(order.Dir)
+		intSortDirection := utils.ConvertOrderEnumToMongoInt(order.Dir)
 		if intSortDirection != 0 {
-			sortOpts = sortOpts.SetSort(
-				bson.D{{
-					Key:   columns[order.Column].Data, // column name
-					Value: intSortDirection,           // column direction as int value
-				}},
-			)
+			sortBson = append(sortBson, bson.E{
+				Key:   columns[order.Column].Data, // column name
+				Value: intSortDirection,           // column direction as int value (0/1)
+			})
 		}
 	}
-	return sortOpts
+	// Always add unique id column at the end to be able fetch non-unique columns in order
+	sortBson = append(sortBson, bson.E{Key: "_id", Value: 1})
+	return sortBson
 }
 
-// generateFindQuery creates main search query using regex by default
-func generateFindQuery(dataTableRequest models.DataTableRequest) bson.M {
+// getSearchBson creates main search query using regex by default
+func getSearchBson(dataTableRequest models.DataTableRequest) bson.M {
 	// DataTable main search request struct
 	dtMainSearch := dataTableRequest.Search
 	if dtMainSearch.Value == "" {
 		return bson.M{}
 	} else {
 		log.Printf("[INFO] dtMainSearch.Value is : %s", dtMainSearch.Value)
-		var findQuery []bson.M
-		findQuery = append(findQuery, bson.M{"dataset": primitive.Regex{Pattern: dtMainSearch.Value, Options: "im"}})
-		// i: case insensitive, m: can use ^ and $. Ref: // https://www.mongodb.com/docs/v5.0/reference/operator/query/regex/
 		// TODO add individual column search
-		return bson.M{"$and": findQuery}
+		return bson.M{"$and": bson.M{"Dataset": primitive.Regex{Pattern: dtMainSearch.Value, Options: "im"}}}
 	}
 }
 
-// paginateResults get query results efficiently
-func paginateResults(ctx context.Context, c *gin.Context, findQuery bson.M, sortOptsOfFind *options.FindOptions) []models.Dataset {
-	var datasets []models.Dataset
-	datasetResults, err := datasetCollection.Find(ctx, findQuery, sortOptsOfFind)
+// getTotalRecCount total document count in the datasetsDb
+func getTotalRecCount(ctx context.Context, c *gin.Context) int64 {
+	if GlobalTotalRecCount == 0 {
+		countTotal, err := mongo.GetCount(ctx, datasetsDb, bson.M{})
+		if err != nil {
+			utils.ErrorResponse(c, "TotalRecCount query failed", err)
+		}
+		GlobalTotalRecCount = countTotal
+	}
+	log.Printf("[INFO] Total Count %d", GlobalTotalRecCount)
+	return GlobalTotalRecCount
+}
+
+// getFilteredRecCount filtered document count in the datasetsDb
+func getFilteredRecCount(ctx context.Context, c *gin.Context, findQuery bson.M) int64 {
+	filteredRecCount, err := mongo.GetCount(ctx, datasetsDb, findQuery)
 	if err != nil {
-		errorResponse(c, "datasetCollection.Find query failed", err)
+		utils.ErrorResponse(c, "FilteredRecCount query failed", err)
 	}
+	return filteredRecCount
+}
 
-	// reading from the db in an optimal way
-	defer func(results *mongo.Cursor, ctx context.Context) {
-		if err := results.Close(ctx); err != nil {
-			errorResponse(c, "MongoDB cursor failed", err)
-		}
-	}(datasetResults, ctx)
+// getQueryResults get query results efficiently
+func getQueryResults(ctx context.Context, c *gin.Context, dataTableRequest models.DataTableRequest) []models.Dataset {
+	var datasets []models.Dataset
+	searchQuery := getSearchBson(dataTableRequest)
+	sortQuery := getSortBson(dataTableRequest)
+	limit := dataTableRequest.Length
+	skip := dataTableRequest.Start
 
-	for datasetResults.Next(ctx) {
+	datasetsCursor, err := mongo.GetResults(ctx, datasetsDb, searchQuery, sortQuery, skip, limit)
+	if err != nil {
+		utils.ErrorResponse(c, "datasetsDb.Find query failed", err)
+	}
+	for datasetsCursor.Next(ctx) {
 		var singleDataset models.Dataset
-		if err = datasetResults.Decode(&singleDataset); err != nil {
-			errorResponse(c, "datasetResults.Decode failed", err)
+		if err = datasetsCursor.Decode(&singleDataset); err != nil {
+			utils.ErrorResponse(c, "datasetResults.Decode failed", err)
 		}
 		datasets = append(datasets, singleDataset)
 	}
@@ -171,10 +122,8 @@ func GetDatasets() gin.HandlerFunc {
 		dataTableRequest := getRequestBody(c)
 
 		totalRecCount := getTotalRecCount(ctx, c)
-		sortOptsOfFind := generateSortOrder(dataTableRequest)
-		findQuery := generateFindQuery(dataTableRequest)
-		filteredRecCount := getFilteredRecCount(ctx, c, findQuery)
-		datasets := paginateResults(ctx, c, findQuery, sortOptsOfFind)
+		filteredRecCount := getFilteredRecCount(ctx, c, getSearchBson(dataTableRequest))
+		datasets := getQueryResults(ctx, c, dataTableRequest)
 
 		// Send response in DataTable required format
 		//  - Need to return exactly same "Draw" value that DataTable sent in incoming request

src/go/rucio-dataset-mon-go/main.go

@@ -3,8 +3,8 @@ package main
 import (
 	"flag"
 	"fmt"
-	"github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/configs"
 	"github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/controllers"
+	"github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/mongo"
 	"github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/routes"
 	"golang.org/x/sync/errgroup"
 	"log"
@@ -40,7 +40,7 @@ func main() {
 	}
 
 	// connect to database
-	configs.ConnectDB()
+	mongo.GetMongoClient()
 	controllers.GitVersion = gitVersion
 	controllers.ServerInfo = info()
 

src/go/rucio-dataset-mon-go/models/dataset_model.go

@@ -1,16 +1,14 @@
 package models
 
-import "go.mongodb.org/mongo-driver/bson/primitive"
-
 // Dataset struct which includes Rucio and DBS calculated values
 type Dataset struct {
-	Id                     primitive.ObjectID `bson:"_id"`
-	RseType                string             `bson:"RSE_TYPE,omitempty" validate:"required"`
-	Dataset                string             `bson:"dataset,omitempty" validate:"required"`
-	LastAccess             string             `bson:"last_access"`
-	LastAccessMs           int64              `bson:"last_access_ms"`
-	MaxDatasetSizeInRsesGB float64            `bson:"max_dataset_size_in_rses(GB)"`
-	MinDatasetSizeInRsesGB float64            `bson:"min_dataset_size_in_rses(GB)"`
-	SumDatasetSizeInRsesGB float64            `bson:"sum_dataset_size_in_rses(GB)"`
-	RSEs                   string             `bson:"RSE(s)"`
+	RseType                string  `bson:"RseType,omitempty" validate:"required"`
+	Dataset                string  `bson:"Dataset,omitempty" validate:"required"`
+	LastAccess             string  `bson:"LastAccess"`
+	LastAccessMs           int64   `bson:"LastAccessMs"`
+	MaxDatasetSizeInRsesGB float64 `bson:"MaxDatasetSizeInRsesGB"`
+	MinDatasetSizeInRsesGB float64 `bson:"MinDatasetSizeInRsesGB"`
+	AvgDatasetSizeInRsesGB float64 `bson:"AvgDatasetSizeInRsesGB"`
+	SumDatasetSizeInRsesGB float64 `bson:"SumDatasetSizeInRsesGB"`
+	RSEs                   string  `bson:"RSEs"`
 }

src/go/rucio-dataset-mon-go/mongo/mongo.go

@@ -0,0 +1,25 @@
+package mongo
+
+import (
+	"context"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+// GetResults returns cursor of aggregate query results
+func GetResults(ctx context.Context, collection *mongo.Collection, match bson.M, sort bson.D, skip int64, limit int64) (*mongo.Cursor, error) {
+	pipeline := []bson.M{
+		{"$match": match},
+		{"$sort": sort},
+		{"$skip": skip},
+		{"$limit": limit},
+	}
+	cursor, err := collection.Aggregate(ctx, pipeline)
+	return cursor, err
+}
+
+// GetCount returns count of query result
+func GetCount(ctx context.Context, collection *mongo.Collection, match bson.M) (int64, error) {
+	count, err := collection.CountDocuments(ctx, match)
+	return count, err
+}

src/go/rucio-dataset-mon-go/mongo/setup.go

@@ -0,0 +1,36 @@
+package mongo
+
+import (
+	"context"
+	"github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/configs"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"log"
+	"time"
+)
+
+// DBClient MongoDb Client instance(mongo.Client)
+var DBClient = GetMongoClient()
+
+// GetMongoClient returns mongo client
+func GetMongoClient() *mongo.Client {
+	var err error
+	var client *mongo.Client
+	opts := options.Client()
+	opts.ApplyURI(configs.EnvMongoURI())
+	opts.SetMaxPoolSize(100)
+	opts.SetConnectTimeout(time.Duration(configs.EnvConnTimeout()) * time.Second)
+	if client, err = mongo.Connect(context.Background(), opts); err != nil {
+		log.Fatal(err)
+	}
+	if err := client.Ping(context.Background(), nil); err != nil {
+		log.Fatal(err)
+	}
+	return client
+}
+
+// GetCollection returns Mongo db collection with the given collection name
+func GetCollection(client *mongo.Client, collectionName string) *mongo.Collection {
+	collection := client.Database(configs.EnvMongoDB()).Collection(collectionName)
+	return collection
+}