Add graph for vulnerabilities over time and daily imports

.github/workflows/pr.yaml

@@ -42,4 +42,4 @@ jobs:
         run: |
           make fmt
           make generate
-          git diff --exit-code --name-only
+          git diff --exit-code --name-only

.graphqlrc.yaml

@@ -0,0 +1,4 @@
+projects:
+  default:
+    schema:
+      - ./internal/graph/graphgls/*.graphqls

Makefile

@@ -35,7 +35,7 @@ local:
 	env bash -c 'source local.env; go run ./cmd/api'
 
 test:
-	go test ./...
+	go test -cover ./...
 
 check: staticcheck vulncheck deadcode
 

cmd/setup_local/main.go

@@ -11,6 +11,8 @@ import (
 	"time"
 	"unicode"
 
+	"github.com/jackc/pgx/v5/pgtype"
+
 	"cloud.google.com/go/pubsub"
 	"github.com/google/uuid"
 	"github.com/nais/api/internal/database"
@@ -36,10 +38,16 @@ type seedConfig struct {
 	NumTeams          *int
 	NumOwnersPerTeam  *int
 	NumMembersPerTeam *int
+	VulnSeed          *VulnSeed
 	ForceSeed         *bool
 	ProvisionPubSub   *bool
 }
 
+type VulnSeed struct {
+	NumVulnAppsForTeam *int
+	NumVulnPerApp      *int
+}
+
 func newSeedConfig(ctx context.Context) (*seedConfig, error) {
 	cfg := &seedConfig{}
 	err := envconfig.Process(ctx, cfg)
@@ -53,6 +61,8 @@ func newSeedConfig(ctx context.Context) (*seedConfig, error) {
 	cfg.NumMembersPerTeam = flag.Int("members", 10, "number of members per team")
 	cfg.ForceSeed = flag.Bool("force", false, "seed regardless of existing database content")
 	cfg.ProvisionPubSub = flag.Bool("provision_pub_sub", true, "set up pubsub credentials")
+	cfg.VulnSeed.NumVulnAppsForTeam = flag.Int("vuln-apps", 5, "number of vulnerable apps per team")
+	cfg.VulnSeed.NumVulnPerApp = flag.Int("vuln-per-app", 10, "number of vulnerabilities per app")
 	flag.Parse()
 
 	return cfg, nil
@@ -83,7 +93,7 @@ func run(ctx context.Context, cfg *seedConfig, log logrus.FieldLogger) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	if false {
+	if *cfg.ProvisionPubSub {
 		if err := os.Setenv("PUBSUB_EMULATOR_HOST", "localhost:3004"); err != nil {
 			return err
 		}
@@ -225,6 +235,11 @@ func run(ctx context.Context, cfg *seedConfig, log logrus.FieldLogger) error {
 			}
 		}
 
+		err = seedVulnerabilities(ctx, *cfg, dbtx, devteam, log)
+		if err != nil {
+			return err
+		}
+
 		err = dbtx.SetTeamMemberRole(ctx, devUser.ID, devteam.Slug, gensql.RoleNameTeamowner)
 		if err != nil {
 			return err
@@ -269,6 +284,57 @@ func run(ctx context.Context, cfg *seedConfig, log logrus.FieldLogger) error {
 	return nil
 }
 
+func seedVulnerabilities(ctx context.Context, cfg seedConfig, dbtx database.Database, team *database.Team, log logrus.FieldLogger) error {
+	numbOfErrors := 0
+	for j := 0; j < *cfg.VulnSeed.NumVulnAppsForTeam; j++ {
+		appName := fmt.Sprintf("app-%d", j)
+		projectId := uuid.New()
+		err := dbtx.CreateDependencytrackProject(ctx, gensql.CreateDependencytrackProjectParams{
+			Environment: "dev",
+			TeamSlug:    team.Slug,
+			App:         appName,
+			Projectid:   projectId,
+		})
+		if err != nil {
+			return err
+		}
+
+		var vulnbBatch []gensql.VulnerabilityMetricsUpsertParams
+		date := time.Now()
+		var critical int
+		var high int
+		var medium int
+		var low int
+		var unassigned int
+		for k := 0; k < *cfg.VulnSeed.NumVulnPerApp; k++ {
+			critical = rand.Intn(10)
+			high = rand.Intn(10)
+			medium = rand.Intn(10)
+			low = rand.Intn(10)
+			unassigned = rand.Intn(10)
+			vulnbBatch = append(vulnbBatch, gensql.VulnerabilityMetricsUpsertParams{
+				Date:                     pgtype.Date{Time: date.AddDate(0, 0, -k).UTC(), Valid: true},
+				DependencytrackProjectID: projectId,
+				RiskScore:                float64((critical * 10) + (high * 5) + (medium * 3) + (low * 1) + (unassigned * 5)),
+				Critical:                 int32(critical),
+				High:                     int32(high),
+				Medium:                   int32(medium),
+				Low:                      int32(low),
+				Unassigned:               int32(unassigned),
+			})
+		}
+
+		dbtx.VulnerabilityMetricsUpsert(ctx, vulnbBatch).Exec(func(i int, err error) {
+			if err != nil {
+				log.Errorf("error updating vulnerability metrics for team %s: %v", team.Slug, err)
+				numbOfErrors++
+			}
+		})
+	}
+	log.Infof("vulnerability metrics for team %s seeded", team.Slug)
+	return nil
+}
+
 func teamName() string {
 	letters := []byte("abcdefghijklmnopqrstuvwxyz")
 	b := make([]byte, 10)

data/k8s/dev/devteam/deploy-canary.yaml

@@ -0,0 +1,61 @@
+apiVersion: nais.io/v1alpha1
+kind: Application
+metadata:
+  annotations:
+    deploy.nais.io/client-version: 2023-01-23-7071cd7
+    nais.io/deploymentCorrelationID: f8c04f82-6a84-4a8e-9f8b-563b5894d0cf
+    nais.io/skipDeploymentMessage: "true"
+  creationTimestamp: "2023-01-20T10:51:47Z"
+  finalizers:
+    - naiserator.nais.io/finalizer
+  generation: 407981
+  name: nais-deploy-canary
+  resourceVersion: "3701834314"
+  uid: 91ba6c9d-0199-4123-aff6-aa27ce5d2056
+spec:
+  env:
+    - name: DEPLOY_START
+      value: "1704981602000000000"
+  image: ghcr.io/nais/testapp/testapp:2020-02-25-f61e7b7
+  liveness:
+    path: /ping
+  port: 8080
+  prometheus:
+    enabled: true
+    path: /metrics
+  readiness:
+    path: /ping
+  replicas:
+    max: 1
+    min: 1
+  resources:
+    limits:
+      cpu: 250m
+      memory: 256Mi
+    requests:
+      cpu: 100m
+      memory: 128Mi
+  skipCaBundle: true
+status:
+  conditions:
+    - lastTransitionTime: "2024-01-11T14:00:04Z"
+      message: complete
+      reason: RolloutComplete
+      status: "True"
+      type: Ready
+    - lastTransitionTime: "2024-01-11T14:00:04Z"
+      message: complete
+      reason: RolloutComplete
+      status: "False"
+      type: Stalled
+    - lastTransitionTime: "2024-01-11T14:00:04Z"
+      message: complete
+      reason: RolloutComplete
+      status: "False"
+      type: Reconciling
+  correlationID: f8c04f82-6a84-4a8e-9f8b-563b5894d0cf
+  deploymentRolloutStatus: complete
+  rolloutCompleteTime: 1704981612597504354
+  synchronizationHash: 7fc5fa83f2ae4eaa
+  synchronizationState: RolloutComplete
+  synchronizationTime: 1704981603962494011

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/jackc/pgx/v5 v5.5.3
 	github.com/joho/godotenv v1.5.1
 	github.com/lithammer/fuzzysearch v1.1.8
-	github.com/nais/dependencytrack v0.0.0-20240131225920-a6a8df4c0ad2
+	github.com/nais/dependencytrack v0.0.0-20240208124913-513b3ad1d961
 	github.com/nais/liberator v0.0.0-20240208114703-a4ddc1dd2ffa
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pressly/goose/v3 v3.18.0
@@ -80,6 +80,7 @@ require (
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
 	github.com/fatih/structtag v1.2.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/frankban/quicktest v1.14.6 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-jose/go-jose/v3 v3.0.1 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
@@ -147,6 +148,7 @@ require (
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/riza-io/grpc-go v0.2.0 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/rs/zerolog v1.30.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sagikazarmark/locafero v0.3.0 // indirect

go.sum

@@ -116,6 +116,7 @@ github.com/coreos/go-oidc/v3 v3.9.0/go.mod h1:rTKz2PYwftcrtoCzV5g5kvfJoWcm0Mk8AF
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cubicdaiya/gonp v1.0.4 h1:ky2uIAJh81WiLcGKBVD5R7KsM/36W6IqqTy6Bo6rGws=
 github.com/cubicdaiya/gonp v1.0.4/go.mod h1:iWGuP/7+JVTn02OWhRemVbMmG1DOUnmrGTYYACpOI0I=
 github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
@@ -162,8 +163,8 @@ github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4
 github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
-github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA=
@@ -400,8 +401,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/nais/dependencytrack v0.0.0-20240131225920-a6a8df4c0ad2 h1:4LNq0LISYLyJSC3hAxNCe1YM0QU7y5QiAbLIMeXVZ7o=
-github.com/nais/dependencytrack v0.0.0-20240131225920-a6a8df4c0ad2/go.mod h1:zC59yj0La1TS291o/9XSVZ0XAY9UlJS6pUFD3ouHmjU=
+github.com/nais/dependencytrack v0.0.0-20240208124913-513b3ad1d961 h1:FtfEwEQq2hWEZFmg3F8Ix9vq6x8o0PF4bEvCn7qwCxw=
+github.com/nais/dependencytrack v0.0.0-20240208124913-513b3ad1d961/go.mod h1:zC59yj0La1TS291o/9XSVZ0XAY9UlJS6pUFD3ouHmjU=
 github.com/nais/liberator v0.0.0-20240208114703-a4ddc1dd2ffa h1:AkBC8XlZH+VvZRrHHnjiaTl/mkDRYNMRq6qpfZslUlk=
 github.com/nais/liberator v0.0.0-20240208114703-a4ddc1dd2ffa/go.mod h1:cWThp1WBBbkRFhMI2DQMvBTTEN+6GPzmmh+Xjv8vffE=
 github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY=
@@ -436,6 +437,7 @@ github.com/pingcap/log v1.1.0 h1:ELiPxACz7vdo1qAvvaWJg1NrYFoY6gqAh/+Uo6aXdD8=
 github.com/pingcap/log v1.1.0/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
 github.com/pingcap/tidb/pkg/parser v0.0.0-20231103154709-4f00ece106b1 h1:SwGY3zMnK4wO85vvRIqrR3Yh6VpIC9pydG0QNOUPHCY=
 github.com/pingcap/tidb/pkg/parser v0.0.0-20231103154709-4f00ece106b1/go.mod h1:yRkiqLFwIqibYg2P7h4bclHjHcJiIFRLKhGRyBcKYus=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -461,8 +463,9 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qq
 github.com/riza-io/grpc-go v0.2.0 h1:2HxQKFVE7VuYstcJ8zqpN84VnAoJ4dCL6YFhJewNcHQ=
 github.com/riza-io/grpc-go v0.2.0/go.mod h1:2bDvR9KkKC3KhtlSHfR3dAXjUMT86kg4UfWFyVGWqi8=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
-github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo=
 github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=

internal/cmd/api/api.go

@@ -30,6 +30,7 @@ import (
 	"github.com/nais/api/internal/thirdparty/hookd"
 	fakehookd "github.com/nais/api/internal/thirdparty/hookd/fake"
 	"github.com/nais/api/internal/usersync"
+	"github.com/nais/api/internal/vulnerability"
 	"github.com/sethvargo/go-envconfig"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/oauth2/google"
@@ -153,7 +154,7 @@ func run(ctx context.Context, cfg *Config, log logrus.FieldLogger) error {
 	pubsubTopic := pubsubClient.Topic("nais-api")
 
 	var hookdClient graph.HookdClient
-	var dependencyTrackClient graph.DependencytrackClient
+	var dependencyTrackClient vulnerability.DependencytrackClient
 	if cfg.WithFakeClients {
 		hookdClient = fakehookd.New()
 		dependencyTrackClient = faketrack.New()
@@ -214,6 +215,8 @@ func run(ctx context.Context, cfg *Config, log logrus.FieldLogger) error {
 		return costUpdater(ctx, cfg, db, log)
 	})
 
+	wg.Go(func() error { return vulnerabilityMetricUpdater(ctx, cfg, db, k8sClient, dependencyTrackClient, log) })
+
 	authHandler, err := setupAuthHandler(cfg.OAuth, db, log)
 	if err != nil {
 		return err

internal/cmd/api/config.go

@@ -131,6 +131,9 @@ type Config struct {
 	// ResourceUtilization is the configuration for the resource utilization service
 	ResourceUtilizationImportEnabled bool `env:"RESOURCE_UTILIZATION_IMPORT_ENABLED"`
 
+	// VulnerabilityMetricsImportEnabled is the configuration for the vulnerability metrics service
+	VulnerabilityMetricsImportEnabled bool `env:"VULNERABILITY_METRICS_IMPORT_ENABLED"`
+
 	// WithFakeKubernetes When set to true, the api will use a fake kubernetes client.
 	WithFakeClients bool `env:"WITH_FAKE_CLIENTS"`
 

internal/cmd/api/vulnerability_metrics_updater.go

@@ -0,0 +1,54 @@
+package api
+
+import (
+	"context"
+	"time"
+
+	"github.com/nais/api/internal/database"
+	"github.com/nais/api/internal/k8s"
+	"github.com/nais/api/internal/vulnerability"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	vulnerabilityUpdateSchedule = time.Hour
+)
+
+func vulnerabilityMetricUpdater(ctx context.Context, cfg *Config, db database.Database, k8sClient *k8s.Client, dTrackClient vulnerability.DependencytrackClient, log logrus.FieldLogger) error {
+	if !cfg.VulnerabilityMetricsImportEnabled {
+		log.Warningf(`vulnerability metrics import is not enabled. Enable by setting the "VULNERABILITY_METRICS_IMPORT_ENABLED" environment variable to "true"`)
+		return nil
+	}
+
+	vulnerabilityMetricUpdater := vulnerability.NewMetricUpdater(k8sClient, dTrackClient, db, log)
+	if err := runVulnerabilityUpdater(ctx, vulnerabilityMetricUpdater, log.WithField("task", "vulnerability_updater")); err != nil {
+		log.WithError(err).Errorf("error in vulnerability updater")
+	}
+	return nil
+}
+
+func runVulnerabilityUpdater(ctx context.Context, updater *vulnerability.Updater, log logrus.FieldLogger) error {
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+			ticker.Reset(vulnerabilityUpdateSchedule * 24) // schedule next run, same time next day
+			start := time.Now()
+			log.Infof("start scheduled vulnerability update run")
+			rows, err := updater.UpdateVulnerabilityMetrics(ctx)
+			if err != nil {
+				log = log.WithError(err)
+			}
+			log.
+				WithFields(logrus.Fields{
+					"rows_upserted": rows,
+					"duration":      time.Since(start),
+				}).
+				Infof("finished scheduled vulnerability metrics update run")
+		}
+	}
+}

internal/database/database.go

@@ -45,6 +45,7 @@ type Database interface {
 	SessionRepo
 	TeamRepo
 	UserRepo
+	VulnerabilityMetricsRepo
 
 	Transaction(ctx context.Context, fn DatabaseTransactionFunc) error
 }