From a7a0a07edf5e4dce40034fe7419b04aa9df08cc1 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Thu, 23 Mar 2023 15:02:05 +0000 Subject: [PATCH 001/192] Add HOME env var --- node-red-container/Dockerfile | 1 + node-red-container/Dockerfile-2.2.x | 1 + node-red-container/Dockerfile-3.1 | 1 + 3 files changed, 3 insertions(+) diff --git a/node-red-container/Dockerfile b/node-red-container/Dockerfile index c9512156..fc586ec8 100644 --- a/node-red-container/Dockerfile +++ b/node-red-container/Dockerfile @@ -21,6 +21,7 @@ USER node-red RUN npm install @flowforge/nr-launcher@${BUILD_TAG} ENV NODE_PATH=/usr/src/node-red +ENV HOME=/usr/src/node-red EXPOSE 2880 diff --git a/node-red-container/Dockerfile-2.2.x b/node-red-container/Dockerfile-2.2.x index d32c65f1..96b42fe5 100644 --- a/node-red-container/Dockerfile-2.2.x +++ b/node-red-container/Dockerfile-2.2.x @@ -21,6 +21,7 @@ USER node-red RUN npm install @flowforge/nr-launcher@${BUILD_TAG} ENV NODE_PATH=/usr/src/node-red +ENV HOME=/usr/src/node-red EXPOSE 2880 diff --git a/node-red-container/Dockerfile-3.1 b/node-red-container/Dockerfile-3.1 index 72b45351..97550775 100644 --- a/node-red-container/Dockerfile-3.1 +++ b/node-red-container/Dockerfile-3.1 @@ -21,6 +21,7 @@ USER node-red RUN npm install @flowforge/nr-launcher@${BUILD_TAG} ENV NODE_PATH=/usr/src/node-red +ENV HOME=/usr/src/node-red EXPOSE 2880 From 3380572e62a261e2c0b0066c410bf4d2aa1a330b Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 27 Mar 2023 13:40:07 +0100 Subject: [PATCH 002/192] Add deviceCost to configmap --- helm/flowforge/templates/configmap.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helm/flowforge/templates/configmap.yaml b/helm/flowforge/templates/configmap.yaml index 7ba61f04..1f840c7d 100644 --- a/helm/flowforge/templates/configmap.yaml +++ b/helm/flowforge/templates/configmap.yaml @@ -92,6 +92,9 @@ data: {{ if .Values.forge.ee.billing.stripe.device_product -}} device_product: {{ .Values.forge.ee.billing.stripe.device_product }} {{- end}} + {{ if .Values.forge.ee.billing.stripe.deviceCost -}} + deviceCost: {{ .Values.forge.ee.billing.stripe.deviceCost }} + {{- end}} {{ if .Values.forge.ee.billing.stripe.new_customer_free_credit -}} new_customer_free_credit: {{ .Values.forge.ee.billing.stripe.new_customer_free_credit | int }} {{- end -}} From f16aea9b46cbf51e50f53b52096a87902d9b9e07 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 27 Mar 2023 14:31:56 +0100 Subject: [PATCH 003/192] Update README.md --- helm/flowforge/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 4daa5bea..007b2950 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -87,6 +87,7 @@ Enables FlowForge Telemetry - `forge.ee.billing.stripe.project_product` Stripe product id for default Project Type - `forge.ee.billing.stripe.device_price` Stripe price id for Device (optional) - `forge.ee.billing.stripe.device_product` Stripe product id for Device (optional) + - `forge.ee.billing.stripe.deviceCost` Set the displayed price for a Device (optional) - `forge.ee.billing.stripe.new_customer_free_credit` Value in cents to be awarded as credit to new users - `forge.ee.billing.stripe.teams` a map containing Stripe Product & Price ids for named Team Types From 364738ffa3d902dc7982dbdf1106ad6faf1c2deb Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Fri, 31 Mar 2023 12:32:53 +0200 Subject: [PATCH 004/192] ingress: class name and annotations --- helm/flowforge/templates/service-ingress.yaml | 6 ++++++ helm/flowforge/values.yaml | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/helm/flowforge/templates/service-ingress.yaml b/helm/flowforge/templates/service-ingress.yaml index 62b06d52..10038fed 100644 --- a/helm/flowforge/templates/service-ingress.yaml +++ b/helm/flowforge/templates/service-ingress.yaml @@ -15,7 +15,13 @@ kind: Ingress metadata: name: flowforge-ingress annotations: + {{- if .Values.ingress.annotations }} + {{- toYaml .Values.ingress.annotations | nindent 4 }} + {{- end }} spec: + {{- if and $.Values.ingress.className (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ $.Values.ingress.className }} + {{- end }} rules: {{- if .Values.forge.entryPoint }} - host: {{ .Values.forge.entryPoint }} diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index 8043af5a..0681a29d 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -36,4 +36,7 @@ postgresql: postgresqlUsername: forge postgresqlPassword: Zai1Wied postgresqlDatabase: flowforge - \ No newline at end of file + +ingress: + annotations: "" + className: "" From d89468b44f0a339b20aeee952751865ac44087bc Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Fri, 31 Mar 2023 14:08:11 +0200 Subject: [PATCH 005/192] ingress: document in schema --- helm/flowforge/templates/deployment.yaml | 2 +- helm/flowforge/values.schema.json | 18 +++++++++++++++--- helm/flowforge/values.yaml | 5 ++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/helm/flowforge/templates/deployment.yaml b/helm/flowforge/templates/deployment.yaml index 381c762c..96bafd3b 100644 --- a/helm/flowforge/templates/deployment.yaml +++ b/helm/flowforge/templates/deployment.yaml @@ -46,7 +46,7 @@ spec: {{- end }} ports: - containerPort: 3000 - securityContext: + securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true {{- if .Values.forge.registrySecrets }} diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index dac88ba1..2361125b 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -248,8 +248,8 @@ "type", "options" ] - } - + } + }, "support": { "type": "object", @@ -295,6 +295,18 @@ "postgres" ] } + }, + "ingress": { + "type": "object", + "properties": { + "annotations": { + "type": "object", + "minProperties": 0 + }, + "className": { + "type": "string" + } + } } }, "required": [ @@ -302,4 +314,4 @@ ], "title": "Values", "type": "object" -} \ No newline at end of file +} diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index 0681a29d..ec75641d 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -1,4 +1,7 @@ forge: + domain: "TO FILL" + entryPoint: "TO FILL" + registry: "TO FILL" dbUsername: forge dbPassword: Zai1Wied dbName: flowforge @@ -38,5 +41,5 @@ postgresql: postgresqlDatabase: flowforge ingress: - annotations: "" + annotations: {} className: "" From 0210c564df3332a8ddf060cf7a7bf74640ca2e99 Mon Sep 17 00:00:00 2001 From: Elena Viter Date: Fri, 31 Mar 2023 16:31:07 +0200 Subject: [PATCH 006/192] deployment.yaml: add ingress-related environment variables (in use for ingress-ing project editors instances) --- helm/flowforge/templates/deployment.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/helm/flowforge/templates/deployment.yaml b/helm/flowforge/templates/deployment.yaml index 96bafd3b..fa831c0a 100644 --- a/helm/flowforge/templates/deployment.yaml +++ b/helm/flowforge/templates/deployment.yaml @@ -28,6 +28,14 @@ spec: {{ end -}} imagePullPolicy: Always env: + {{- if .Values.ingress.annotations }} + - name: INGRESS_ANNOTATIONS + value: {{ .Values.ingress.annotations | toJson | quote }} + {{- end }} + {{- if .Values.ingress.className }} + - name: INGRESS_CLASS_NAME + value: {{ .Values.ingress.className }} + {{- end }} {{- if .Values.forge.cloudProvider }} - name: FLOWFORGE_CLOUD_PROVIDER value: {{ .Values.forge.cloudProvider }} From adc4127e620f240692768aad4225ee5a71c375dd Mon Sep 17 00:00:00 2001 From: Elena Viter Date: Tue, 4 Apr 2023 16:09:48 +0200 Subject: [PATCH 007/192] FlowForge helm ingress settings documentation --- helm/flowforge/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 007b2950..fb2d49d6 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -104,4 +104,8 @@ Enables FlowForge Telemetry ### Private Certificate Authority - `forge.privateCA.configMapName` name of ConfigMap to store the CA Cert bundle (default `ff-ca-certs`) - - `forge.privateCA.certs` base64 encoded CA certificate PEM bundle of trusted certificates. This needs to be generated without line breaks e.g. `base64 -w 0 certs.pem` (default not set) \ No newline at end of file + - `forge.privateCA.certs` base64 encoded CA certificate PEM bundle of trusted certificates. This needs to be generated without line breaks e.g. `base64 -w 0 certs.pem` (default not set) + + ### Ingress + - `ingress.annotations` ingress annotations (default is `{}`). This value is also applied to Editor instances created by FlowForge. + - `ingress.className` ingress class name (default is `"""`). This value is also applied to Editor instances created by FlowForge. \ No newline at end of file From 2fb4518cbd3770b8eaa3e9dadcb14d30eb04d917 Mon Sep 17 00:00:00 2001 From: Elena Viter Date: Mon, 3 Apr 2023 11:09:03 +0200 Subject: [PATCH 008/192] FlowForge helm: add tolerations settings for Project instances --- helm/flowforge/README.md | 5 +++-- helm/flowforge/templates/deployment.yaml | 4 ++++ helm/flowforge/values.yaml | 6 ++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index fb2d49d6..133a7370 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -27,9 +27,10 @@ This chart uses the Bitnami PostgreSQL Chart to provide an instance of a Postgre - `forge.projectNamespace` namespace Project Pods will run in (default `flowforge`) - `forge.license` FlowForge EE license string (optional, default not set) - `forge.branding` Object holding branding inserts (default not set) - + - `forge.projectDeploymentTolerations` tolerations settings for Project instances. Default is `[]`. + note: `forge.projectSelector` and `forge.managementSelector` defaults mean that you must have at least 2 nodes in your cluster and they need to be labeled before installing. - + ### AWS If `forge.cloudProvider` is set to `aws` then the following should be set diff --git a/helm/flowforge/templates/deployment.yaml b/helm/flowforge/templates/deployment.yaml index fa831c0a..870c2e89 100644 --- a/helm/flowforge/templates/deployment.yaml +++ b/helm/flowforge/templates/deployment.yaml @@ -36,6 +36,10 @@ spec: - name: INGRESS_CLASS_NAME value: {{ .Values.ingress.className }} {{- end }} + {{- if .Values.forge.projectDeploymentTolerations }} + - name: DEPLOYMENT_TOLERATIONS + value: {{ .Values.forge.projectDeploymentTolerations | toJson | quote }} + {{- end }} {{- if .Values.forge.cloudProvider }} - name: FLOWFORGE_CLOUD_PROVIDER value: {{ .Values.forge.cloudProvider }} diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index ec75641d..691c0094 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -10,6 +10,12 @@ forge: projectNamespace: flowforge projectSelector: role: projects + + projectDeploymentTolerations: [] +# - key: purpose +# operator: Equal +# value: flowforge-projects +# effect: NoSchedule managementSelector: role: management telemetry: From 45474b715280b4fe9f4e42a9a6ddd42522c75933 Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Tue, 4 Apr 2023 16:49:52 +0200 Subject: [PATCH 009/192] values: remove defaults for domain --- helm/flowforge/values.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index ec75641d..dedf9cde 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -1,7 +1,4 @@ forge: - domain: "TO FILL" - entryPoint: "TO FILL" - registry: "TO FILL" dbUsername: forge dbPassword: Zai1Wied dbName: flowforge From a291db7fccb562916582a1c95fa1b787792d53fe Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Thu, 13 Apr 2023 11:44:21 +0100 Subject: [PATCH 010/192] Update CHANGELOG for 1.6.0 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03c7a7e7..91e2f6f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +#### 1.6.0: Release + + - Fix Release.Name in job-upgrade-db (#122) @flecoufle + - Add deviceCost to configmap (#124) @hardillb + - Add HOME env var (#123) @hardillb + #### 1.5.0: Release - Add NR 3.1.0 build (#120) @hardillb From 58a736fb5731159f0bf3c3e4c70897793d0399bd Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Thu, 13 Apr 2023 11:44:26 +0100 Subject: [PATCH 011/192] Update version to 1.6.0 --- VERSION | 2 +- file-server/package.json | 4 ++-- flowforge-container/package.json | 6 +++--- helm/flowforge/Chart.yaml | 4 ++-- node-red-container/package.json | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/VERSION b/VERSION index bc80560f..dc1e644a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.0 +1.6.0 diff --git a/file-server/package.json b/file-server/package.json index 785082ae..f3730a71 100644 --- a/file-server/package.json +++ b/file-server/package.json @@ -1,9 +1,9 @@ { "name": "@flowforge/file-server-container", - "version": "1.5.0", + "version": "1.6.0", "private": true, "dependencies": { - "@flowforge/file-server": "^1.5.0" + "@flowforge/file-server": "^1.6.0" }, "license": "Apache-2.0" } diff --git a/flowforge-container/package.json b/flowforge-container/package.json index 80db3334..51ebb463 100644 --- a/flowforge-container/package.json +++ b/flowforge-container/package.json @@ -1,14 +1,14 @@ { "name": "@flowforge/flowforge-k8s", - "version": "1.5.0", + "version": "1.6.0", "description": "FlowForge in Docker", "private": true, "author": { "name": "FlowForge Inc." }, "dependencies": { - "@flowforge/flowforge": "^1.5.0", - "@flowforge/kubernetes": "^1.5.0", + "@flowforge/flowforge": "^1.6.0", + "@flowforge/kubernetes": "^1.6.0", "pg": "^8.7.1", "pg-hstore": "^2.3.4" }, diff --git a/helm/flowforge/Chart.yaml b/helm/flowforge/Chart.yaml index 6cfce374..d76cc844 100644 --- a/helm/flowforge/Chart.yaml +++ b/helm/flowforge/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "flowforge" -version: "1.5.0" +version: "1.6.0" description: "FlowForge" type: "application" home: "https://flowforge.com" @@ -16,4 +16,4 @@ dependencies: maintainers: - name: "FlowForge Inc" url: "https://flowforge.com" -appVersion: "1.5.0" +appVersion: "1.6.0" diff --git a/node-red-container/package.json b/node-red-container/package.json index e0525267..d4faec3c 100644 --- a/node-red-container/package.json +++ b/node-red-container/package.json @@ -1,7 +1,7 @@ { "name": "node-red-project", "description": "A Node-RED Project", - "version": "1.5.0", + "version": "1.6.0", "private": true, "dependencies": {} } From e67d365f67acb10dd004dd007ea309f283d5d8bd Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 9 May 2023 11:13:40 +0100 Subject: [PATCH 012/192] Update helm/flowforge/values.yaml --- helm/flowforge/values.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index 691c0094..a464fa61 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -1,7 +1,4 @@ forge: - domain: "TO FILL" - entryPoint: "TO FILL" - registry: "TO FILL" dbUsername: forge dbPassword: Zai1Wied dbName: flowforge From 39c312283746507391a44a8a39aa72422a52c427 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 9 May 2023 11:15:04 +0100 Subject: [PATCH 013/192] Update helm/flowforge/values.schema.json --- helm/flowforge/values.schema.json | 1 - 1 file changed, 1 deletion(-) diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index 2361125b..ae749c8f 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -249,7 +249,6 @@ "options" ] } - }, "support": { "type": "object", From 1ad391c0299ed8abe281914e9ebca9e568826e80 Mon Sep 17 00:00:00 2001 From: Steve-Mcl Date: Thu, 11 May 2023 10:23:30 +0100 Subject: [PATCH 014/192] Update CHANGELOG for 1.7.0 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91e2f6f8..7ba70b38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +#### 1.7.0: Release + + - Editor ingress settings (#126) @elenaviter + - Ingress settings for Flowforge in K8s (#125) @andreikop + #### 1.6.0: Release - Fix Release.Name in job-upgrade-db (#122) @flecoufle From b7ec9649140e40ffdae8350be9417b5fea962da5 Mon Sep 17 00:00:00 2001 From: Steve-Mcl Date: Thu, 11 May 2023 10:23:30 +0100 Subject: [PATCH 015/192] Update version to 1.7.0 --- VERSION | 2 +- file-server/package.json | 4 ++-- flowforge-container/package.json | 6 +++--- helm/flowforge/Chart.yaml | 4 ++-- node-red-container/package.json | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/VERSION b/VERSION index dc1e644a..bd8bf882 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.6.0 +1.7.0 diff --git a/file-server/package.json b/file-server/package.json index f3730a71..147a8933 100644 --- a/file-server/package.json +++ b/file-server/package.json @@ -1,9 +1,9 @@ { "name": "@flowforge/file-server-container", - "version": "1.6.0", + "version": "1.7.0", "private": true, "dependencies": { - "@flowforge/file-server": "^1.6.0" + "@flowforge/file-server": "^1.7.0" }, "license": "Apache-2.0" } diff --git a/flowforge-container/package.json b/flowforge-container/package.json index 51ebb463..b8c82b99 100644 --- a/flowforge-container/package.json +++ b/flowforge-container/package.json @@ -1,14 +1,14 @@ { "name": "@flowforge/flowforge-k8s", - "version": "1.6.0", + "version": "1.7.0", "description": "FlowForge in Docker", "private": true, "author": { "name": "FlowForge Inc." }, "dependencies": { - "@flowforge/flowforge": "^1.6.0", - "@flowforge/kubernetes": "^1.6.0", + "@flowforge/flowforge": "^1.7.0", + "@flowforge/kubernetes": "^1.7.0", "pg": "^8.7.1", "pg-hstore": "^2.3.4" }, diff --git a/helm/flowforge/Chart.yaml b/helm/flowforge/Chart.yaml index d76cc844..9c34cc7b 100644 --- a/helm/flowforge/Chart.yaml +++ b/helm/flowforge/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "flowforge" -version: "1.6.0" +version: "1.7.0" description: "FlowForge" type: "application" home: "https://flowforge.com" @@ -16,4 +16,4 @@ dependencies: maintainers: - name: "FlowForge Inc" url: "https://flowforge.com" -appVersion: "1.6.0" +appVersion: "1.7.0" diff --git a/node-red-container/package.json b/node-red-container/package.json index d4faec3c..7592ee85 100644 --- a/node-red-container/package.json +++ b/node-red-container/package.json @@ -1,7 +1,7 @@ { "name": "node-red-project", "description": "A Node-RED Project", - "version": "1.6.0", + "version": "1.7.0", "private": true, "dependencies": {} } From fce511bb1821773695c8fb8ca89bed56a8ab11cb Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Fri, 26 May 2023 13:25:56 +0100 Subject: [PATCH 016/192] Add permission to list endpoints This is to find all replicas --- helm/flowforge/templates/service-account.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helm/flowforge/templates/service-account.yaml b/helm/flowforge/templates/service-account.yaml index d455f68a..2b244db4 100644 --- a/helm/flowforge/templates/service-account.yaml +++ b/helm/flowforge/templates/service-account.yaml @@ -27,6 +27,9 @@ rules: - apiGroups: [""] resources: ["services"] verbs: ["create", "patch", "get", "list", "watch", "delete"] +- apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list"] - apiGroups: ["networking.k8s.io"] resources: ["ingresses"] verbs: ["create", "patch", "get", "list", "watch", "delete"] From c1ab8f1d9b236a53824860433ec95973df48bdef Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Wed, 31 May 2023 09:20:55 +0100 Subject: [PATCH 017/192] Update Dockerfile-3.1 Bump to beta.3 --- node-red-container/Dockerfile-3.1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node-red-container/Dockerfile-3.1 b/node-red-container/Dockerfile-3.1 index 97550775..d221a943 100644 --- a/node-red-container/Dockerfile-3.1 +++ b/node-red-container/Dockerfile-3.1 @@ -1,4 +1,4 @@ -FROM nodered/node-red-dev:v3.1.0-beta.2 +FROM nodered/node-red-dev:v3.1.0-beta.3 ARG REGISTRY ARG REGISTRY_TOKEN @@ -25,4 +25,4 @@ ENV HOME=/usr/src/node-red EXPOSE 2880 -ENTRYPOINT ["./node_modules/.bin/flowforge-node-red", "-p", "2880", "-n", "/usr/src/node-red"] \ No newline at end of file +ENTRYPOINT ["./node_modules/.bin/flowforge-node-red", "-p", "2880", "-n", "/usr/src/node-red"] From 06a6e80ace1408de430d936d9259735f2f5eee0c Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 5 Jun 2023 15:44:31 +0100 Subject: [PATCH 018/192] Update Dockerfile-3.1 Make the 3.1.0-beta.3 build based on NodeJS 18 --- node-red-container/Dockerfile-3.1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node-red-container/Dockerfile-3.1 b/node-red-container/Dockerfile-3.1 index d221a943..9634714e 100644 --- a/node-red-container/Dockerfile-3.1 +++ b/node-red-container/Dockerfile-3.1 @@ -1,4 +1,4 @@ -FROM nodered/node-red-dev:v3.1.0-beta.3 +FROM nodered/node-red-dev:3.1.0-beta.3-18 ARG REGISTRY ARG REGISTRY_TOKEN From 06832eb08b089ff865246aa2dfc2971c09d6caa5 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 6 Jun 2023 13:43:36 +0100 Subject: [PATCH 019/192] Update README.md Fix mount and container name --- flowforge-device-agent/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowforge-device-agent/README.md b/flowforge-device-agent/README.md index b07bbaf5..5647384c 100644 --- a/flowforge-device-agent/README.md +++ b/flowforge-device-agent/README.md @@ -6,6 +6,6 @@ be [registered on your FlowForge instance](https://flowforge.com/docs/user/devic The YAML with configuration needs to be mounted inside the container. ``` -docker run --mount /path/to/device.yml:/opt/flowforge-device/device.yml -p 1880:1880 flowforge-device-agent:latest +docker run -v /path/to/device.yml:/opt/flowforge-device/device.yml -p 1880:1880 flowforge/flowforge-device-agent:latest ``` From e121c65dcb6ba37336ec6eebb68bb2bb6b5958b5 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 6 Jun 2023 13:45:46 +0100 Subject: [PATCH 020/192] Add details for verbose logging --- flowforge-device-agent/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flowforge-device-agent/README.md b/flowforge-device-agent/README.md index 5647384c..e989e56a 100644 --- a/flowforge-device-agent/README.md +++ b/flowforge-device-agent/README.md @@ -9,3 +9,8 @@ The YAML with configuration needs to be mounted inside the container. docker run -v /path/to/device.yml:/opt/flowforge-device/device.yml -p 1880:1880 flowforge/flowforge-device-agent:latest ``` +To run with verbose logging run as follows: + +``` +docker run -v /path/to/device.yml:/opt/flowforge-device/device.yml -p 1880:1880 flowforge/flowforge-device-agent:latest flowforge-device-agent -v +``` \ No newline at end of file From 09f13fb2e1a405d575b5de13798902d1d4440f64 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Wed, 7 Jun 2023 11:05:09 +0100 Subject: [PATCH 021/192] Apply suggestions from code review --- flowforge-device-agent/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flowforge-device-agent/README.md b/flowforge-device-agent/README.md index e989e56a..d6e9c9c9 100644 --- a/flowforge-device-agent/README.md +++ b/flowforge-device-agent/README.md @@ -6,11 +6,11 @@ be [registered on your FlowForge instance](https://flowforge.com/docs/user/devic The YAML with configuration needs to be mounted inside the container. ``` -docker run -v /path/to/device.yml:/opt/flowforge-device/device.yml -p 1880:1880 flowforge/flowforge-device-agent:latest +docker run -v /path/to/device.yml:/opt/flowforge-device/device.yml -p 1880:1880 flowforge/device-agent:latest ``` To run with verbose logging run as follows: ``` -docker run -v /path/to/device.yml:/opt/flowforge-device/device.yml -p 1880:1880 flowforge/flowforge-device-agent:latest flowforge-device-agent -v +docker run -v /path/to/device.yml:/opt/flowforge-device/device.yml -p 1880:1880 flowforge/device-agent:latest flowforge-device-agent -v ``` \ No newline at end of file From 9c16ad8049a71cb85be694c35e1083b497e7933a Mon Sep 17 00:00:00 2001 From: MarianRapahel <73583313+MarianRaphael@users.noreply.github.com> Date: Thu, 8 Jun 2023 11:06:24 +0200 Subject: [PATCH 022/192] Update CHANGELOG for 1.8.0 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ba70b38..964ab049 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +#### 1.8.0: Release + + - Update README.md (#137) @hardillb + - Update Dockerfile-3.1 (#136) @hardillb + - Update Dockerfile-3.1 (#134) @hardillb + - Add permission to list endpoints (#133) @hardillb + #### 1.7.0: Release - Editor ingress settings (#126) @elenaviter From 3d547076027b4291b8989634e6f5be88c59b1469 Mon Sep 17 00:00:00 2001 From: MarianRapahel <73583313+MarianRaphael@users.noreply.github.com> Date: Thu, 8 Jun 2023 11:06:24 +0200 Subject: [PATCH 023/192] Update version to 1.8.0 --- VERSION | 2 +- file-server/package.json | 4 ++-- flowforge-container/package.json | 6 +++--- helm/flowforge/Chart.yaml | 4 ++-- node-red-container/package.json | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/VERSION b/VERSION index bd8bf882..27f9cd32 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.7.0 +1.8.0 diff --git a/file-server/package.json b/file-server/package.json index 147a8933..503676a1 100644 --- a/file-server/package.json +++ b/file-server/package.json @@ -1,9 +1,9 @@ { "name": "@flowforge/file-server-container", - "version": "1.7.0", + "version": "1.8.0", "private": true, "dependencies": { - "@flowforge/file-server": "^1.7.0" + "@flowforge/file-server": "^1.8.0" }, "license": "Apache-2.0" } diff --git a/flowforge-container/package.json b/flowforge-container/package.json index b8c82b99..1546c9ff 100644 --- a/flowforge-container/package.json +++ b/flowforge-container/package.json @@ -1,14 +1,14 @@ { "name": "@flowforge/flowforge-k8s", - "version": "1.7.0", + "version": "1.8.0", "description": "FlowForge in Docker", "private": true, "author": { "name": "FlowForge Inc." }, "dependencies": { - "@flowforge/flowforge": "^1.7.0", - "@flowforge/kubernetes": "^1.7.0", + "@flowforge/flowforge": "^1.8.0", + "@flowforge/kubernetes": "^1.8.0", "pg": "^8.7.1", "pg-hstore": "^2.3.4" }, diff --git a/helm/flowforge/Chart.yaml b/helm/flowforge/Chart.yaml index 9c34cc7b..22d4ce11 100644 --- a/helm/flowforge/Chart.yaml +++ b/helm/flowforge/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "flowforge" -version: "1.7.0" +version: "1.8.0" description: "FlowForge" type: "application" home: "https://flowforge.com" @@ -16,4 +16,4 @@ dependencies: maintainers: - name: "FlowForge Inc" url: "https://flowforge.com" -appVersion: "1.7.0" +appVersion: "1.8.0" diff --git a/node-red-container/package.json b/node-red-container/package.json index 7592ee85..2eee03c6 100644 --- a/node-red-container/package.json +++ b/node-red-container/package.json @@ -1,7 +1,7 @@ { "name": "node-red-project", "description": "A Node-RED Project", - "version": "1.7.0", + "version": "1.8.0", "private": true, "dependencies": {} } From 736f0e770cc01124dd09f6440bd4a7c867f201cc Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Fri, 9 Jun 2023 13:30:51 +0100 Subject: [PATCH 024/192] Remove device-agent container build part of flowforge/flowforge-device-agent#103 --- .github/workflows/build-containers.yml | 45 +------------------------- flowforge-device-agent/Dockerfile | 8 ----- flowforge-device-agent/README.md | 16 --------- 3 files changed, 1 insertion(+), 68 deletions(-) delete mode 100644 flowforge-device-agent/Dockerfile delete mode 100644 flowforge-device-agent/README.md diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 539276bc..a086686a 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -206,51 +206,8 @@ jobs: password: ${{ secrets.DOCKER_HUB_PASSWORD }} readme-filepath: ./helm/file-server/README.md - - build_device_agent_container: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - repository: 'flowforge/helm' - path: 'helm' - - name: Docker Meta Data - id: meta - uses: docker/metadata-action@v3 - with: - tags: | - type=semver,event=tag,pattern={{version}} - flavor: | - latest=true - images: | - flowforge/device-agent - - name: Setup QEMU - uses: docker/setup-qemu-action@v1 - - name: Setup Docker buildx - uses: docker/setup-buildx-action@v1 - - name: docker login - uses: docker/login-action@v1 - with: - username: flowforge - password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - name: Build and push FlowForge Application container - uses: docker/build-push-action@v2 - with: - context: helm/flowforge-device-agent - file: helm/flowforge-device-agent/Dockerfile - platforms: linux/amd64, linux/arm64, linux/arm/v7 - tags: ${{ steps.meta.outputs.tags }} - push: true - - name: Push README - uses: peter-evans/dockerhub-description@v3 - with: - repository: flowforge/device-agent - username: flowforge - password: ${{ secrets.DOCKER_HUB_PASSWORD }} - readme-filepath: ./helm/flowforge-device-agent/README.md - publish_helm: - needs: [build_application_container, build_nodered_container, build_nodered_container_223, build_file_server_container, build_device_agent_container] + needs: [build_application_container, build_nodered_container, build_nodered_container_223, build_file_server_container] runs-on: ubuntu-latest steps: - name: Install Helm diff --git a/flowforge-device-agent/Dockerfile b/flowforge-device-agent/Dockerfile deleted file mode 100644 index 0a999e35..00000000 --- a/flowforge-device-agent/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM node:16-alpine - -RUN mkdir /opt/flowforge-device -RUN npm install -g @flowforge/flowforge-device-agent - -EXPOSE 1880 - -CMD ["flowforge-device-agent"] diff --git a/flowforge-device-agent/README.md b/flowforge-device-agent/README.md deleted file mode 100644 index d6e9c9c9..00000000 --- a/flowforge-device-agent/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# FlowForge Device Agent - -This container can be used to start a FlowForge device. The device needs to -be [registered on your FlowForge instance](https://flowforge.com/docs/user/devices/#register-the-device). - -The YAML with configuration needs to be mounted inside the container. - -``` -docker run -v /path/to/device.yml:/opt/flowforge-device/device.yml -p 1880:1880 flowforge/device-agent:latest -``` - -To run with verbose logging run as follows: - -``` -docker run -v /path/to/device.yml:/opt/flowforge-device/device.yml -p 1880:1880 flowforge/device-agent:latest flowforge-device-agent -v -``` \ No newline at end of file From db293de83a4694173c0d2952e67bc043a9bb7830 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 3 Jul 2023 09:54:09 +0100 Subject: [PATCH 025/192] Add comment about PostgreSQL constraints PostgreSQL can only be installed on x86_64 at the moment. part of #144 --- helm/flowforge/README.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 133a7370..c9a561a9 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -4,7 +4,18 @@ Access to FlowForge Management App via the host `forge` on what ever domain is p ## Database -This chart uses the Bitnami PostgreSQL Chart to provide an instance of a PostgreSQL Database to store state. +This chart can use the Bitnami PostgreSQL Chart to provide an instance of a PostgreSQL Database to store state (`forge.localPostgresql: true`). + +The chart is currently pinned at the Bitanmi PostgreSQL v14 release, which only supports x86_64 deployments when +using a local database + +If using an external PostgreSQL Database you will need to create the database and user to pass to the helm chart using the following values: + +- `forge.dbName` +- `forge.dbUsername` +- `forge.dbPassword` +- `forge.postgres.host` +- `forge.postgres.port` ## Configuration Values @@ -18,7 +29,7 @@ This chart uses the Bitnami PostgreSQL Chart to provide an instance of a Postgre - `forge.dbUsername` (default `forge`) - `forge.dbPassword` (default `Zai1Wied`) - `forge.dbName` (default `flowforge`) - - `forge.localPostrgresql` Deploy a PostgreSQL Database into Kubernetes(default `true`) + - `forge.localPostrgresql` Deploy a PostgreSQL v14 Database into Kubernetes cluster (default `true`) - `forge.postgres.host` the hostname of an external PostgreSQL database (default not set) - `forge.postgres.port` the port of an external PostgreSQL database (default `5432`) - `forge.cloudProvider` currently only accepts `aws` but will include more as needed (default not set) @@ -109,4 +120,4 @@ Enables FlowForge Telemetry ### Ingress - `ingress.annotations` ingress annotations (default is `{}`). This value is also applied to Editor instances created by FlowForge. - - `ingress.className` ingress class name (default is `"""`). This value is also applied to Editor instances created by FlowForge. \ No newline at end of file + - `ingress.className` ingress class name (default is `"""`). This value is also applied to Editor instances created by FlowForge. From b9e756effcca0290ada5dffb240c29bc51c5e67f Mon Sep 17 00:00:00 2001 From: Pez Cuckow Date: Thu, 6 Jul 2023 10:18:03 +0200 Subject: [PATCH 026/192] Update CHANGELOG for 1.9.0 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 964ab049..1e3cb86c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +#### 1.9.0: Release + + - Add comment about PostgreSQL constraints (#145) @hardillb + - Remove device-agent container build (#139) @hardillb + #### 1.8.0: Release - Update README.md (#137) @hardillb From a4c6932f7db22f443ac7e6ed463c6a5e40083aab Mon Sep 17 00:00:00 2001 From: Pez Cuckow Date: Thu, 6 Jul 2023 10:18:03 +0200 Subject: [PATCH 027/192] Update version to 1.9.0 --- VERSION | 2 +- file-server/package.json | 4 ++-- flowforge-container/package.json | 6 +++--- helm/flowforge/Chart.yaml | 4 ++-- node-red-container/package.json | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/VERSION b/VERSION index 27f9cd32..f8e233b2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.8.0 +1.9.0 diff --git a/file-server/package.json b/file-server/package.json index 503676a1..0b249795 100644 --- a/file-server/package.json +++ b/file-server/package.json @@ -1,9 +1,9 @@ { "name": "@flowforge/file-server-container", - "version": "1.8.0", + "version": "1.9.0", "private": true, "dependencies": { - "@flowforge/file-server": "^1.8.0" + "@flowforge/file-server": "^1.9.0" }, "license": "Apache-2.0" } diff --git a/flowforge-container/package.json b/flowforge-container/package.json index 1546c9ff..94780eef 100644 --- a/flowforge-container/package.json +++ b/flowforge-container/package.json @@ -1,14 +1,14 @@ { "name": "@flowforge/flowforge-k8s", - "version": "1.8.0", + "version": "1.9.0", "description": "FlowForge in Docker", "private": true, "author": { "name": "FlowForge Inc." }, "dependencies": { - "@flowforge/flowforge": "^1.8.0", - "@flowforge/kubernetes": "^1.8.0", + "@flowforge/flowforge": "^1.9.0", + "@flowforge/kubernetes": "^1.9.0", "pg": "^8.7.1", "pg-hstore": "^2.3.4" }, diff --git a/helm/flowforge/Chart.yaml b/helm/flowforge/Chart.yaml index 22d4ce11..65848cd0 100644 --- a/helm/flowforge/Chart.yaml +++ b/helm/flowforge/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "flowforge" -version: "1.8.0" +version: "1.9.0" description: "FlowForge" type: "application" home: "https://flowforge.com" @@ -16,4 +16,4 @@ dependencies: maintainers: - name: "FlowForge Inc" url: "https://flowforge.com" -appVersion: "1.8.0" +appVersion: "1.9.0" diff --git a/node-red-container/package.json b/node-red-container/package.json index 2eee03c6..5c2343b7 100644 --- a/node-red-container/package.json +++ b/node-red-container/package.json @@ -1,7 +1,7 @@ { "name": "node-red-project", "description": "A Node-RED Project", - "version": "1.8.0", + "version": "1.9.0", "private": true, "dependencies": {} } From 51a8c9b2fe18112de44d50dd5b67ab772403321a Mon Sep 17 00:00:00 2001 From: Elena Viter Date: Thu, 6 Jul 2023 15:18:50 +0200 Subject: [PATCH 028/192] FlowForge helm: 1. Editors: optional service account provisioning. 2. broker: propagate ingress definitions to broker helm. 3. remove secrets from referent values.yml, adding ref for service account definition. 4. Update README.md with IAM section --- helm/flowforge/README.md | 17 ++++++++++ helm/flowforge/templates/broker.yaml | 6 ++++ helm/flowforge/templates/deployment.yaml | 4 +++ helm/flowforge/templates/service-account.yaml | 20 +++++++++-- helm/flowforge/values.yaml | 34 +++++++++++++------ 5 files changed, 69 insertions(+), 12 deletions(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index c9a561a9..f4cf472d 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -121,3 +121,20 @@ Enables FlowForge Telemetry ### Ingress - `ingress.annotations` ingress annotations (default is `{}`). This value is also applied to Editor instances created by FlowForge. - `ingress.className` ingress class name (default is `"""`). This value is also applied to Editor instances created by FlowForge. + +### Editors IAM + Provision default service account for Editors if `editors.serviceAccount.create` is `true`. + +- `editors.serviceAccount.create` flag, indicates whether default Editors service account is going to be provisioned. +- `editors.serviceAccount.annotations` k8s service account annotations. +- `editors.serviceAccount.name` name of the service account for Editors. + +Example for AWS: +```yaml +editors: + serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::${ACCOUNT_ID}:role/${ROLE_NAME} + create: true + name: editors +``` diff --git a/helm/flowforge/templates/broker.yaml b/helm/flowforge/templates/broker.yaml index a4b1a606..28f02a2d 100644 --- a/helm/flowforge/templates/broker.yaml +++ b/helm/flowforge/templates/broker.yaml @@ -131,7 +131,13 @@ metadata: labels: app: flowforge-broker annotations: + {{- if .Values.ingress.annotations }} + {{- toYaml .Values.ingress.annotations | nindent 4 }} + {{- end }} spec: + {{- if $.Values.ingress.className }} + ingressClassName: {{ $.Values.ingress.className }} + {{- end }} rules: - host: mqtt.{{ .Values.forge.domain }} http: diff --git a/helm/flowforge/templates/deployment.yaml b/helm/flowforge/templates/deployment.yaml index 870c2e89..f10a14a3 100644 --- a/helm/flowforge/templates/deployment.yaml +++ b/helm/flowforge/templates/deployment.yaml @@ -36,6 +36,10 @@ spec: - name: INGRESS_CLASS_NAME value: {{ .Values.ingress.className }} {{- end }} + {{- if .Values.editors.serviceAccount }} + - name: EDITOR_SERVICE_ACCOUNT + value: {{ .Values.editors.serviceAccount.name }} + {{- end }} {{- if .Values.forge.projectDeploymentTolerations }} - name: DEPLOYMENT_TOLERATIONS value: {{ .Values.forge.projectDeploymentTolerations | toJson | quote }} diff --git a/helm/flowforge/templates/service-account.yaml b/helm/flowforge/templates/service-account.yaml index 2b244db4..a9ec261f 100644 --- a/helm/flowforge/templates/service-account.yaml +++ b/helm/flowforge/templates/service-account.yaml @@ -9,11 +9,27 @@ metadata: eks.amazonaws.com/sts-regional-endpoints: "true" {{- end }} {{- end }} + + +{{- if .Values.editors.serviceAccount.create }} --- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.editors.serviceAccount.name }} + namespace: {{ .Values.forge.projectNamespace }} + {{- with .Values.editors.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + +--- + apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: create-pod + name: {{ .Release.Name }}-create-pod rules: - apiGroups: [""] resources: ["pods", "pods/log", "pods/exec", "pods/status"] @@ -45,5 +61,5 @@ subjects: namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole - name: create-pod + name: {{ .Release.Name }}-create-pod apiGroup: rbac.authorization.k8s.io diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index a464fa61..025adeeb 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -1,18 +1,17 @@ forge: dbUsername: forge - dbPassword: Zai1Wied + dbPassword: "" dbName: flowforge localPostgresql: true https: true projectNamespace: flowforge projectSelector: role: projects - projectDeploymentTolerations: [] -# - key: purpose -# operator: Equal -# value: flowforge-projects -# effect: NoSchedule + # - key: purpose + # operator: Equal + # value: flowforge-projects + # effect: NoSchedule managementSelector: role: management telemetry: @@ -32,17 +31,32 @@ forge: type: postgres host: flowforge-postgresql username: forge - password: Zai1Wied + password: "" database: ff-context support: enabled: false + domain: "" + entryPoint: "" + environment: {} + image: 355908013639.dkr.ecr.eu-west-1.amazonaws.com/flowforge/forge-k8s:1.5.0 + registry: 355908013639.dkr.ecr.eu-west-1.amazonaws.com + postgresql: - postgresqlPostgresPassword: Moomiet0 - postgresqlUsername: forge - postgresqlPassword: Zai1Wied postgresqlDatabase: flowforge + postgresqlPassword: "" + postgresqlPostgresPassword: "" + postgresqlUsername: forge + global: + storageClass: default ingress: annotations: {} className: "" + +editors: + serviceAccount: + create: true + annotations: {} + name: editors + namespace: \ No newline at end of file From 3c60935e371db6475d89a28e6d80eaa3e06db4be Mon Sep 17 00:00:00 2001 From: Elena Viter Date: Thu, 6 Jul 2023 15:45:51 +0200 Subject: [PATCH 029/192] values.yaml: remove non-needed attribite editors.serviceAccount.namespace --- helm/flowforge/values.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index 025adeeb..40b88cad 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -58,5 +58,4 @@ editors: serviceAccount: create: true annotations: {} - name: editors - namespace: \ No newline at end of file + name: editors \ No newline at end of file From 1506a49c35a7fda9e4f775dcdce7f58494395dfb Mon Sep 17 00:00:00 2001 From: Elena Viter Date: Thu, 6 Jul 2023 18:30:12 +0200 Subject: [PATCH 030/192] revert the default secrets in values.yaml --- helm/flowforge/values.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index 40b88cad..dca54c37 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -1,6 +1,6 @@ forge: dbUsername: forge - dbPassword: "" + dbPassword: Zai1Wied dbName: flowforge localPostgresql: true https: true @@ -39,14 +39,14 @@ forge: domain: "" entryPoint: "" environment: {} - image: 355908013639.dkr.ecr.eu-west-1.amazonaws.com/flowforge/forge-k8s:1.5.0 - registry: 355908013639.dkr.ecr.eu-west-1.amazonaws.com + image: "" + registry: "" postgresql: - postgresqlDatabase: flowforge - postgresqlPassword: "" - postgresqlPostgresPassword: "" + postgresqlPostgresPassword: Moomiet0 postgresqlUsername: forge + postgresqlPassword: Zai1Wied + postgresqlDatabase: flowforge global: storageClass: default From 0e6f54274ba06e49cd4719f5a33335e7329a335e Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 17 Jul 2023 14:18:45 +0200 Subject: [PATCH 031/192] Introduce flowforge container build pipeline --- .github/workflows/flowforge-container.yml | 63 +++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/flowforge-container.yml diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml new file mode 100644 index 00000000..0fbd0422 --- /dev/null +++ b/.github/workflows/flowforge-container.yml @@ -0,0 +1,63 @@ +name: Build Flowforge container +on: + workflow_dispatch: + schedule: + - cron: '30 23 * * *' + push: + branches: + - 'feat-*' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set nigthly release + # if: ${{ github.event_name == 'schedule' }} + run: echo "release_name=nightly" >> $GITHUB_ENV + + - name: "Set FF versions" + run: | + cat flowforge-container/package.json | jq '.dependencies["@flowforge/flowforge"] = "${{ env.release_name }}" | .dependencies["@flowforge/kubernetes"] = "${{ env.release_name }}" ' > flowforge-container/package.json-patched + mv flowforge-container/package.json-patched flowforge-container/package.json + + - name: Configure AWS credentials + if: ${{ env.release_name == 'nightly' }} + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-access-key-id: ${{ secrets.STAGING_AWS_ID }} + aws-secret-access-key: ${{ secrets.STAGING_AWS_KEY }} + aws-region: eu-west-1 + + - name: "Login to Staging ECR" + if: ${{ env.release_name == 'nightly' }} + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Get image metadata + id: image_metadata + uses: docker/metadata-action@v3 + with: + tags: | + type=raw,value=${{ env.release_name }} + flavor: | + latest=false + images: | + ${{ steps.login-ecr.outputs.registry }}/flowforge/forge-k8s + + - name: Build and push Forge container + uses: docker/build-push-action@v3.1.1 + with: + context: flowforge-container + file: flowforge-container/Dockerfile + tags: ${{ steps.image_metadata.outputs.tags }} + platforms: | + linux/amd64 + linux/arm64 + build-args: | + REGISTRY=npm.pkg.github.com + REGISTRY_TOKEN=${{ secrets.GITHUB_TOKEN }} + push: true + \ No newline at end of file From 9e477bc157127d1272aaf1b70b5765fe2a2a2d34 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 17 Jul 2023 14:26:42 +0200 Subject: [PATCH 032/192] Add buildx setup --- .github/workflows/flowforge-container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 0fbd0422..e691d542 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -11,8 +11,8 @@ jobs: build: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v3 + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v2 - name: Set nigthly release # if: ${{ github.event_name == 'schedule' }} From a58d542b4d959ac87915fe04406e9d0b2fa65b74 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 17 Jul 2023 14:39:39 +0200 Subject: [PATCH 033/192] re-add checkout step --- .github/workflows/flowforge-container.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index e691d542..83d5e604 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -11,6 +11,9 @@ jobs: build: runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Docker buildx uses: docker/setup-buildx-action@v2 From 11996020c67bf23aeb79a8a1c26809258eca35f0 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 17 Jul 2023 14:43:36 +0200 Subject: [PATCH 034/192] Add permissions --- .github/workflows/flowforge-container.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 83d5e604..946bb0da 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -10,6 +10,9 @@ on: jobs: build: runs-on: ubuntu-latest + permissions: + contents: read + packages: read steps: - name: Checkout uses: actions/checkout@v3 From 8ca78708dd8e49c40da4838326925c0bf08c9294 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Tue, 18 Jul 2023 07:30:54 +0200 Subject: [PATCH 035/192] Add inputs --- .github/workflows/flowforge-container.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 946bb0da..4344f557 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,6 +1,14 @@ name: Build Flowforge container on: workflow_dispatch: + inputs: + flowforge_release_name: + description: 'flowforge package version' + required: false + default: 'nightly' + flowforge_ref: + description: 'flowforge package ref' + required: false schedule: - cron: '30 23 * * *' push: From e7e1de2cd0f2960357420ab1e909ff3d582519d6 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Tue, 18 Jul 2023 10:27:58 +0200 Subject: [PATCH 036/192] Add deployment pipeline trigger --- .github/workflows/flowforge-container.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 4344f557..879f889a 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -74,4 +74,20 @@ jobs: REGISTRY=npm.pkg.github.com REGISTRY_TOKEN=${{ secrets.GITHUB_TOKEN }} push: true + + - name: Generate a token + id: generate_token + uses: tibdex/github-app-token@v1 + with: + app_id: ${{ secrets.GH_BOT_APP_ID }} + private_key: ${{ secrets.GH_BOT_APP_KEY }} + + - name: Trigger cloud deployment + uses: benc-uk/workflow-dispatch@v1 + with: + workflow: deploy.yml + repo: flowforge/CloudProject + ref: feat-deployment-poc + token: ${{ steps.generate_token.outputs.token }} + inputs: '{"flowforge_image_ref": "${{ github.ref }}", "flowforge_image_release_name": "${{ env.release_name }}"}' \ No newline at end of file From d467b3885dd52107c900491be1c192fe6b5b218a Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Tue, 18 Jul 2023 11:52:57 +0200 Subject: [PATCH 037/192] Change image tagging --- .github/workflows/flowforge-container.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 879f889a..de00fb37 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -28,11 +28,13 @@ jobs: - name: Setup Docker buildx uses: docker/setup-buildx-action@v2 - - name: Set nigthly release + - name: Set release name and image tag # if: ${{ github.event_name == 'schedule' }} - run: echo "release_name=nightly" >> $GITHUB_ENV + run: | + echo "release_name=nightly" >> $GITHUB_ENV + echo "image_tag=nightly-$(date +%Y%m%d%H%m%S)" >> $GITHUB_ENV - - name: "Set FF versions" + - name: "Set dependencies versions" run: | cat flowforge-container/package.json | jq '.dependencies["@flowforge/flowforge"] = "${{ env.release_name }}" | .dependencies["@flowforge/kubernetes"] = "${{ env.release_name }}" ' > flowforge-container/package.json-patched mv flowforge-container/package.json-patched flowforge-container/package.json @@ -52,14 +54,15 @@ jobs: - name: Get image metadata id: image_metadata - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4 with: + images: | + ${{ steps.login-ecr.outputs.registry }}/flowforge/forge-k8s tags: | - type=raw,value=${{ env.release_name }} + type=raw,value=${{ env.image_tag }} flavor: | latest=false - images: | - ${{ steps.login-ecr.outputs.registry }}/flowforge/forge-k8s + - name: Build and push Forge container uses: docker/build-push-action@v3.1.1 @@ -89,5 +92,5 @@ jobs: repo: flowforge/CloudProject ref: feat-deployment-poc token: ${{ steps.generate_token.outputs.token }} - inputs: '{"flowforge_image_ref": "${{ github.ref }}", "flowforge_image_release_name": "${{ env.release_name }}"}' + inputs: '{"flowforge_image_tag": "${{ env.image_tag }}"}' \ No newline at end of file From 807472415316e2fe4d4abdca720e43de58415564 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 24 Jul 2023 13:26:24 +0100 Subject: [PATCH 038/192] Allow DB connection to use SSL part of #151 --- helm/flowforge/templates/configmap.yaml | 3 +++ helm/flowforge/values.schema.json | 3 +++ 2 files changed, 6 insertions(+) diff --git a/helm/flowforge/templates/configmap.yaml b/helm/flowforge/templates/configmap.yaml index 1f840c7d..16ce5077 100644 --- a/helm/flowforge/templates/configmap.yaml +++ b/helm/flowforge/templates/configmap.yaml @@ -25,6 +25,9 @@ data: user: {{ .Values.forge.dbUsername }} password: {{ .Values.forge.dbPassword }} db: {{ .Values.forge.dbName }} + {{- if and (hasKey .Values.forge "postgres") (hasKey .Values.forge.postgres "ssl") }} + ssl: {{ .Values.forge.postgres.ssl }} + {{- end }} driver: type: kubernetes options: diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index ae749c8f..d0f5d070 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -101,6 +101,9 @@ }, "port": { "type": "integer" + }, + "ssl": { + "type": "boolean" } }, "required": [ From fd2da4e15627fb0564e1ccbbed7dfd52d2de759f Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Wed, 26 Jul 2023 10:25:45 +0100 Subject: [PATCH 039/192] First pass at substitution --- helm/flowforge/templates/service-ingress.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/helm/flowforge/templates/service-ingress.yaml b/helm/flowforge/templates/service-ingress.yaml index 10038fed..6469a984 100644 --- a/helm/flowforge/templates/service-ingress.yaml +++ b/helm/flowforge/templates/service-ingress.yaml @@ -1,3 +1,4 @@ +{{- $forgeHostname := or (.Values.forge.entryPoint) (printf "%s%s" "forge." .Values.forge.domain) -}} apiVersion: v1 kind: Service metadata: @@ -14,20 +15,18 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: flowforge-ingress - annotations: {{- if .Values.ingress.annotations }} - {{- toYaml .Values.ingress.annotations | nindent 4 }} + annotations: + {{- range $keys, $values := .Values.ingress.annotations }} + {{ $keys }}: {{ $values | replace "{{ instanceHost }}" $forgeHostname | replace "{{ serviceName }}" "forge"}} + {{- end }} {{- end }} spec: {{- if and $.Values.ingress.className (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} ingressClassName: {{ $.Values.ingress.className }} {{- end }} rules: - {{- if .Values.forge.entryPoint }} - - host: {{ .Values.forge.entryPoint }} - {{- else }} - - host: forge.{{ .Values.forge.domain}} - {{- end }} + - host: {{ $forgeHostname}} http: paths: - pathType: Prefix From ffe87ca98b1407debd1dc454da46d43b176aac76 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Wed, 26 Jul 2023 10:26:45 +0100 Subject: [PATCH 040/192] Add Docs --- helm/flowforge/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index c9a561a9..6c1d5bfb 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -32,6 +32,7 @@ If using an external PostgreSQL Database you will need to create the database an - `forge.localPostrgresql` Deploy a PostgreSQL v14 Database into Kubernetes cluster (default `true`) - `forge.postgres.host` the hostname of an external PostgreSQL database (default not set) - `forge.postgres.port` the port of an external PostgreSQL database (default `5432`) + - `forge.postgres.ssl` sets the connection to the database to use SSL/TLS (default `false`) - `forge.cloudProvider` currently only accepts `aws` but will include more as needed (default not set) - `forge.projectSelector` a collection of labels and values to filter nodes that Project Pods will run on (default `role: projects`) - `forge.managementSelector` a collection of labels and values to filter nodes the Forge App will run on (default `role: management`) From 7687f2b167a36274cd058e2fdeb949453227aeac Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 24 Jul 2023 13:26:24 +0100 Subject: [PATCH 041/192] Allow DB connection to use SSL part of #151 --- helm/flowforge/templates/configmap.yaml | 3 +++ helm/flowforge/values.schema.json | 3 +++ 2 files changed, 6 insertions(+) diff --git a/helm/flowforge/templates/configmap.yaml b/helm/flowforge/templates/configmap.yaml index 1f840c7d..16ce5077 100644 --- a/helm/flowforge/templates/configmap.yaml +++ b/helm/flowforge/templates/configmap.yaml @@ -25,6 +25,9 @@ data: user: {{ .Values.forge.dbUsername }} password: {{ .Values.forge.dbPassword }} db: {{ .Values.forge.dbName }} + {{- if and (hasKey .Values.forge "postgres") (hasKey .Values.forge.postgres "ssl") }} + ssl: {{ .Values.forge.postgres.ssl }} + {{- end }} driver: type: kubernetes options: diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index ae749c8f..d0f5d070 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -101,6 +101,9 @@ }, "port": { "type": "integer" + }, + "ssl": { + "type": "boolean" } }, "required": [ From 8272559c2a8c1db740d8c0a74d0e354b1a3b8383 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Wed, 26 Jul 2023 10:26:45 +0100 Subject: [PATCH 042/192] Add Docs --- helm/flowforge/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index f4cf472d..67acc634 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -32,6 +32,7 @@ If using an external PostgreSQL Database you will need to create the database an - `forge.localPostrgresql` Deploy a PostgreSQL v14 Database into Kubernetes cluster (default `true`) - `forge.postgres.host` the hostname of an external PostgreSQL database (default not set) - `forge.postgres.port` the port of an external PostgreSQL database (default `5432`) + - `forge.postgres.ssl` sets the connection to the database to use SSL/TLS (default `false`) - `forge.cloudProvider` currently only accepts `aws` but will include more as needed (default not set) - `forge.projectSelector` a collection of labels and values to filter nodes that Project Pods will run on (default `role: projects`) - `forge.managementSelector` a collection of labels and values to filter nodes the Forge App will run on (default `role: management`) From a8236e3533a94852c4da836efad07d3b13409a56 Mon Sep 17 00:00:00 2001 From: Elena Viter Date: Mon, 31 Jul 2023 12:44:08 +0200 Subject: [PATCH 043/192] PR #148: add Editors.service account definition to values schema --- helm/flowforge/values.schema.json | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index d0f5d070..94f1d95d 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -309,6 +309,32 @@ "type": "string" } } + }, + "editors": { + "type": "object", + "properties": { + "serviceAccount": { + "type": "object", + "properties": { + "annotations": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + }, + "create": { + "type": "boolean" + }, + "name": { + "type": "string", + "description": "Name of service account (scope of uniqueness is a 'Projects' namespace)" + } + }, + "required": ["annotations", "name"] + } + }, + "required": ["serviceAccount"] } }, "required": [ From 2c85d8a0fd60a43955d5642279d2396aebdd9fd5 Mon Sep 17 00:00:00 2001 From: Elena Viter Date: Mon, 31 Jul 2023 12:50:36 +0200 Subject: [PATCH 044/192] PR #148: allow zero properties in annotations in Editors.service account definition schema to align with default values.yaml --- helm/flowforge/values.schema.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index 94f1d95d..36c276a9 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -318,10 +318,7 @@ "properties": { "annotations": { "type": "object", - "minProperties": 1, - "additionalProperties": { - "type": "string" - } + "minProperties": 0 }, "create": { "type": "boolean" From 046d6c29f69eb7379f951f0aa71236e42261054e Mon Sep 17 00:00:00 2001 From: Elena Viter Date: Mon, 31 Jul 2023 12:56:25 +0200 Subject: [PATCH 045/192] PR #148: documenting forge.projectDeploymentTolerations in values schema --- helm/flowforge/values.schema.json | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index 36c276a9..34d2b7c6 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -116,6 +116,28 @@ "projectSelector": { "type": "object" }, + "projectDeploymentTolerations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "effect": { + "type": "string" + }, + "key": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": ["effect", "key", "operator", "value"] + }, + "default": [] + }, "managementSelector": { "type": "object" }, From 94ca26134756ad7a123605180fc22767ad939b41 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Wed, 26 Jul 2023 10:25:45 +0100 Subject: [PATCH 046/192] First pass at substitution --- helm/flowforge/templates/service-ingress.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/helm/flowforge/templates/service-ingress.yaml b/helm/flowforge/templates/service-ingress.yaml index 10038fed..6469a984 100644 --- a/helm/flowforge/templates/service-ingress.yaml +++ b/helm/flowforge/templates/service-ingress.yaml @@ -1,3 +1,4 @@ +{{- $forgeHostname := or (.Values.forge.entryPoint) (printf "%s%s" "forge." .Values.forge.domain) -}} apiVersion: v1 kind: Service metadata: @@ -14,20 +15,18 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: flowforge-ingress - annotations: {{- if .Values.ingress.annotations }} - {{- toYaml .Values.ingress.annotations | nindent 4 }} + annotations: + {{- range $keys, $values := .Values.ingress.annotations }} + {{ $keys }}: {{ $values | replace "{{ instanceHost }}" $forgeHostname | replace "{{ serviceName }}" "forge"}} + {{- end }} {{- end }} spec: {{- if and $.Values.ingress.className (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} ingressClassName: {{ $.Values.ingress.className }} {{- end }} rules: - {{- if .Values.forge.entryPoint }} - - host: {{ .Values.forge.entryPoint }} - {{- else }} - - host: forge.{{ .Values.forge.domain}} - {{- end }} + - host: {{ $forgeHostname}} http: paths: - pathType: Prefix From 86dc16da4ed64b68ae04d512be50f68c04ca23a2 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Wed, 26 Jul 2023 10:25:45 +0100 Subject: [PATCH 047/192] First pass at substitution --- helm/flowforge/templates/service-ingress.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/helm/flowforge/templates/service-ingress.yaml b/helm/flowforge/templates/service-ingress.yaml index 10038fed..6469a984 100644 --- a/helm/flowforge/templates/service-ingress.yaml +++ b/helm/flowforge/templates/service-ingress.yaml @@ -1,3 +1,4 @@ +{{- $forgeHostname := or (.Values.forge.entryPoint) (printf "%s%s" "forge." .Values.forge.domain) -}} apiVersion: v1 kind: Service metadata: @@ -14,20 +15,18 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: flowforge-ingress - annotations: {{- if .Values.ingress.annotations }} - {{- toYaml .Values.ingress.annotations | nindent 4 }} + annotations: + {{- range $keys, $values := .Values.ingress.annotations }} + {{ $keys }}: {{ $values | replace "{{ instanceHost }}" $forgeHostname | replace "{{ serviceName }}" "forge"}} + {{- end }} {{- end }} spec: {{- if and $.Values.ingress.className (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} ingressClassName: {{ $.Values.ingress.className }} {{- end }} rules: - {{- if .Values.forge.entryPoint }} - - host: {{ .Values.forge.entryPoint }} - {{- else }} - - host: forge.{{ .Values.forge.domain}} - {{- end }} + - host: {{ $forgeHostname}} http: paths: - pathType: Prefix From f2336415cf76f250fc4d9ddf1ec941056b044a5e Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 31 Jul 2023 14:59:15 +0100 Subject: [PATCH 048/192] Fix up broker annotations --- helm/flowforge/README.md | 1 + helm/flowforge/templates/broker.yaml | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 67acc634..af6ff2f8 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -16,6 +16,7 @@ If using an external PostgreSQL Database you will need to create the database an - `forge.dbPassword` - `forge.postgres.host` - `forge.postgres.port` +- `forge.postgres.ssl` ## Configuration Values diff --git a/helm/flowforge/templates/broker.yaml b/helm/flowforge/templates/broker.yaml index 28f02a2d..2099cdcd 100644 --- a/helm/flowforge/templates/broker.yaml +++ b/helm/flowforge/templates/broker.yaml @@ -1,4 +1,5 @@ {{- if .Values.forge.broker.enabled -}} +{{- $brokerHostname := or (.Values.forge.entryPoint) (printf "%s%s" "mqtt." .Values.forge.domain) -}} apiVersion: v1 kind: ConfigMap metadata: @@ -130,9 +131,11 @@ metadata: name: flowforge-broker labels: app: flowforge-broker - annotations: {{- if .Values.ingress.annotations }} - {{- toYaml .Values.ingress.annotations | nindent 4 }} + annotations: + {{- range $keys, $values := .Values.ingress.annotations }} + {{ $keys }}: {{ $values | replace "{{ instanceHost }}" $brokerHostname | replace "{{ serviceName }}" "flowforge-broker"}} + {{-end }} {{- end }} spec: {{- if $.Values.ingress.className }} From 1618e95b8e6a7be5a328b1514cedbfa32e97c3c4 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 31 Jul 2023 15:04:05 +0100 Subject: [PATCH 049/192] Fix missing space --- helm/flowforge/templates/broker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/flowforge/templates/broker.yaml b/helm/flowforge/templates/broker.yaml index 2099cdcd..7161d910 100644 --- a/helm/flowforge/templates/broker.yaml +++ b/helm/flowforge/templates/broker.yaml @@ -135,7 +135,7 @@ metadata: annotations: {{- range $keys, $values := .Values.ingress.annotations }} {{ $keys }}: {{ $values | replace "{{ instanceHost }}" $brokerHostname | replace "{{ serviceName }}" "flowforge-broker"}} - {{-end }} + {{- end }} {{- end }} spec: {{- if $.Values.ingress.className }} From 81889aabdd43e87e478a9bbd471c6a7e15da5c5a Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 31 Jul 2023 18:13:17 +0100 Subject: [PATCH 050/192] Add Rate Limit details part of flowforge/flowforge#2530 --- helm/flowforge/values.schema.json | 20 ++++++++++++++++++++ helm/flowforge/values.yaml | 7 +++++++ test/customizations.yml | 2 ++ 3 files changed, 29 insertions(+) diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index 34d2b7c6..fba76c46 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -300,6 +300,26 @@ }, "branding": { "type": "object" + }, + "rateLimits": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "global": { + "type": "boolean" + }, + "max": { + "type": "integer" + }, + "maxAnonymous": { + "type": "integer" + }, + "timeWindow": { + "type": "integer" + } + } } }, "required": [ diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index dca54c37..b706e205 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -36,6 +36,13 @@ forge: support: enabled: false + rate_limits: + enabled: false + global: false + max: 1000 + maxAnonymous: 10 + timeWindow: 60000 + domain: "" entryPoint: "" environment: {} diff --git a/test/customizations.yml b/test/customizations.yml index 5a0f62f7..ba68a3d7 100644 --- a/test/customizations.yml +++ b/test/customizations.yml @@ -68,3 +68,5 @@ forge: branding: account: signUpTopBanner: HelloWorld + rate_limits: + enabled: true From 22c76eecc6a1b15c5626ac59542ecfe0b5d9108d Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 1 Aug 2023 09:30:34 +0100 Subject: [PATCH 051/192] Add configmap and readme.md --- helm/flowforge/README.md | 10 ++++++++++ helm/flowforge/templates/configmap.yaml | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 67acc634..be5ef148 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -119,6 +119,16 @@ Enables FlowForge Telemetry - `forge.privateCA.configMapName` name of ConfigMap to store the CA Cert bundle (default `ff-ca-certs`) - `forge.privateCA.certs` base64 encoded CA certificate PEM bundle of trusted certificates. This needs to be generated without line breaks e.g. `base64 -w 0 certs.pem` (default not set) + ### Rate Limiting + + - `forge.rate_limits.enabled` (default `false`) + - `forge.rate_limits.global` (default `false`) + - `forge.rate_limits.timeWindow` Time in milliseconds to evaluate requests over (default 60000) + - `forge.rate_limits.max` Max requests per timeWindow (default: 1000) + - `forge.rate_limits.maxAnonymous` Max anonymous requests per timeWindow (default: 10) + +Everything under `forge.rate_limits` is used as input to Fastify Rate Limit plugin, further options can be found [here](https://github.com/fastify/fastify-rate-limit#options) + ### Ingress - `ingress.annotations` ingress annotations (default is `{}`). This value is also applied to Editor instances created by FlowForge. - `ingress.className` ingress class name (default is `"""`). This value is also applied to Editor instances created by FlowForge. diff --git a/helm/flowforge/templates/configmap.yaml b/helm/flowforge/templates/configmap.yaml index 16ce5077..89a7224c 100644 --- a/helm/flowforge/templates/configmap.yaml +++ b/helm/flowforge/templates/configmap.yaml @@ -163,3 +163,7 @@ data: branding: {{ toYaml .Values.forge.branding | indent 6 }} {{- end }} + {{- if .Values.forge.rate_limits }} + rate_limits: +{{ toYaml .Values.forge.rate_limits | indent 6 }} + {{- end }} \ No newline at end of file From 0f5727fdd0a05fd65a75324576639397b64c27a3 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 1 Aug 2023 09:48:00 +0100 Subject: [PATCH 052/192] Update defaults --- helm/flowforge/README.md | 2 +- helm/flowforge/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index be5ef148..a8d5f6c7 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -122,7 +122,7 @@ Enables FlowForge Telemetry ### Rate Limiting - `forge.rate_limits.enabled` (default `false`) - - `forge.rate_limits.global` (default `false`) + - `forge.rate_limits.global` (default `true`) - `forge.rate_limits.timeWindow` Time in milliseconds to evaluate requests over (default 60000) - `forge.rate_limits.max` Max requests per timeWindow (default: 1000) - `forge.rate_limits.maxAnonymous` Max anonymous requests per timeWindow (default: 10) diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index b706e205..1f50fd46 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -38,7 +38,7 @@ forge: rate_limits: enabled: false - global: false + global: true max: 1000 maxAnonymous: 10 timeWindow: 60000 From 40c69046493303c0b4d9657c515b23809b32d919 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 1 Aug 2023 10:22:40 +0100 Subject: [PATCH 053/192] Fix maxAnonymous --- helm/flowforge/README.md | 6 +++--- helm/flowforge/values.yaml | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index a8d5f6c7..855d53d9 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -124,10 +124,10 @@ Enables FlowForge Telemetry - `forge.rate_limits.enabled` (default `false`) - `forge.rate_limits.global` (default `true`) - `forge.rate_limits.timeWindow` Time in milliseconds to evaluate requests over (default 60000) - - `forge.rate_limits.max` Max requests per timeWindow (default: 1000) - - `forge.rate_limits.maxAnonymous` Max anonymous requests per timeWindow (default: 10) + - `forge.rate_limits.max` Max requests per timeWindow (default 1000) + - `forge.rate_limits.maxAnonymous` Max anonymous requests per timeWindow (default `forge.rate_limits.max`) -Everything under `forge.rate_limits` is used as input to Fastify Rate Limit plugin, further options can be found [here](https://github.com/fastify/fastify-rate-limit#options) +Everything under `forge.rate_limits` is used as input to Fastify Rate Limit plugin, further options can be found [here](https://github.com/fastify/fastify-rate-limit#options) and can be included. ### Ingress - `ingress.annotations` ingress annotations (default is `{}`). This value is also applied to Editor instances created by FlowForge. diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index 1f50fd46..a7e4a1c4 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -40,7 +40,6 @@ forge: enabled: false global: true max: 1000 - maxAnonymous: 10 timeWindow: 60000 domain: "" From 8654caedec6b2b02bcbe57359be1b79895afc31e Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 1 Aug 2023 10:53:40 +0100 Subject: [PATCH 054/192] Update helm/flowforge/templates/configmap.yaml --- helm/flowforge/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/flowforge/templates/configmap.yaml b/helm/flowforge/templates/configmap.yaml index 89a7224c..a47d1894 100644 --- a/helm/flowforge/templates/configmap.yaml +++ b/helm/flowforge/templates/configmap.yaml @@ -166,4 +166,4 @@ data: {{- if .Values.forge.rate_limits }} rate_limits: {{ toYaml .Values.forge.rate_limits | indent 6 }} - {{- end }} \ No newline at end of file + {{- end }} From 3275848bd6b3bf85f8fca1effb68c93e22d3b5d0 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 1 Aug 2023 11:44:45 +0100 Subject: [PATCH 055/192] Revert create-pod Role name @elenaviter I've had to revert the renaming of the ClusterRole because it breaks upgrades. We can look at this again after the release this week --- helm/flowforge/templates/service-account.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/flowforge/templates/service-account.yaml b/helm/flowforge/templates/service-account.yaml index a9ec261f..90c4ef99 100644 --- a/helm/flowforge/templates/service-account.yaml +++ b/helm/flowforge/templates/service-account.yaml @@ -29,7 +29,7 @@ metadata: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ .Release.Name }}-create-pod + name: create-pod rules: - apiGroups: [""] resources: ["pods", "pods/log", "pods/exec", "pods/status"] @@ -61,5 +61,5 @@ subjects: namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole - name: {{ .Release.Name }}-create-pod + name: create-pod apiGroup: rbac.authorization.k8s.io From c9dcf47cc40cf51fd315d71aac28804467987241 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Wed, 26 Jul 2023 10:25:45 +0100 Subject: [PATCH 056/192] First pass at substitution --- helm/flowforge/templates/service-ingress.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/helm/flowforge/templates/service-ingress.yaml b/helm/flowforge/templates/service-ingress.yaml index 10038fed..6469a984 100644 --- a/helm/flowforge/templates/service-ingress.yaml +++ b/helm/flowforge/templates/service-ingress.yaml @@ -1,3 +1,4 @@ +{{- $forgeHostname := or (.Values.forge.entryPoint) (printf "%s%s" "forge." .Values.forge.domain) -}} apiVersion: v1 kind: Service metadata: @@ -14,20 +15,18 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: flowforge-ingress - annotations: {{- if .Values.ingress.annotations }} - {{- toYaml .Values.ingress.annotations | nindent 4 }} + annotations: + {{- range $keys, $values := .Values.ingress.annotations }} + {{ $keys }}: {{ $values | replace "{{ instanceHost }}" $forgeHostname | replace "{{ serviceName }}" "forge"}} + {{- end }} {{- end }} spec: {{- if and $.Values.ingress.className (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} ingressClassName: {{ $.Values.ingress.className }} {{- end }} rules: - {{- if .Values.forge.entryPoint }} - - host: {{ .Values.forge.entryPoint }} - {{- else }} - - host: forge.{{ .Values.forge.domain}} - {{- end }} + - host: {{ $forgeHostname}} http: paths: - pathType: Prefix From 5df653782c7d7d8ea2bab9283eaacda92f4016ef Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 31 Jul 2023 14:59:15 +0100 Subject: [PATCH 057/192] Fix up broker annotations --- helm/flowforge/README.md | 1 + helm/flowforge/templates/broker.yaml | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 855d53d9..3f123c3a 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -16,6 +16,7 @@ If using an external PostgreSQL Database you will need to create the database an - `forge.dbPassword` - `forge.postgres.host` - `forge.postgres.port` +- `forge.postgres.ssl` ## Configuration Values diff --git a/helm/flowforge/templates/broker.yaml b/helm/flowforge/templates/broker.yaml index 28f02a2d..2099cdcd 100644 --- a/helm/flowforge/templates/broker.yaml +++ b/helm/flowforge/templates/broker.yaml @@ -1,4 +1,5 @@ {{- if .Values.forge.broker.enabled -}} +{{- $brokerHostname := or (.Values.forge.entryPoint) (printf "%s%s" "mqtt." .Values.forge.domain) -}} apiVersion: v1 kind: ConfigMap metadata: @@ -130,9 +131,11 @@ metadata: name: flowforge-broker labels: app: flowforge-broker - annotations: {{- if .Values.ingress.annotations }} - {{- toYaml .Values.ingress.annotations | nindent 4 }} + annotations: + {{- range $keys, $values := .Values.ingress.annotations }} + {{ $keys }}: {{ $values | replace "{{ instanceHost }}" $brokerHostname | replace "{{ serviceName }}" "flowforge-broker"}} + {{-end }} {{- end }} spec: {{- if $.Values.ingress.className }} From 2094921d9b136140ea270ae7678574aeed78ee7d Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 31 Jul 2023 15:04:05 +0100 Subject: [PATCH 058/192] Fix missing space --- helm/flowforge/templates/broker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/flowforge/templates/broker.yaml b/helm/flowforge/templates/broker.yaml index 2099cdcd..7161d910 100644 --- a/helm/flowforge/templates/broker.yaml +++ b/helm/flowforge/templates/broker.yaml @@ -135,7 +135,7 @@ metadata: annotations: {{- range $keys, $values := .Values.ingress.annotations }} {{ $keys }}: {{ $values | replace "{{ instanceHost }}" $brokerHostname | replace "{{ serviceName }}" "flowforge-broker"}} - {{-end }} + {{- end }} {{- end }} spec: {{- if $.Values.ingress.className }} From 92f232a7416c55390c1f76e59adfbacaa2969aca Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 1 Aug 2023 11:59:49 +0100 Subject: [PATCH 059/192] Add docs --- helm/flowforge/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 3f123c3a..8bc71d5b 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -134,6 +134,13 @@ Everything under `forge.rate_limits` is used as input to Fastify Rate Limit plug - `ingress.annotations` ingress annotations (default is `{}`). This value is also applied to Editor instances created by FlowForge. - `ingress.className` ingress class name (default is `"""`). This value is also applied to Editor instances created by FlowForge. + `ingress.annotations` can contain the following tokens that will be replaced + + - `{{ instanceHost }}` replaced by the hostname of the instance + - `{{ instanceURL }}` replaced by the URL for the instance + - `{{ instanceProtocol }}` replaced by either `http` or `https` + - `{{ serviceName }}` replaced by the service name of the instance + ### Editors IAM Provision default service account for Editors if `editors.serviceAccount.create` is `true`. From 5db998ba296e5dffd6e053f2b2127cf234c374bf Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 1 Aug 2023 12:01:03 +0100 Subject: [PATCH 060/192] Better description --- helm/flowforge/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 8bc71d5b..5caf1afd 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -134,7 +134,7 @@ Everything under `forge.rate_limits` is used as input to Fastify Rate Limit plug - `ingress.annotations` ingress annotations (default is `{}`). This value is also applied to Editor instances created by FlowForge. - `ingress.className` ingress class name (default is `"""`). This value is also applied to Editor instances created by FlowForge. - `ingress.annotations` can contain the following tokens that will be replaced + `ingress.annotations` values can contain the following tokens that will be replaced as follows: - `{{ instanceHost }}` replaced by the hostname of the instance - `{{ instanceURL }}` replaced by the URL for the instance From 1a835575129e333115aa0bb921936ffbebe55008 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 1 Aug 2023 12:04:54 +0100 Subject: [PATCH 061/192] Revert missing default creds --- helm/flowforge/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index a7e4a1c4..8ad6d937 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -31,7 +31,7 @@ forge: type: postgres host: flowforge-postgresql username: forge - password: "" + password: Zai1Wied database: ff-context support: enabled: false From 40cae80013cfee812a43b438dfdaf8b29f63d55e Mon Sep 17 00:00:00 2001 From: Joe Pavitt Date: Thu, 3 Aug 2023 10:03:55 +0100 Subject: [PATCH 062/192] Update CHANGELOG for 1.10.0 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e3cb86c..2e6b79d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +#### 1.10.0: Release + + - Revert create-pod Role name (#159) @hardillb + - Allow variable substituion in Ingress Annotations (#156) @hardillb + - Add Rate Limit details (#158) @hardillb + - FlowForge helm: 1. Editors: service account. 2. Broker: propagate ingress. 3. README (#148) @elenaviter + - Allow DB connection to use SSL (#154) @hardillb + #### 1.9.0: Release - Add comment about PostgreSQL constraints (#145) @hardillb From b6ecbd98d25c8e102868e92c57adffaf6a7e3d97 Mon Sep 17 00:00:00 2001 From: Joe Pavitt Date: Thu, 3 Aug 2023 10:03:55 +0100 Subject: [PATCH 063/192] Update version to 1.10.0 --- VERSION | 2 +- file-server/package.json | 4 ++-- flowforge-container/package.json | 6 +++--- helm/flowforge/Chart.yaml | 4 ++-- node-red-container/package.json | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/VERSION b/VERSION index f8e233b2..81c871de 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.9.0 +1.10.0 diff --git a/file-server/package.json b/file-server/package.json index 0b249795..4f6779c5 100644 --- a/file-server/package.json +++ b/file-server/package.json @@ -1,9 +1,9 @@ { "name": "@flowforge/file-server-container", - "version": "1.9.0", + "version": "1.10.0", "private": true, "dependencies": { - "@flowforge/file-server": "^1.9.0" + "@flowforge/file-server": "^1.10.0" }, "license": "Apache-2.0" } diff --git a/flowforge-container/package.json b/flowforge-container/package.json index 94780eef..b36d9e6f 100644 --- a/flowforge-container/package.json +++ b/flowforge-container/package.json @@ -1,14 +1,14 @@ { "name": "@flowforge/flowforge-k8s", - "version": "1.9.0", + "version": "1.10.0", "description": "FlowForge in Docker", "private": true, "author": { "name": "FlowForge Inc." }, "dependencies": { - "@flowforge/flowforge": "^1.9.0", - "@flowforge/kubernetes": "^1.9.0", + "@flowforge/flowforge": "^1.10.0", + "@flowforge/kubernetes": "^1.10.0", "pg": "^8.7.1", "pg-hstore": "^2.3.4" }, diff --git a/helm/flowforge/Chart.yaml b/helm/flowforge/Chart.yaml index 65848cd0..88416f5d 100644 --- a/helm/flowforge/Chart.yaml +++ b/helm/flowforge/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "flowforge" -version: "1.9.0" +version: "1.10.0" description: "FlowForge" type: "application" home: "https://flowforge.com" @@ -16,4 +16,4 @@ dependencies: maintainers: - name: "FlowForge Inc" url: "https://flowforge.com" -appVersion: "1.9.0" +appVersion: "1.10.0" diff --git a/node-red-container/package.json b/node-red-container/package.json index 5c2343b7..d8303285 100644 --- a/node-red-container/package.json +++ b/node-red-container/package.json @@ -1,7 +1,7 @@ { "name": "node-red-project", "description": "A Node-RED Project", - "version": "1.9.0", + "version": "1.10.0", "private": true, "dependencies": {} } From 1b243663e15c39d9e8d81475a3426dcfe3f8ff6a Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 17 Aug 2023 09:57:08 +0200 Subject: [PATCH 064/192] Make registry_name as configurable env variable --- .github/workflows/flowforge-container.yml | 126 ++++++++++++---------- 1 file changed, 72 insertions(+), 54 deletions(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index de00fb37..d0521d32 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -10,74 +10,92 @@ on: description: 'flowforge package ref' required: false schedule: - - cron: '30 23 * * *' + - cron: '30 5 * * *' push: branches: - 'feat-*' + pull_request: + branches: + - main jobs: build: - runs-on: ubuntu-latest - permissions: - contents: read - packages: read - steps: - - name: Checkout - uses: actions/checkout@v3 + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-container-build + with: + package_dependencies: | + @flowforge/flowforge + @flowforge/kubernetes + registry_name: "dkr.ecr.eu-west-1.amazonaws.com/flowforge/forge-k8s" + build_context: + description: 'Path to the build context' + type: string + required: true + build_arguments: + description: 'Build arguments to pass to the image build step' + type: string + required: false + default: '' - - name: Setup Docker buildx - uses: docker/setup-buildx-action@v2 + # - name: Checkout + # uses: actions/checkout@v3 - - name: Set release name and image tag - # if: ${{ github.event_name == 'schedule' }} - run: | - echo "release_name=nightly" >> $GITHUB_ENV - echo "image_tag=nightly-$(date +%Y%m%d%H%m%S)" >> $GITHUB_ENV + # - name: Setup Docker buildx + # uses: docker/setup-buildx-action@v2 - - name: "Set dependencies versions" - run: | - cat flowforge-container/package.json | jq '.dependencies["@flowforge/flowforge"] = "${{ env.release_name }}" | .dependencies["@flowforge/kubernetes"] = "${{ env.release_name }}" ' > flowforge-container/package.json-patched - mv flowforge-container/package.json-patched flowforge-container/package.json + # - name: Set release name and image tag + # # if: ${{ github.event_name == 'schedule' }} + # run: | + # echo "release_name=nightly" >> $GITHUB_ENV + # echo "image_tag=nightly-$(date +%Y%m%d%H%m%S)" >> $GITHUB_ENV - - name: Configure AWS credentials - if: ${{ env.release_name == 'nightly' }} - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-access-key-id: ${{ secrets.STAGING_AWS_ID }} - aws-secret-access-key: ${{ secrets.STAGING_AWS_KEY }} - aws-region: eu-west-1 + # - name: "Set dependencies versions" + # run: | + # cat flowforge-container/package.json | jq '.dependencies["@flowforge/flowforge"] = "${{ env.release_name }}" | .dependencies["@flowforge/kubernetes"] = "${{ env.release_name }}" ' > flowforge-container/package.json-patched + # mv flowforge-container/package.json-patched flowforge-container/package.json - - name: "Login to Staging ECR" - if: ${{ env.release_name == 'nightly' }} - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 + # - name: Configure AWS credentials + # if: ${{ env.release_name == 'nightly' }} + # uses: aws-actions/configure-aws-credentials@v2 + # with: + # aws-access-key-id: ${{ secrets.STAGING_AWS_ID }} + # aws-secret-access-key: ${{ secrets.STAGING_AWS_KEY }} + # aws-region: eu-west-1 - - name: Get image metadata - id: image_metadata - uses: docker/metadata-action@v4 - with: - images: | - ${{ steps.login-ecr.outputs.registry }}/flowforge/forge-k8s - tags: | - type=raw,value=${{ env.image_tag }} - flavor: | - latest=false - + # - name: "Login to Staging ECR" + # if: ${{ env.release_name == 'nightly' }} + # id: login-ecr + # uses: aws-actions/amazon-ecr-login@v1 - - name: Build and push Forge container - uses: docker/build-push-action@v3.1.1 - with: - context: flowforge-container - file: flowforge-container/Dockerfile - tags: ${{ steps.image_metadata.outputs.tags }} - platforms: | - linux/amd64 - linux/arm64 - build-args: | - REGISTRY=npm.pkg.github.com - REGISTRY_TOKEN=${{ secrets.GITHUB_TOKEN }} - push: true + # - name: Get image metadata + # id: image_metadata + # uses: docker/metadata-action@v4 + # with: + # images: | + # ${{ steps.login-ecr.outputs.registry }}/flowforge/forge-k8s + # tags: | + # type=raw,value=${{ env.image_tag }} + # flavor: | + # latest=false + + # - name: Build and push Forge container + # uses: docker/build-push-action@v3.1.1 + # with: + # context: flowforge-container + # file: flowforge-container/Dockerfile + # tags: ${{ steps.image_metadata.outputs.tags }} + # platforms: | + # linux/amd64 + # linux/arm64 + # build-args: | + # REGISTRY=npm.pkg.github.com + # REGISTRY_TOKEN=${{ secrets.GITHUB_TOKEN }} + # push: true + dispatch: + runs-on: ubuntu-latest + needs: [build] + if: false + steps: - name: Generate a token id: generate_token uses: tibdex/github-app-token@v1 From ec5d92b12a77c9b7bc7c70eb3b721afffa6d7897 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 17 Aug 2023 12:33:16 +0200 Subject: [PATCH 065/192] Test reusable container build workflow --- .github/workflows/flowforge-container.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index d0521d32..d4e92fd0 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -22,19 +22,12 @@ jobs: build: uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-container-build with: + image_name: 'forge-k8s' package_dependencies: | @flowforge/flowforge @flowforge/kubernetes - registry_name: "dkr.ecr.eu-west-1.amazonaws.com/flowforge/forge-k8s" - build_context: - description: 'Path to the build context' - type: string - required: true - build_arguments: - description: 'Build arguments to pass to the image build step' - type: string - required: false - default: '' + build_context: 'flowforge-container' + push_image: false # - name: Checkout # uses: actions/checkout@v3 From d8d537d3b0c5c75b6a5b8b80fed565b7ead99601 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 17 Aug 2023 12:37:48 +0200 Subject: [PATCH 066/192] Test reusable container build workflow --- .github/workflows/flowforge-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index d4e92fd0..ca8e6f8d 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,4 +1,5 @@ name: Build Flowforge container + on: workflow_dispatch: inputs: From 78ba33558295f979e3f0d54f299d7a82c4c505a7 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 18 Aug 2023 07:25:55 +0200 Subject: [PATCH 067/192] Test reusable container build workflow --- .github/workflows/flowforge-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index ca8e6f8d..d4e92fd0 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,5 +1,4 @@ name: Build Flowforge container - on: workflow_dispatch: inputs: From 6a69a15287a58a37b986dd20c75bcc8d7258f19c Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 18 Aug 2023 10:11:40 +0200 Subject: [PATCH 068/192] Test reusable container build workflow --- .github/workflows/flowforge-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index d4e92fd0..ca8e6f8d 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,4 +1,5 @@ name: Build Flowforge container + on: workflow_dispatch: inputs: From f1f81d664412858f8a29989375f742a5f72ba7bb Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 18 Aug 2023 10:14:48 +0200 Subject: [PATCH 069/192] Test reusable container build workflow --- .github/workflows/flowforge-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index ca8e6f8d..d4e92fd0 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,5 +1,4 @@ name: Build Flowforge container - on: workflow_dispatch: inputs: From fd81cfaf7d2a6aced7f13d204ff0e56911d12297 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 18 Aug 2023 11:02:15 +0200 Subject: [PATCH 070/192] Test container build workflow --- .github/workflows/flowforge-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index d4e92fd0..ca8e6f8d 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,4 +1,5 @@ name: Build Flowforge container + on: workflow_dispatch: inputs: From d124b45acdad175f7a51e51abcc89e39219d3a40 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 18 Aug 2023 14:38:34 +0200 Subject: [PATCH 071/192] Test reusable container build workflow --- .github/workflows/flowforge-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index ca8e6f8d..d4e92fd0 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,5 +1,4 @@ name: Build Flowforge container - on: workflow_dispatch: inputs: From 1ab4645e566a7bdebca715bafe8de2b7ff2b4f77 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 18 Aug 2023 16:49:19 +0200 Subject: [PATCH 072/192] Remove registry URL schema --- flowforge-container/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowforge-container/Dockerfile b/flowforge-container/Dockerfile index 68c855d2..3625aa53 100644 --- a/flowforge-container/Dockerfile +++ b/flowforge-container/Dockerfile @@ -4,7 +4,7 @@ ARG REGISTRY ARG REGISTRY_TOKEN ARG TAG RUN if [[ ! -z "$REGISTRY_TOKEN" ]]; then echo "//$REGISTRY/:_authToken=$REGISTRY_TOKEN" >> ~/.npmrc ; fi -RUN if [[ ! -z "$REGISTRY" ]] ; then npm config set @flowforge:registry "https://$REGISTRY"; fi +RUN if [[ ! -z "$REGISTRY" ]] ; then npm config set @flowforge:registry "$REGISTRY"; fi RUN apk add --no-cache --virtual build-base g++ make py3-pip sqlite-dev python3 RUN npm config set python `which python3` --global From 1d3a1f540b2e2ba3bee0d34efc3fba49ada46a19 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Sun, 20 Aug 2023 21:40:59 +0200 Subject: [PATCH 073/192] Add `npm_registry_auth_token` input --- .github/workflows/flowforge-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index d4e92fd0..d2f03358 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -28,6 +28,7 @@ jobs: @flowforge/kubernetes build_context: 'flowforge-container' push_image: false + npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} # - name: Checkout # uses: actions/checkout@v3 From 080333e01750213a8efd1a338a17195be254dfad Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Sun, 20 Aug 2023 21:43:44 +0200 Subject: [PATCH 074/192] Add `npm_registry_auth_token` input --- .github/workflows/flowforge-container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index d2f03358..9138651b 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -27,8 +27,8 @@ jobs: @flowforge/flowforge @flowforge/kubernetes build_context: 'flowforge-container' - push_image: false - npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + push_image: 'false' + npm_registry_auth_token: ${{ github.token }} # - name: Checkout # uses: actions/checkout@v3 From 59eaf7ea33295fc0bdd135a20c60d2468b92d6d1 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Sun, 20 Aug 2023 21:44:45 +0200 Subject: [PATCH 075/192] Add `npm_registry_auth_token` input --- .github/workflows/flowforge-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 9138651b..5adac037 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -27,7 +27,7 @@ jobs: @flowforge/flowforge @flowforge/kubernetes build_context: 'flowforge-container' - push_image: 'false' + push_image: false npm_registry_auth_token: ${{ github.token }} # - name: Checkout From 13b9a3ea3c45a450ac4689a60ededbe0c462d1ba Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Sun, 20 Aug 2023 21:51:10 +0200 Subject: [PATCH 076/192] Move `npm_registry_auth_token` to secrets --- .github/workflows/flowforge-container.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 5adac037..5035c50f 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -28,7 +28,8 @@ jobs: @flowforge/kubernetes build_context: 'flowforge-container' push_image: false - npm_registry_auth_token: ${{ github.token }} + secrets: + npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} # - name: Checkout # uses: actions/checkout@v3 From 3831c34b1f19efed351c831746a7f2028cab997d Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Sun, 20 Aug 2023 22:01:05 +0200 Subject: [PATCH 077/192] Build debug --- .github/workflows/flowforge-container.yml | 46 ++++++++++++++++------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 5035c50f..040b0e42 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -20,6 +20,7 @@ on: jobs: build: + if: false uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-container-build with: image_name: 'forge-k8s' @@ -30,23 +31,26 @@ jobs: push_image: false secrets: npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + directBuild: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 - # - name: Checkout - # uses: actions/checkout@v3 - - # - name: Setup Docker buildx - # uses: docker/setup-buildx-action@v2 + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v2 - # - name: Set release name and image tag - # # if: ${{ github.event_name == 'schedule' }} - # run: | - # echo "release_name=nightly" >> $GITHUB_ENV - # echo "image_tag=nightly-$(date +%Y%m%d%H%m%S)" >> $GITHUB_ENV + - name: Set release name and image tag + # if: ${{ github.event_name == 'schedule' }} + run: | + echo "release_name=nightly" >> $GITHUB_ENV + echo "npm_registry=https://npm.pkg.github.com" >> $GITHUB_ENV + echo "image_tag=nightly-$(date +%Y%m%d%H%m%S)" >> $GITHUB_ENV - # - name: "Set dependencies versions" - # run: | - # cat flowforge-container/package.json | jq '.dependencies["@flowforge/flowforge"] = "${{ env.release_name }}" | .dependencies["@flowforge/kubernetes"] = "${{ env.release_name }}" ' > flowforge-container/package.json-patched - # mv flowforge-container/package.json-patched flowforge-container/package.json + - name: "Set dependencies versions" + run: | + cat flowforge-container/package.json | jq '.dependencies["@flowforge/flowforge"] = "${{ env.release_name }}" | .dependencies["@flowforge/kubernetes"] = "${{ env.release_name }}" ' > flowforge-container/package.json-patched + mv flowforge-container/package.json-patched flowforge-container/package.json # - name: Configure AWS credentials # if: ${{ env.release_name == 'nightly' }} @@ -72,6 +76,20 @@ jobs: # flavor: | # latest=false + - name: Build container image + uses: docker/build-push-action@v4 + with: + context: flowforge-container + file: flowforge-container/Dockerfile + tags: "forge-k8s:${{ env.image_tag }}" + platforms: | + linux/amd64 + linux/arm64 + build-args: | + REGISTRY=${{ env.npm_registry }} + REGISTRY_TOKEN=${{ secrets.npm_registry_auth_token }} + push: false + # - name: Build and push Forge container # uses: docker/build-push-action@v3.1.1 # with: From 025ca40b6c1e4ef7d3dc3365b7d1ccae1a74863c Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Sun, 20 Aug 2023 22:05:29 +0200 Subject: [PATCH 078/192] Build debug --- .github/workflows/flowforge-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 040b0e42..dbcc623d 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -87,7 +87,7 @@ jobs: linux/arm64 build-args: | REGISTRY=${{ env.npm_registry }} - REGISTRY_TOKEN=${{ secrets.npm_registry_auth_token }} + REGISTRY_TOKEN=${{ secrets.GITHUB_TOKEN }} push: false # - name: Build and push Forge container From 91209d98be261f0ccc392cc46d9f932b88ebe859 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Sun, 20 Aug 2023 22:17:55 +0200 Subject: [PATCH 079/192] Build debug --- .github/workflows/flowforge-container.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index dbcc623d..fca0ac99 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -20,7 +20,7 @@ on: jobs: build: - if: false + if: true uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-container-build with: image_name: 'forge-k8s' @@ -33,6 +33,7 @@ jobs: npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} directBuild: runs-on: ubuntu-latest + if: false steps: - name: Checkout uses: actions/checkout@v3 From 016163f6c748f108c1d98e5105f65f25a46c6e02 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Sun, 20 Aug 2023 22:21:23 +0200 Subject: [PATCH 080/192] Debug build --- flowforge-container/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/flowforge-container/Dockerfile b/flowforge-container/Dockerfile index 3625aa53..9dc6ff1e 100644 --- a/flowforge-container/Dockerfile +++ b/flowforge-container/Dockerfile @@ -6,6 +6,7 @@ ARG TAG RUN if [[ ! -z "$REGISTRY_TOKEN" ]]; then echo "//$REGISTRY/:_authToken=$REGISTRY_TOKEN" >> ~/.npmrc ; fi RUN if [[ ! -z "$REGISTRY" ]] ; then npm config set @flowforge:registry "$REGISTRY"; fi +RUN cat ~/.npmrc RUN apk add --no-cache --virtual build-base g++ make py3-pip sqlite-dev python3 RUN npm config set python `which python3` --global From 50842ad997007985d2989899d4c0320c036c2f9c Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Sun, 20 Aug 2023 22:23:45 +0200 Subject: [PATCH 081/192] Build debug --- flowforge-container/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/flowforge-container/Dockerfile b/flowforge-container/Dockerfile index 9dc6ff1e..3625aa53 100644 --- a/flowforge-container/Dockerfile +++ b/flowforge-container/Dockerfile @@ -6,7 +6,6 @@ ARG TAG RUN if [[ ! -z "$REGISTRY_TOKEN" ]]; then echo "//$REGISTRY/:_authToken=$REGISTRY_TOKEN" >> ~/.npmrc ; fi RUN if [[ ! -z "$REGISTRY" ]] ; then npm config set @flowforge:registry "$REGISTRY"; fi -RUN cat ~/.npmrc RUN apk add --no-cache --virtual build-base g++ make py3-pip sqlite-dev python3 RUN npm config set python `which python3` --global From 11e0a09b5b24092a0b089131071532352b7848af Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Sun, 20 Aug 2023 22:37:22 +0200 Subject: [PATCH 082/192] Rollback scheme removal --- flowforge-container/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowforge-container/Dockerfile b/flowforge-container/Dockerfile index 3625aa53..68c855d2 100644 --- a/flowforge-container/Dockerfile +++ b/flowforge-container/Dockerfile @@ -4,7 +4,7 @@ ARG REGISTRY ARG REGISTRY_TOKEN ARG TAG RUN if [[ ! -z "$REGISTRY_TOKEN" ]]; then echo "//$REGISTRY/:_authToken=$REGISTRY_TOKEN" >> ~/.npmrc ; fi -RUN if [[ ! -z "$REGISTRY" ]] ; then npm config set @flowforge:registry "$REGISTRY"; fi +RUN if [[ ! -z "$REGISTRY" ]] ; then npm config set @flowforge:registry "https://$REGISTRY"; fi RUN apk add --no-cache --virtual build-base g++ make py3-pip sqlite-dev python3 RUN npm config set python `which python3` --global From 2bafbe7f1b89e37c97c569daaed3e0c51bc5d283 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Sun, 20 Aug 2023 22:55:09 +0200 Subject: [PATCH 083/192] Define private npm registry url --- .github/workflows/flowforge-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index fca0ac99..98146792 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -28,6 +28,7 @@ jobs: @flowforge/flowforge @flowforge/kubernetes build_context: 'flowforge-container' + npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} push_image: false secrets: npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} From fdbb6c94efc824e181ab32c4877d940bb31f1a97 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Sun, 20 Aug 2023 23:24:40 +0200 Subject: [PATCH 084/192] Build debug --- .github/workflows/flowforge-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 98146792..9eae3489 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,4 +1,5 @@ name: Build Flowforge container + on: workflow_dispatch: inputs: From 42c6f97ef69b525ae5d17d52041129d0db76c146 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Sun, 20 Aug 2023 23:34:16 +0200 Subject: [PATCH 085/192] Debug --- .github/workflows/flowforge-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 9eae3489..98146792 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,5 +1,4 @@ name: Build Flowforge container - on: workflow_dispatch: inputs: From 1873dcc3ffe5c255b80b28772a34e80b50f4b3b2 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Sun, 20 Aug 2023 23:50:08 +0200 Subject: [PATCH 086/192] Debug --- .github/workflows/flowforge-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 98146792..9eae3489 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,4 +1,5 @@ name: Build Flowforge container + on: workflow_dispatch: inputs: From 32cdc4d3909f47f6c4f950b28baee84b75fd1892 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Sun, 20 Aug 2023 23:56:21 +0200 Subject: [PATCH 087/192] Debug --- .github/workflows/flowforge-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 9eae3489..98146792 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,5 +1,4 @@ name: Build Flowforge container - on: workflow_dispatch: inputs: From 0462a2d1f175a4bb276d1ffc3887cb931b27e7d1 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 07:10:48 +0200 Subject: [PATCH 088/192] Debug --- .github/workflows/flowforge-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 98146792..9eae3489 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,4 +1,5 @@ name: Build Flowforge container + on: workflow_dispatch: inputs: From f0bfdc7467f50f54e83e393138b2e19981dce343 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 09:55:16 +0200 Subject: [PATCH 089/192] Cleanup --- .github/workflows/flowforge-container.yml | 73 ----------------------- 1 file changed, 73 deletions(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 9eae3489..babc6a1a 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -33,79 +33,6 @@ jobs: push_image: false secrets: npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} - directBuild: - runs-on: ubuntu-latest - if: false - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Docker buildx - uses: docker/setup-buildx-action@v2 - - - name: Set release name and image tag - # if: ${{ github.event_name == 'schedule' }} - run: | - echo "release_name=nightly" >> $GITHUB_ENV - echo "npm_registry=https://npm.pkg.github.com" >> $GITHUB_ENV - echo "image_tag=nightly-$(date +%Y%m%d%H%m%S)" >> $GITHUB_ENV - - - name: "Set dependencies versions" - run: | - cat flowforge-container/package.json | jq '.dependencies["@flowforge/flowforge"] = "${{ env.release_name }}" | .dependencies["@flowforge/kubernetes"] = "${{ env.release_name }}" ' > flowforge-container/package.json-patched - mv flowforge-container/package.json-patched flowforge-container/package.json - - # - name: Configure AWS credentials - # if: ${{ env.release_name == 'nightly' }} - # uses: aws-actions/configure-aws-credentials@v2 - # with: - # aws-access-key-id: ${{ secrets.STAGING_AWS_ID }} - # aws-secret-access-key: ${{ secrets.STAGING_AWS_KEY }} - # aws-region: eu-west-1 - - # - name: "Login to Staging ECR" - # if: ${{ env.release_name == 'nightly' }} - # id: login-ecr - # uses: aws-actions/amazon-ecr-login@v1 - - # - name: Get image metadata - # id: image_metadata - # uses: docker/metadata-action@v4 - # with: - # images: | - # ${{ steps.login-ecr.outputs.registry }}/flowforge/forge-k8s - # tags: | - # type=raw,value=${{ env.image_tag }} - # flavor: | - # latest=false - - - name: Build container image - uses: docker/build-push-action@v4 - with: - context: flowforge-container - file: flowforge-container/Dockerfile - tags: "forge-k8s:${{ env.image_tag }}" - platforms: | - linux/amd64 - linux/arm64 - build-args: | - REGISTRY=${{ env.npm_registry }} - REGISTRY_TOKEN=${{ secrets.GITHUB_TOKEN }} - push: false - - # - name: Build and push Forge container - # uses: docker/build-push-action@v3.1.1 - # with: - # context: flowforge-container - # file: flowforge-container/Dockerfile - # tags: ${{ steps.image_metadata.outputs.tags }} - # platforms: | - # linux/amd64 - # linux/arm64 - # build-args: | - # REGISTRY=npm.pkg.github.com - # REGISTRY_TOKEN=${{ secrets.GITHUB_TOKEN }} - # push: true dispatch: runs-on: ubuntu-latest From 9e3675be1629a2bc2cdc7d82eec8ba8db784a927 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 09:56:07 +0200 Subject: [PATCH 090/192] Test --- .github/workflows/flowforge-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index babc6a1a..a6a9744b 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -30,7 +30,6 @@ jobs: @flowforge/kubernetes build_context: 'flowforge-container' npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} - push_image: false secrets: npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} From d59271b05befceb6f2c318be9214868dee86bb8a Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 11:52:52 +0200 Subject: [PATCH 091/192] Add stage build --- .github/workflows/flowforge-container.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index a6a9744b..f8ee2e8c 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -20,10 +20,11 @@ on: - main jobs: - build: + build-stage: if: true uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-container-build with: + environment: 'stage' image_name: 'forge-k8s' package_dependencies: | @flowforge/flowforge From 099aa7a649ec1f1e18b53fe6b4d5df6fe9440792 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 11:53:30 +0200 Subject: [PATCH 092/192] Dependency fix --- .github/workflows/flowforge-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index f8ee2e8c..62660825 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -36,7 +36,7 @@ jobs: dispatch: runs-on: ubuntu-latest - needs: [build] + needs: [build-stage] if: false steps: - name: Generate a token From 72c7d57dc5843d8a020eed893869e9d00fa3df54 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 11:59:07 +0200 Subject: [PATCH 093/192] Adjust secrets --- .github/workflows/flowforge-container.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 62660825..08d7f544 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -33,6 +33,8 @@ jobs: npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} secrets: npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} + aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} dispatch: runs-on: ubuntu-latest From 8a382e5beaab429885caf3efab48f0ecc8081b4f Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 12:04:44 +0200 Subject: [PATCH 094/192] Test --- .github/workflows/flowforge-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 08d7f544..e44b386f 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,5 +1,4 @@ name: Build Flowforge container - on: workflow_dispatch: inputs: From be6ada3d1dab15073b683b2ea3a9d6078239cbb0 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 12:07:29 +0200 Subject: [PATCH 095/192] Test --- .github/workflows/flowforge-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index e44b386f..08d7f544 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,4 +1,5 @@ name: Build Flowforge container + on: workflow_dispatch: inputs: From fe754f179f2d6aabe0aadcc1bfe40d3c5ac1e557 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 12:09:59 +0200 Subject: [PATCH 096/192] Test --- .github/workflows/flowforge-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 08d7f544..e44b386f 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,5 +1,4 @@ name: Build Flowforge container - on: workflow_dispatch: inputs: From 5bee6bd00b61fcdd27e736e8e9b7fa7f2b26a50f Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 12:20:24 +0200 Subject: [PATCH 097/192] Test --- .github/workflows/flowforge-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index e44b386f..08d7f544 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,4 +1,5 @@ name: Build Flowforge container + on: workflow_dispatch: inputs: From 63b1f38ec1fbb36373a7e2a128571f5d9559afa7 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 12:22:02 +0200 Subject: [PATCH 098/192] Test --- .github/workflows/flowforge-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 08d7f544..e44b386f 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,5 +1,4 @@ name: Build Flowforge container - on: workflow_dispatch: inputs: From 493f6a9b1bbeded0bcefe9ea9f42f5b91e2994d3 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 12:27:17 +0200 Subject: [PATCH 099/192] Test --- .github/workflows/flowforge-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index e44b386f..08d7f544 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,4 +1,5 @@ name: Build Flowforge container + on: workflow_dispatch: inputs: From 0a8b627ed54f2f7e7fc3943e6f6b8f3acb520df8 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 12:46:56 +0200 Subject: [PATCH 100/192] Add main banch to push triger --- .github/workflows/flowforge-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 08d7f544..8c525342 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -15,6 +15,7 @@ on: push: branches: - 'feat-*' + - main pull_request: branches: - main From 430929b910cac23032001b1123880c37789be295 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 13:02:20 +0200 Subject: [PATCH 101/192] Add file-server image build pipeline --- .github/workflows/fileserver-container.yml | 60 ++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/fileserver-container.yml diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml new file mode 100644 index 00000000..2974aa98 --- /dev/null +++ b/.github/workflows/fileserver-container.yml @@ -0,0 +1,60 @@ +# file-server +name: Build file-server container + +on: + workflow_dispatch: + inputs: + fileserver_release_name: + description: 'file-server package version' + required: false + default: 'nightly' + fileserver_ref: + description: 'file-server package ref' + required: false + schedule: + - cron: '30 5 * * *' + push: + branches: + - main + - 'feat-*' + pull_request: + branches: + - main + +jobs: + build-stage: + if: true + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-container-build + with: + environment: 'stage' + image_name: 'file-server' + package_dependencies: | + @flowforge/file-server + build_context: 'file-server' + npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} + secrets: + npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} + aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} + + dispatch: + runs-on: ubuntu-latest + needs: [build-stage] + if: false + steps: + - name: Generate a token + id: generate_token + uses: tibdex/github-app-token@v1 + with: + app_id: ${{ secrets.GH_BOT_APP_ID }} + private_key: ${{ secrets.GH_BOT_APP_KEY }} + + - name: Trigger cloud deployment + uses: benc-uk/workflow-dispatch@v1 + with: + workflow: deploy.yml + repo: flowforge/CloudProject + ref: feat-deployment-poc + token: ${{ steps.generate_token.outputs.token }} + inputs: '{"fileserver_image_tag": "${{ env.image_tag }}"}' + \ No newline at end of file From 0d08293e784a607cb32cf6cc04f2a35724181127 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 21:47:59 +0200 Subject: [PATCH 102/192] Add node-red image build pipeline --- .github/workflows/nodered-container.yml | 51 +++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/nodered-container.yml diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml new file mode 100644 index 00000000..f55a0e77 --- /dev/null +++ b/.github/workflows/nodered-container.yml @@ -0,0 +1,51 @@ +name: Build Node-RED container + +on: + workflow_dispatch: + inputs: + nr_project_nodes_release_name: + description: 'nr-project-nodes package version' + required: false + default: 'nightly' + nr_project_nodes_ref: + description: 'nr-project-nodes package ref' + required: false + schedule: + - cron: '30 5 * * *' + push: + branches: + - 'feat-*' + - main + pull_request: + branches: + - main + +jobs: + build-stage: + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-container-build + with: + environment: 'stage' + image_name: 'node-red' + package_dependencies: | + @flowforge/nr-project-nodes + build_context: 'node-red-container' + npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} + secrets: + npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} + aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} + build-stage-223: + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-container-build + with: + environment: 'stage' + image_name: 'node-red' + image_tag_prefix: '2.2.3-' + package_dependencies: | + @flowforge/nr-project-nodes + build_context: 'node-red-container' + npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} + secrets: + npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} + aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} + \ No newline at end of file From e62179ad8b9223cc3408ad7f190280bda4c896c3 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 22:06:15 +0200 Subject: [PATCH 103/192] Test pipeline --- .github/workflows/nodered-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index f55a0e77..62f58a12 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -1,5 +1,4 @@ name: Build Node-RED container - on: workflow_dispatch: inputs: From b7389bf53580501f2044e4fa2c1f28276f4defda Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 22:12:24 +0200 Subject: [PATCH 104/192] Test pipeline --- .github/workflows/nodered-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index 62f58a12..f55a0e77 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -1,4 +1,5 @@ name: Build Node-RED container + on: workflow_dispatch: inputs: From d29900ec08ed2b148fa6f04a1639334b030947fe Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 22:15:28 +0200 Subject: [PATCH 105/192] Test pipeline --- .github/workflows/nodered-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index f55a0e77..62f58a12 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -1,5 +1,4 @@ name: Build Node-RED container - on: workflow_dispatch: inputs: From 872e544ec163638b8f58cf97d2e28dbba892b0fa Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 22:21:12 +0200 Subject: [PATCH 106/192] Test pipeline --- .github/workflows/nodered-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index 62f58a12..f55a0e77 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -1,4 +1,5 @@ name: Build Node-RED container + on: workflow_dispatch: inputs: From 1212d83f511e487110daa9a97955bd1cb3684932 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 22:26:46 +0200 Subject: [PATCH 107/192] Test pipeline --- .github/workflows/nodered-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index f55a0e77..62f58a12 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -1,5 +1,4 @@ name: Build Node-RED container - on: workflow_dispatch: inputs: From 2a7aa92906805effefa287983d5dd6d281f4d2a0 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 22:34:23 +0200 Subject: [PATCH 108/192] Test pipeline --- .github/workflows/nodered-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index 62f58a12..f55a0e77 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -1,4 +1,5 @@ name: Build Node-RED container + on: workflow_dispatch: inputs: From bd972138550a41ff07ecebb1adedb0ae28a03fdd Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 21 Aug 2023 23:00:44 +0200 Subject: [PATCH 109/192] Test pipeline --- .github/workflows/nodered-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index f55a0e77..62f58a12 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -1,5 +1,4 @@ name: Build Node-RED container - on: workflow_dispatch: inputs: From f791ee5d2fce8ee91ba5da6fc21185cb8b7aa5a2 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Tue, 22 Aug 2023 10:44:30 +0200 Subject: [PATCH 110/192] Prepare for PR --- .github/workflows/fileserver-container.yml | 26 +--------------------- .github/workflows/flowforge-container.yml | 26 +--------------------- .github/workflows/nodered-container.yml | 1 - 3 files changed, 2 insertions(+), 51 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 2974aa98..7a67e85f 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -1,6 +1,5 @@ # file-server name: Build file-server container - on: workflow_dispatch: inputs: @@ -16,15 +15,13 @@ on: push: branches: - main - - 'feat-*' pull_request: branches: - main jobs: build-stage: - if: true - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-container-build + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'stage' image_name: 'file-server' @@ -36,25 +33,4 @@ jobs: npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} - - dispatch: - runs-on: ubuntu-latest - needs: [build-stage] - if: false - steps: - - name: Generate a token - id: generate_token - uses: tibdex/github-app-token@v1 - with: - app_id: ${{ secrets.GH_BOT_APP_ID }} - private_key: ${{ secrets.GH_BOT_APP_KEY }} - - - name: Trigger cloud deployment - uses: benc-uk/workflow-dispatch@v1 - with: - workflow: deploy.yml - repo: flowforge/CloudProject - ref: feat-deployment-poc - token: ${{ steps.generate_token.outputs.token }} - inputs: '{"fileserver_image_tag": "${{ env.image_tag }}"}' \ No newline at end of file diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 8c525342..85d162ba 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,5 +1,4 @@ name: Build Flowforge container - on: workflow_dispatch: inputs: @@ -14,7 +13,6 @@ on: - cron: '30 5 * * *' push: branches: - - 'feat-*' - main pull_request: branches: @@ -22,8 +20,7 @@ on: jobs: build-stage: - if: true - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-container-build + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'stage' image_name: 'forge-k8s' @@ -36,25 +33,4 @@ jobs: npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} - - dispatch: - runs-on: ubuntu-latest - needs: [build-stage] - if: false - steps: - - name: Generate a token - id: generate_token - uses: tibdex/github-app-token@v1 - with: - app_id: ${{ secrets.GH_BOT_APP_ID }} - private_key: ${{ secrets.GH_BOT_APP_KEY }} - - - name: Trigger cloud deployment - uses: benc-uk/workflow-dispatch@v1 - with: - workflow: deploy.yml - repo: flowforge/CloudProject - ref: feat-deployment-poc - token: ${{ steps.generate_token.outputs.token }} - inputs: '{"flowforge_image_tag": "${{ env.image_tag }}"}' \ No newline at end of file diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index 62f58a12..204cb03a 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -13,7 +13,6 @@ on: - cron: '30 5 * * *' push: branches: - - 'feat-*' - main pull_request: branches: From ec14f7641dc6e6900f99f3017ec777eeb7883585 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Tue, 22 Aug 2023 10:54:10 +0200 Subject: [PATCH 111/192] Prepare for PR --- .github/workflows/nodered-container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index 204cb03a..1f52f2b5 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -20,7 +20,7 @@ on: jobs: build-stage: - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-container-build + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'stage' image_name: 'node-red' @@ -33,7 +33,7 @@ jobs: aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} build-stage-223: - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-container-build + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'stage' image_name: 'node-red' From 205b7b20e6148370347887d4df5b4fae4fc6ef64 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 24 Aug 2023 13:14:27 +0200 Subject: [PATCH 112/192] Add deploy-stage job --- .github/workflows/flowforge-container.yml | 31 +++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 85d162ba..92f5959a 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -20,7 +20,7 @@ on: jobs: build-stage: - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-image-tag-output with: environment: 'stage' image_name: 'forge-k8s' @@ -33,4 +33,31 @@ jobs: npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} - \ No newline at end of file + + deploy-stage: + name: Deploy to staging environment + environment: stage + runs-on: ubuntu-latest + needs: build-stage + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.STAGING_AWS_ID }} + aws-secret-access-key: ${{ secrets.STAGING_AWS_KEY }} + aws-region: eu-west-1 + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/K8sAdmin + role-duration-seconds: 1200 + + - name: Setup kubectl + uses: azure/setup-kubectl@v3 + with: + version: '1.23.4' + + - name: Configure kubectl + run: + aws eks update-kubeconfig --region eu-west-1 --name ${{ secrets.EKS_CLUSTER_NAME }} + + - name: Debug kubectl + run: | + kubectl -n default get pods -l app=ubuntu -o jsonpath="{.items[*].status.phase}" From 50c613f5636473bd2f3df5a7e200a71413f59a78 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 24 Aug 2023 13:36:09 +0200 Subject: [PATCH 113/192] Test container build job --- .github/workflows/flowforge-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 92f5959a..b988739d 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,4 +1,5 @@ name: Build Flowforge container + on: workflow_dispatch: inputs: From 21c6ea80b677ff78793bfb94092d99a7cd013945 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 24 Aug 2023 13:52:51 +0200 Subject: [PATCH 114/192] Temporarily build only flowforge container --- .github/workflows/fileserver-container.yml | 1 + .github/workflows/nodered-container.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 7a67e85f..33bcf1c3 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -21,6 +21,7 @@ on: jobs: build-stage: + if: false uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'stage' diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index 1f52f2b5..53a43165 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -20,6 +20,7 @@ on: jobs: build-stage: + if: false uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'stage' @@ -33,6 +34,7 @@ jobs: aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} build-stage-223: + if: false uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'stage' From a39b3774ca58af7f5a450772404a1843d45a6003 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 24 Aug 2023 14:56:11 +0200 Subject: [PATCH 115/192] Set proper kubectl version --- .github/workflows/flowforge-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index b988739d..4a270671 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -53,7 +53,7 @@ jobs: - name: Setup kubectl uses: azure/setup-kubectl@v3 with: - version: '1.23.4' + version: 'v1.23.4' - name: Configure kubectl run: From b092d40bc544eb7065d8cfd8f1e9f27ae8a554f7 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 25 Aug 2023 07:31:06 +0200 Subject: [PATCH 116/192] Debug build job outputs --- .github/workflows/flowforge-container.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 4a270671..7de43104 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -62,3 +62,7 @@ jobs: - name: Debug kubectl run: | kubectl -n default get pods -l app=ubuntu -o jsonpath="{.items[*].status.phase}" + + - name: Debug previous job outputs + run: | + echo "Image tag: ${{ needs.build-stage.outputs.private_image_tag }}" From a74b1a3727bba4526e0c3e0f3aea067c1c128746 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 25 Aug 2023 07:51:21 +0200 Subject: [PATCH 117/192] Debug build job outputs --- .github/workflows/flowforge-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 7de43104..9fc5a7a0 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -66,3 +66,4 @@ jobs: - name: Debug previous job outputs run: | echo "Image tag: ${{ needs.build-stage.outputs.private_image_tag }}" + kubectl run test --rm --image=${{ needs.build-stage.outputs.private_image_tag }} --restart=Never --command -- sleep 3600 From af05999dbf91c79e031c677c476523fbdc9b402e Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 25 Aug 2023 09:10:04 +0200 Subject: [PATCH 118/192] Debug outputs --- .github/workflows/flowforge-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 9fc5a7a0..7de43104 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -66,4 +66,3 @@ jobs: - name: Debug previous job outputs run: | echo "Image tag: ${{ needs.build-stage.outputs.private_image_tag }}" - kubectl run test --rm --image=${{ needs.build-stage.outputs.private_image_tag }} --restart=Never --command -- sleep 3600 From fb3f2e06ad2a928096321186fe451c3f27a07991 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 25 Aug 2023 10:26:14 +0200 Subject: [PATCH 119/192] Debug outputs --- .github/workflows/flowforge-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 7de43104..b41ac416 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -66,3 +66,4 @@ jobs: - name: Debug previous job outputs run: | echo "Image tag: ${{ needs.build-stage.outputs.private_image_tag }}" + From be5fbfcb851f77c2e1ec390ca4d6490d748b9460 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 25 Aug 2023 13:54:13 +0200 Subject: [PATCH 120/192] Debug outputs --- .github/workflows/flowforge-container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index b41ac416..a2d75b86 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -65,5 +65,5 @@ jobs: - name: Debug previous job outputs run: | - echo "Image tag: ${{ needs.build-stage.outputs.private_image_tag }}" - + echo "Image tag: ${{ needs.build-stage.outputs.private_image }}" + echo "Image tag: ${{ secrets.AWS_ACCOUNT_ID }}".dkr.ecr.eu-west-1.amazonaws.com/flowforge/${{ needs.build-stage.outputs.private_image }} From 95ac6d1c586b2d5b597a605a1535ecb93740ab07 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 28 Aug 2023 09:58:37 +0200 Subject: [PATCH 121/192] Perform stage deployment --- .github/workflows/flowforge-container.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index a2d75b86..85b5fe9f 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -59,11 +59,8 @@ jobs: run: aws eks update-kubeconfig --region eu-west-1 --name ${{ secrets.EKS_CLUSTER_NAME }} - - name: Debug kubectl + - name: Update image and wait for deployment to finish run: | - kubectl -n default get pods -l app=ubuntu -o jsonpath="{.items[*].status.phase}" + kubectl -n defaut set image deployment/flowforge forge=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/forge-k8s:${{ needs.build-stage.outputs.private_image }} --record + kubectl -n default rollout status deployment/flowforge - - name: Debug previous job outputs - run: | - echo "Image tag: ${{ needs.build-stage.outputs.private_image }}" - echo "Image tag: ${{ secrets.AWS_ACCOUNT_ID }}".dkr.ecr.eu-west-1.amazonaws.com/flowforge/${{ needs.build-stage.outputs.private_image }} From d875725d540efe42c52a49db25f6842af0073c78 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 28 Aug 2023 10:05:01 +0200 Subject: [PATCH 122/192] Namespace typo --- .github/workflows/flowforge-container.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 85b5fe9f..68cfd14f 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -61,6 +61,5 @@ jobs: - name: Update image and wait for deployment to finish run: | - kubectl -n defaut set image deployment/flowforge forge=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/forge-k8s:${{ needs.build-stage.outputs.private_image }} --record + kubectl -n default set image deployment/flowforge forge=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/forge-k8s:${{ needs.build-stage.outputs.private_image }} --record kubectl -n default rollout status deployment/flowforge - From 064ab6cc2dfd0f7ee05f0eab08f56bcdd5c6083f Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 28 Aug 2023 10:18:02 +0200 Subject: [PATCH 123/192] Add rollback step --- .github/workflows/flowforge-container.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 68cfd14f..3bc92e79 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -60,6 +60,13 @@ jobs: aws eks update-kubeconfig --region eu-west-1 --name ${{ secrets.EKS_CLUSTER_NAME }} - name: Update image and wait for deployment to finish + timeout-minutes: 5 run: | kubectl -n default set image deployment/flowforge forge=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/forge-k8s:${{ needs.build-stage.outputs.private_image }} --record kubectl -n default rollout status deployment/flowforge + + - name: Rollback failed deployment + if: failure() + run: | + kubectl -n default rollout undo deployment/flowforge + kubectl -n default rollout status deployment/flowforge From 532e696d3ce882b931573d9eb9078638d4c6d2ac Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 28 Aug 2023 10:41:57 +0200 Subject: [PATCH 124/192] Fix image name --- .github/workflows/flowforge-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 3bc92e79..58f96cf1 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -62,7 +62,7 @@ jobs: - name: Update image and wait for deployment to finish timeout-minutes: 5 run: | - kubectl -n default set image deployment/flowforge forge=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/forge-k8s:${{ needs.build-stage.outputs.private_image }} --record + kubectl -n default set image deployment/flowforge forge=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/${{ needs.build-stage.outputs.private_image }} kubectl -n default rollout status deployment/flowforge - name: Rollback failed deployment From 17a1c7ced7aa757f84cdbd3a97e39f2a202e5f60 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 28 Aug 2023 14:17:25 +0200 Subject: [PATCH 125/192] Add production environment jobs --- .github/workflows/flowforge-container.yml | 56 ++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 58f96cf1..105fc21e 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -21,6 +21,7 @@ on: jobs: build-stage: + name: Build stage image uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-image-tag-output with: environment: 'stage' @@ -35,6 +36,22 @@ jobs: aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} + build-prod: + name: Build production image + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-image-tag-output + with: + environment: 'production' + image_name: 'forge-k8s' + package_dependencies: | + @flowforge/flowforge + @flowforge/kubernetes + build_context: 'flowforge-container' + npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} + secrets: + npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + aws_access_key_id: ${{ secrets.PRODUCTION_AWS_ID }} + aws_access_key_secret: ${{ secrets.PRODUCTION_AWS_KEY }} + deploy-stage: name: Deploy to staging environment environment: stage @@ -62,7 +79,44 @@ jobs: - name: Update image and wait for deployment to finish timeout-minutes: 5 run: | - kubectl -n default set image deployment/flowforge forge=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/${{ needs.build-stage.outputs.private_image }} + kubectl -n default set image deployment/flowforge forge=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/${{ needs.build-stage.outputs.image }} + kubectl -n default rollout status deployment/flowforge + + - name: Rollback failed deployment + if: failure() + run: | + kubectl -n default rollout undo deployment/flowforge + kubectl -n default rollout status deployment/flowforge + + deploy-prod: + if: false + name: Deploy to production environment + environment: production + runs-on: ubuntu-latest + needs: [build-prod, deploy-stage] + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.PRODUCTION_AWS_ID }} + aws-secret-access-key: ${{ secrets.PRODUCTION_AWS_KEY }} + aws-region: eu-west-1 + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/K8sAdmin + role-duration-seconds: 1200 + + - name: Setup kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'v1.23.4' + + - name: Configure kubectl + run: + aws eks update-kubeconfig --region eu-west-1 --name ${{ secrets.EKS_CLUSTER_NAME }} + + - name: Update image and wait for deployment to finish + timeout-minutes: 5 + run: | + kubectl -n default set image deployment/flowforge forge=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/${{ needs.build-prod.outputs.image }} kubectl -n default rollout status deployment/flowforge - name: Rollback failed deployment From 536b72f9939d5da8a7c5f90698b5f93341f7510a Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 28 Aug 2023 15:36:24 +0200 Subject: [PATCH 126/192] Add fileserver stage and prod jobs --- .github/workflows/fileserver-container.yml | 94 +++++++++++++++++++++- 1 file changed, 91 insertions(+), 3 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 33bcf1c3..a344a7ab 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -21,8 +21,7 @@ on: jobs: build-stage: - if: false - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-image-tag-output with: environment: 'stage' image_name: 'file-server' @@ -34,4 +33,93 @@ jobs: npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} - \ No newline at end of file + + build-prod: + name: Build production image + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-image-tag-output + with: + environment: 'production' + image_name: 'forge-k8s' + package_dependencies: | + @flowforge/flowforge + @flowforge/kubernetes + build_context: 'flowforge-container' + npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} + secrets: + npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + aws_access_key_id: ${{ secrets.PRODUCTION_AWS_ID }} + aws_access_key_secret: ${{ secrets.PRODUCTION_AWS_KEY }} + + deploy-stage: + name: Deploy to staging environment + environment: stage + runs-on: ubuntu-latest + needs: build-stage + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.STAGING_AWS_ID }} + aws-secret-access-key: ${{ secrets.STAGING_AWS_KEY }} + aws-region: eu-west-1 + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/K8sAdmin + role-duration-seconds: 1200 + + - name: Setup kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'v1.23.4' + + - name: Configure kubectl + run: + aws eks update-kubeconfig --region eu-west-1 --name ${{ secrets.EKS_CLUSTER_NAME }} + + - name: Update image and wait for deployment to finish + timeout-minutes: 5 + run: | + kubectl -n default set image deployment/flowforge-file file-storage=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/${{ needs.build-stage.outputs.image }} + kubectl -n default rollout status deployment/flowforge-file + + - name: Rollback failed deployment + if: failure() + run: | + kubectl -n default rollout undo deployment/flowforge-file + kubectl -n default rollout status deployment/flowforge-file + + deploy-prod: + if: false + name: Deploy to production environment + environment: production + runs-on: ubuntu-latest + needs: [build-prod, deploy-stage] + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.PRODUCTION_AWS_ID }} + aws-secret-access-key: ${{ secrets.PRODUCTION_AWS_KEY }} + aws-region: eu-west-1 + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/K8sAdmin + role-duration-seconds: 1200 + + - name: Setup kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'v1.23.4' + + - name: Configure kubectl + run: + aws eks update-kubeconfig --region eu-west-1 --name ${{ secrets.EKS_CLUSTER_NAME }} + + - name: Update image and wait for deployment to finish + timeout-minutes: 5 + run: | + kubectl -n default set image deployment/flowforge-file file-storage=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/${{ needs.build-prod.outputs.image }} + kubectl -n default rollout status deployment/flowforge-file + + - name: Rollback failed deployment + if: failure() + run: | + kubectl -n default rollout undo deployment/flowforge-file + kubectl -n default rollout status deployment/flowforge-file + \ No newline at end of file From 36e12d2c7d884a849e88fc842a8f295f675f15b3 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 28 Aug 2023 15:53:24 +0200 Subject: [PATCH 127/192] File-server prod builds parameters --- .github/workflows/fileserver-container.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index a344a7ab..275cf89d 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -39,11 +39,10 @@ jobs: uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-image-tag-output with: environment: 'production' - image_name: 'forge-k8s' + image_name: 'file-server' package_dependencies: | - @flowforge/flowforge - @flowforge/kubernetes - build_context: 'flowforge-container' + @flowforge/file-server + build_context: 'file-server' npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} secrets: npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} From e9c5818a50646d4ad52d9b958b87b989fb80a2c9 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Tue, 29 Aug 2023 07:23:36 +0200 Subject: [PATCH 128/192] Cleanup for PR --- .github/workflows/fileserver-container.yml | 5 ++--- .github/workflows/flowforge-container.yml | 5 ++--- .github/workflows/nodered-container.yml | 2 -- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 275cf89d..d4f595eb 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -1,4 +1,3 @@ -# file-server name: Build file-server container on: workflow_dispatch: @@ -21,7 +20,7 @@ on: jobs: build-stage: - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-image-tag-output + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'stage' image_name: 'file-server' @@ -36,7 +35,7 @@ jobs: build-prod: name: Build production image - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-image-tag-output + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'production' image_name: 'file-server' diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 105fc21e..b912aa63 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,5 +1,4 @@ name: Build Flowforge container - on: workflow_dispatch: inputs: @@ -22,7 +21,7 @@ on: jobs: build-stage: name: Build stage image - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-image-tag-output + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'stage' image_name: 'forge-k8s' @@ -38,7 +37,7 @@ jobs: build-prod: name: Build production image - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@feat-image-tag-output + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'production' image_name: 'forge-k8s' diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index 53a43165..1f52f2b5 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -20,7 +20,6 @@ on: jobs: build-stage: - if: false uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'stage' @@ -34,7 +33,6 @@ jobs: aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} build-stage-223: - if: false uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'stage' From 687324736d30e15c1936a80a8eeb98f9ad55ae08 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Wed, 30 Aug 2023 15:04:43 +0200 Subject: [PATCH 129/192] Disable production containers build --- .github/workflows/fileserver-container.yml | 1 + .github/workflows/flowforge-container.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index d4f595eb..1e1c5fdc 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -35,6 +35,7 @@ jobs: build-prod: name: Build production image + if: false uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'production' diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index b912aa63..5f9405da 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -37,6 +37,7 @@ jobs: build-prod: name: Build production image + if: false uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'production' From 3e233a8a96c8d2f65c13c484ba718e26dbf74d6f Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 31 Aug 2023 13:38:13 +0200 Subject: [PATCH 130/192] Update CHANGELOG for 1.11.0 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e6b79d4..9d2e50bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +#### 1.11.0: Release + + - Disable production containers build (#167) @ppawlowski + - Add possibility to deploy flowforge container to stage environment (#165) @ppawlowski + - Introduce dedicated containers images build pipeline (#163) @ppawlowski + #### 1.10.0: Release - Revert create-pod Role name (#159) @hardillb From 3e8245debd3326d5766f24ba1997ff998dbab6e9 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 31 Aug 2023 13:38:14 +0200 Subject: [PATCH 131/192] Update version to 1.11.0 --- VERSION | 2 +- file-server/package.json | 4 ++-- flowforge-container/package.json | 6 +++--- helm/flowforge/Chart.yaml | 4 ++-- node-red-container/package.json | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/VERSION b/VERSION index 81c871de..1cac385c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.10.0 +1.11.0 diff --git a/file-server/package.json b/file-server/package.json index 4f6779c5..f42e2197 100644 --- a/file-server/package.json +++ b/file-server/package.json @@ -1,9 +1,9 @@ { "name": "@flowforge/file-server-container", - "version": "1.10.0", + "version": "1.11.0", "private": true, "dependencies": { - "@flowforge/file-server": "^1.10.0" + "@flowforge/file-server": "^1.11.0" }, "license": "Apache-2.0" } diff --git a/flowforge-container/package.json b/flowforge-container/package.json index b36d9e6f..c694dc9f 100644 --- a/flowforge-container/package.json +++ b/flowforge-container/package.json @@ -1,14 +1,14 @@ { "name": "@flowforge/flowforge-k8s", - "version": "1.10.0", + "version": "1.11.0", "description": "FlowForge in Docker", "private": true, "author": { "name": "FlowForge Inc." }, "dependencies": { - "@flowforge/flowforge": "^1.10.0", - "@flowforge/kubernetes": "^1.10.0", + "@flowforge/flowforge": "^1.11.0", + "@flowforge/kubernetes": "^1.11.0", "pg": "^8.7.1", "pg-hstore": "^2.3.4" }, diff --git a/helm/flowforge/Chart.yaml b/helm/flowforge/Chart.yaml index 88416f5d..c041d084 100644 --- a/helm/flowforge/Chart.yaml +++ b/helm/flowforge/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "flowforge" -version: "1.10.0" +version: "1.11.0" description: "FlowForge" type: "application" home: "https://flowforge.com" @@ -16,4 +16,4 @@ dependencies: maintainers: - name: "FlowForge Inc" url: "https://flowforge.com" -appVersion: "1.10.0" +appVersion: "1.11.0" diff --git a/node-red-container/package.json b/node-red-container/package.json index d8303285..90c369ce 100644 --- a/node-red-container/package.json +++ b/node-red-container/package.json @@ -1,7 +1,7 @@ { "name": "node-red-project", "description": "A Node-RED Project", - "version": "1.10.0", + "version": "1.11.0", "private": true, "dependencies": {} } From 3e04078a244f831e4ed43ad8de8a734cc8c20978 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Tue, 5 Sep 2023 07:18:30 +0200 Subject: [PATCH 132/192] Disable scheduled images build --- .github/workflows/fileserver-container.yml | 2 -- .github/workflows/flowforge-container.yml | 2 -- .github/workflows/nodered-container.yml | 2 -- 3 files changed, 6 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 1e1c5fdc..bf210182 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -9,8 +9,6 @@ on: fileserver_ref: description: 'file-server package ref' required: false - schedule: - - cron: '30 5 * * *' push: branches: - main diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 5f9405da..fc2623ef 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -9,8 +9,6 @@ on: flowforge_ref: description: 'flowforge package ref' required: false - schedule: - - cron: '30 5 * * *' push: branches: - main diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index 1f52f2b5..e6169f68 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -9,8 +9,6 @@ on: nr_project_nodes_ref: description: 'nr-project-nodes package ref' required: false - schedule: - - cron: '30 5 * * *' push: branches: - main From 9f8439f40518b463db33659ab3f4b79be0a92873 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Wed, 6 Sep 2023 16:02:01 +0100 Subject: [PATCH 133/192] Update Node-RED container to 3.1.0 final --- node-red-container/Dockerfile-3.1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node-red-container/Dockerfile-3.1 b/node-red-container/Dockerfile-3.1 index 9634714e..c2d13a3d 100644 --- a/node-red-container/Dockerfile-3.1 +++ b/node-red-container/Dockerfile-3.1 @@ -1,4 +1,4 @@ -FROM nodered/node-red-dev:3.1.0-beta.3-18 +FROM nodered/node-red:3.1.0-16 ARG REGISTRY ARG REGISTRY_TOKEN From 1c3cdad1c78157768c5e1c2239a4ae95ffe3432e Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Thu, 7 Sep 2023 10:47:28 +0100 Subject: [PATCH 134/192] Pin default Dockerfile to 3.0.x not latest --- node-red-container/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node-red-container/Dockerfile b/node-red-container/Dockerfile index fc586ec8..6288914c 100644 --- a/node-red-container/Dockerfile +++ b/node-red-container/Dockerfile @@ -1,4 +1,4 @@ -FROM nodered/node-red:latest +FROM nodered/node-red:3.0.2-16 ARG REGISTRY ARG REGISTRY_TOKEN From d62aefa58c0bc941ba192ffd120bf65321a07461 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Thu, 7 Sep 2023 10:47:56 +0100 Subject: [PATCH 135/192] Upate nr container build workflow to create 3.1 container --- .github/workflows/nodered-container.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index e6169f68..63c35727 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -17,11 +17,12 @@ on: - main jobs: - build-stage: + build-stage-302: uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main with: environment: 'stage' image_name: 'node-red' + dockerfile_path: Dockerfile package_dependencies: | @flowforge/nr-project-nodes build_context: 'node-red-container' @@ -36,6 +37,22 @@ jobs: environment: 'stage' image_name: 'node-red' image_tag_prefix: '2.2.3-' + dockerfile_path: Dockerfile-2.2.x + package_dependencies: | + @flowforge/nr-project-nodes + build_context: 'node-red-container' + npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} + secrets: + npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} + aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} + build-stage-310: + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main + with: + environment: 'stage' + image_name: 'node-red' + image_tag_prefix: '3.1.0-' + dockerfile_path: Dockerfile-3.1 package_dependencies: | @flowforge/nr-project-nodes build_context: 'node-red-container' @@ -44,4 +61,3 @@ jobs: npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} - \ No newline at end of file From 012ba59472f04ed30a96febe2a29e9f7ef5c07b8 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 11 Sep 2023 18:42:14 +0200 Subject: [PATCH 136/192] Pin workflow to commit SHA --- .github/workflows/fileserver-container.yml | 4 ++-- .github/workflows/flowforge-container.yml | 4 ++-- .github/workflows/nodered-container.yml | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index bf210182..1927f919 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -18,7 +18,7 @@ on: jobs: build-stage: - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 with: environment: 'stage' image_name: 'file-server' @@ -34,7 +34,7 @@ jobs: build-prod: name: Build production image if: false - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 with: environment: 'production' image_name: 'file-server' diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index fc2623ef..206bc5ad 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -19,7 +19,7 @@ on: jobs: build-stage: name: Build stage image - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 with: environment: 'stage' image_name: 'forge-k8s' @@ -36,7 +36,7 @@ jobs: build-prod: name: Build production image if: false - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 with: environment: 'production' image_name: 'forge-k8s' diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index 63c35727..722bc3f7 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -18,7 +18,7 @@ on: jobs: build-stage-302: - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 with: environment: 'stage' image_name: 'node-red' @@ -32,7 +32,7 @@ jobs: aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} build-stage-223: - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 with: environment: 'stage' image_name: 'node-red' @@ -47,7 +47,7 @@ jobs: aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} build-stage-310: - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@main + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 with: environment: 'stage' image_name: 'node-red' From 2a09d48e706336f92a07b833293de1645f07b401 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 18 Sep 2023 15:24:32 +0200 Subject: [PATCH 137/192] Build container images using packages from public npm registry --- .github/workflows/fileserver-container.yml | 8 ++++---- .github/workflows/flowforge-container.yml | 8 ++++---- .github/workflows/nodered-container.yml | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 1927f919..d374fb14 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -25,9 +25,9 @@ jobs: package_dependencies: | @flowforge/file-server build_context: 'file-server' - npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} + npm_registry_url: ${{ vars.PUBLIC_NPM_REGISTRY_URL }} secrets: - npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + npm_registry_auth_token: ${{ secrets.NPM_PUBLISH_TOKEN }} aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} @@ -41,9 +41,9 @@ jobs: package_dependencies: | @flowforge/file-server build_context: 'file-server' - npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} + npm_registry_url: ${{ vars.PUBLIC_NPM_REGISTRY_URL }} secrets: - npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + npm_registry_auth_token: ${{ secrets.NPM_PUBLISH_TOKEN }} aws_access_key_id: ${{ secrets.PRODUCTION_AWS_ID }} aws_access_key_secret: ${{ secrets.PRODUCTION_AWS_KEY }} diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 206bc5ad..b59b23d5 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -27,9 +27,9 @@ jobs: @flowforge/flowforge @flowforge/kubernetes build_context: 'flowforge-container' - npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} + npm_registry_url: ${{ vars.PUBLIC_NPM_REGISTRY_URL }} secrets: - npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + npm_registry_auth_token: ${{ secrets.NPM_PUBLISH_TOKEN }} aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} @@ -44,9 +44,9 @@ jobs: @flowforge/flowforge @flowforge/kubernetes build_context: 'flowforge-container' - npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} + npm_registry_url: ${{ vars.PUBLIC_NPM_REGISTRY_URL }} secrets: - npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + npm_registry_auth_token: ${{ secrets.NPM_PUBLISH_TOKEN }} aws_access_key_id: ${{ secrets.PRODUCTION_AWS_ID }} aws_access_key_secret: ${{ secrets.PRODUCTION_AWS_KEY }} diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index 722bc3f7..50b47bea 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -26,9 +26,9 @@ jobs: package_dependencies: | @flowforge/nr-project-nodes build_context: 'node-red-container' - npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} + npm_registry_url: ${{ vars.PUBLIC_NPM_REGISTRY_URL }} secrets: - npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + npm_registry_auth_token: ${{ secrets.NPM_PUBLISH_TOKEN }} aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} build-stage-223: @@ -41,9 +41,9 @@ jobs: package_dependencies: | @flowforge/nr-project-nodes build_context: 'node-red-container' - npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} + npm_registry_url: ${{ vars.PUBLIC_NPM_REGISTRY_URL }} secrets: - npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + npm_registry_auth_token: ${{ secrets.NPM_PUBLISH_TOKEN }} aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} build-stage-310: @@ -56,8 +56,8 @@ jobs: package_dependencies: | @flowforge/nr-project-nodes build_context: 'node-red-container' - npm_registry_url: ${{ vars.PRIVATE_NPM_REGISTRY_URL }} + npm_registry_url: ${{ vars.PUBLIC_NPM_REGISTRY_URL }} secrets: - npm_registry_auth_token: ${{ secrets.GITHUB_TOKEN }} + npm_registry_auth_token: ${{ secrets.NPM_PUBLISH_TOKEN }} aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} From 3dff300b2c66365776c4b62e4221beff0f2902ad Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Wed, 20 Sep 2023 07:54:43 +0200 Subject: [PATCH 138/192] Deploy image from temporary registry --- .github/workflows/fileserver-container.yml | 104 ++++++++++++++++----- 1 file changed, 82 insertions(+), 22 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index d374fb14..269f9555 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -17,10 +17,10 @@ on: - main jobs: - build-stage: - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 + build: + name: Build single-architecture container images + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@9d655b572fffebb432b259566b68be1a7aff7ed5 with: - environment: 'stage' image_name: 'file-server' package_dependencies: | @flowforge/file-server @@ -28,32 +28,58 @@ jobs: npm_registry_url: ${{ vars.PUBLIC_NPM_REGISTRY_URL }} secrets: npm_registry_auth_token: ${{ secrets.NPM_PUBLISH_TOKEN }} - aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} - aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} - - build-prod: - name: Build production image - if: false - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 + temporary_registry_token: ${{ secrets.GITHUB_TOKEN }} + + build-multi-architecture: + name: Build multi-architecture container image + needs: build + uses: flowforge/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@9d655b572fffebb432b259566b68be1a7aff7ed5 with: - environment: 'production' image_name: 'file-server' - package_dependencies: | - @flowforge/file-server - build_context: 'file-server' - npm_registry_url: ${{ vars.PUBLIC_NPM_REGISTRY_URL }} secrets: - npm_registry_auth_token: ${{ secrets.NPM_PUBLISH_TOKEN }} - aws_access_key_id: ${{ secrets.PRODUCTION_AWS_ID }} - aws_access_key_secret: ${{ secrets.PRODUCTION_AWS_KEY }} + temporary_registry_token: ${{ secrets.GITHUB_TOKEN }} deploy-stage: name: Deploy to staging environment environment: stage runs-on: ubuntu-latest - needs: build-stage + needs: build-multi-architecture steps: + - name: Set unique image tag + id: set-image-tag + run: | + echo "image_tag=nightly-$(date +%Y%m%d%H%m%S)" >> $GITHUB_ENV + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.STAGING_AWS_ID }} + aws-secret-access-key: ${{ secrets.STAGING_AWS_KEY }} + aws-region: eu-west-1 + + - name: Login to AWS ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + with: + mask-password: 'true' + + - name: Login to temporary registry + id: login-ghcr + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push image to ECR + uses: akhilerm/tag-push-action@v2.0.0 + with: + src: ${{ needs.build-multi-architecture.outputs.image }} + dst: | + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/file-server:nightly + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/file-server:${{ env.image_tag }} + + - name: Configure AWS credentials for EKS interaction uses: aws-actions/configure-aws-credentials@v3 with: aws-access-key-id: ${{ secrets.STAGING_AWS_ID }} @@ -74,7 +100,7 @@ jobs: - name: Update image and wait for deployment to finish timeout-minutes: 5 run: | - kubectl -n default set image deployment/flowforge-file file-storage=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/${{ needs.build-stage.outputs.image }} + kubectl -n default set image deployment/flowforge-file file-storage=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/file-server:${{ env.image_tag }} kubectl -n default rollout status deployment/flowforge-file - name: Rollback failed deployment @@ -88,14 +114,48 @@ jobs: name: Deploy to production environment environment: production runs-on: ubuntu-latest - needs: [build-prod, deploy-stage] + needs: [build-multi-architecture, deploy-stage] steps: + - name: Set unique image tag + id: set-image-tag + run: | + echo "image_tag=nightly-$(date +%Y%m%d%H%m%S)" >> $GITHUB_ENV + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v3 with: aws-access-key-id: ${{ secrets.PRODUCTION_AWS_ID }} aws-secret-access-key: ${{ secrets.PRODUCTION_AWS_KEY }} aws-region: eu-west-1 + + - name: Login to AWS ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + with: + mask-password: 'true' + + - name: Login to temporary registry + id: login-ghcr + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push image to ECR + uses: akhilerm/tag-push-action@v2.0.0 + with: + src: ${{ needs.build-multi-architecture.outputs.image }} + dst: | + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/file-server:nightly + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/file-server:${{ env.image_tag }} + + - name: Configure AWS credentials for EKS interaction + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.STAGING_AWS_ID }} + aws-secret-access-key: ${{ secrets.STAGING_AWS_KEY }} + aws-region: eu-west-1 role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/K8sAdmin role-duration-seconds: 1200 @@ -111,7 +171,7 @@ jobs: - name: Update image and wait for deployment to finish timeout-minutes: 5 run: | - kubectl -n default set image deployment/flowforge-file file-storage=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/${{ needs.build-prod.outputs.image }} + kubectl -n default set image deployment/flowforge-file file-storage=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/file-server:${{ env.image_tag }} kubectl -n default rollout status deployment/flowforge-file - name: Rollback failed deployment From 6b73966ef1bf7a227417337c87e6a91a7f1cc13e Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Wed, 20 Sep 2023 08:56:54 +0200 Subject: [PATCH 139/192] Add permissions to deploy jobs --- .github/workflows/fileserver-container.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 269f9555..88555b28 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -44,6 +44,9 @@ jobs: environment: stage runs-on: ubuntu-latest needs: build-multi-architecture + permissions: + packages: read + contents: read steps: - name: Set unique image tag id: set-image-tag @@ -115,6 +118,9 @@ jobs: environment: production runs-on: ubuntu-latest needs: [build-multi-architecture, deploy-stage] + permissions: + packages: read + contents: read steps: - name: Set unique image tag id: set-image-tag From 9cac9f5f9c214788a8d2dad8a505f49612077c92 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Wed, 20 Sep 2023 09:53:06 +0200 Subject: [PATCH 140/192] Temporary disable build on push --- .github/workflows/flowforge-container.yml | 6 +++--- .github/workflows/nodered-container.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index b59b23d5..58f0ee16 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -12,9 +12,9 @@ on: push: branches: - main - pull_request: - branches: - - main + # pull_request: + # branches: + # - main jobs: build-stage: diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index 50b47bea..54b68629 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -12,9 +12,9 @@ on: push: branches: - main - pull_request: - branches: - - main + # pull_request: + # branches: + # - main jobs: build-stage-302: From 3ef205e18c998ab392a26ce7984080c0839ebf73 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Wed, 20 Sep 2023 09:54:28 +0200 Subject: [PATCH 141/192] Bump actions version --- .github/workflows/fileserver-container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 88555b28..4fa0a65a 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -142,14 +142,14 @@ jobs: - name: Login to temporary registry id: login-ghcr - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Push image to ECR - uses: akhilerm/tag-push-action@v2.0.0 + uses: akhilerm/tag-push-action@v2.1.0 with: src: ${{ needs.build-multi-architecture.outputs.image }} dst: | From c63c4951763a7764849324615024b0baa98258cc Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Wed, 20 Sep 2023 10:00:45 +0200 Subject: [PATCH 142/192] Fix rollback step condition --- .github/workflows/fileserver-container.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 4fa0a65a..dfa4b909 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -101,13 +101,14 @@ jobs: aws eks update-kubeconfig --region eu-west-1 --name ${{ secrets.EKS_CLUSTER_NAME }} - name: Update image and wait for deployment to finish + id: update-image timeout-minutes: 5 run: | kubectl -n default set image deployment/flowforge-file file-storage=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/file-server:${{ env.image_tag }} kubectl -n default rollout status deployment/flowforge-file - name: Rollback failed deployment - if: failure() + if: ${{ failure() && steps.update-image.conclusion == 'failure'}} run: | kubectl -n default rollout undo deployment/flowforge-file kubectl -n default rollout status deployment/flowforge-file @@ -175,13 +176,14 @@ jobs: aws eks update-kubeconfig --region eu-west-1 --name ${{ secrets.EKS_CLUSTER_NAME }} - name: Update image and wait for deployment to finish + id: update-image timeout-minutes: 5 run: | kubectl -n default set image deployment/flowforge-file file-storage=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/file-server:${{ env.image_tag }} kubectl -n default rollout status deployment/flowforge-file - name: Rollback failed deployment - if: failure() + if: ${{ failure() && steps.update-image.conclusion == 'failure'}} run: | kubectl -n default rollout undo deployment/flowforge-file kubectl -n default rollout status deployment/flowforge-file From 1635fa70548bd2ee943096f37eecf7a8f5317cca Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Wed, 20 Sep 2023 10:09:50 +0200 Subject: [PATCH 143/192] Debug image pull --- .github/workflows/fileserver-container.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index dfa4b909..9621f6cb 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -149,6 +149,10 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: [DEBUG] Pull temporary image + run: + docker pull ${{ needs.build-multi-architecture.outputs.image }} + - name: Push image to ECR uses: akhilerm/tag-push-action@v2.1.0 with: From 452f2996b70d4dd07e819a36958462587c649283 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Wed, 20 Sep 2023 11:15:59 +0200 Subject: [PATCH 144/192] Fix step name --- .github/workflows/fileserver-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 9621f6cb..7f7b0af9 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -149,7 +149,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: [DEBUG] Pull temporary image + - name: "[DEBUG] Pull temporary image" run: docker pull ${{ needs.build-multi-architecture.outputs.image }} From 4bbb592a8127c4d931e0f20dcc8a1d8d4df59df7 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Wed, 20 Sep 2023 11:30:57 +0200 Subject: [PATCH 145/192] Debug image pull --- .github/workflows/fileserver-container.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 7f7b0af9..10c5bba8 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -68,14 +68,18 @@ jobs: - name: Login to temporary registry id: login-ghcr - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: "[DEBUG] Pull temporary image" + run: + docker pull ${{ needs.build-multi-architecture.outputs.image }} + - name: Push image to ECR - uses: akhilerm/tag-push-action@v2.0.0 + uses: akhilerm/tag-push-action@v2.1.0 with: src: ${{ needs.build-multi-architecture.outputs.image }} dst: | From 5311c2890f4ea580c63a0af41640e1221a6ea1b3 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Wed, 20 Sep 2023 13:26:18 +0200 Subject: [PATCH 146/192] Disable login to temporary registry --- .github/workflows/fileserver-container.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 10c5bba8..bf89d7d1 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -66,13 +66,13 @@ jobs: with: mask-password: 'true' - - name: Login to temporary registry - id: login-ghcr - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + # - name: Login to temporary registry + # id: login-ghcr + # uses: docker/login-action@v3 + # with: + # registry: ghcr.io + # username: ${{ github.actor }} + # password: ${{ secrets.GITHUB_TOKEN }} - name: "[DEBUG] Pull temporary image" run: From 166d3633c81b5e1014c0c0c08f36107e854554b6 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Wed, 20 Sep 2023 14:04:26 +0200 Subject: [PATCH 147/192] Debug image pull --- .github/workflows/fileserver-container.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index bf89d7d1..f3f5840c 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -66,16 +66,20 @@ jobs: with: mask-password: 'true' - # - name: Login to temporary registry - # id: login-ghcr - # uses: docker/login-action@v3 - # with: - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} + - name: "[DEBUG] logout from ghcr registry" + run: | + docker logout ghcr.io + + - name: Login to temporary registry + id: login-ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: "[DEBUG] Pull temporary image" - run: + run: | docker pull ${{ needs.build-multi-architecture.outputs.image }} - name: Push image to ECR From 1595638459cfb758fb58d83f88940353eb172132 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Thu, 21 Sep 2023 09:24:48 +0100 Subject: [PATCH 148/192] Only config file-server if needed flowforge.yml was including fileStore key even if file-server disabled --- helm/flowforge/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/flowforge/templates/configmap.yaml b/helm/flowforge/templates/configmap.yaml index a47d1894..36c013b3 100644 --- a/helm/flowforge/templates/configmap.yaml +++ b/helm/flowforge/templates/configmap.yaml @@ -155,7 +155,7 @@ data: hubspot: trackingcode: {{ .Values.forge.support.hubspot | int }} {{- end }} - {{- if .Values.forge.fileStore }} + {{- if .Values.forge.fileStore.enabled }} fileStore: url: http://flowforge-file.{{ .Release.Namespace }} {{- end }} From 27c8cdaa3750b64393707ff1d9e52c9537a77e0e Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 21 Sep 2023 13:58:14 +0200 Subject: [PATCH 149/192] Pull image in separate workflow --- .github/workflows/debug-image-pull.yaml | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/debug-image-pull.yaml diff --git a/.github/workflows/debug-image-pull.yaml b/.github/workflows/debug-image-pull.yaml new file mode 100644 index 00000000..0f15aa10 --- /dev/null +++ b/.github/workflows/debug-image-pull.yaml @@ -0,0 +1,29 @@ +name: Debug image pull from ghcr.io +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + pull: + name: Pull image + runs-on: ubuntu-latest + permissions: + packages: read + contents: read + steps: + - name: Login to temporary registry + id: login-ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: "Pull temporary image" + run: | + docker pull ${{ needs.build-multi-architecture.outputs.image }} From 6dac616a5e7146df17ab8bad7edd2cb868403bcc Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 21 Sep 2023 13:59:56 +0200 Subject: [PATCH 150/192] Pull image in separate workflow --- .github/workflows/debug-image-pull.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debug-image-pull.yaml b/.github/workflows/debug-image-pull.yaml index 0f15aa10..ad8ddaa0 100644 --- a/.github/workflows/debug-image-pull.yaml +++ b/.github/workflows/debug-image-pull.yaml @@ -26,4 +26,4 @@ jobs: - name: "Pull temporary image" run: | - docker pull ${{ needs.build-multi-architecture.outputs.image }} + docker pull ghcr.io/flowforge/file-server:main From 61d32f7ac8dc5e5f357779bbc48bcb6b9e4a2a90 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 21 Sep 2023 18:15:22 +0200 Subject: [PATCH 151/192] Verify image build fix --- .github/workflows/fileserver-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index f3f5840c..222ca507 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -19,7 +19,7 @@ on: jobs: build: name: Build single-architecture container images - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@9d655b572fffebb432b259566b68be1a7aff7ed5 + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@fix-image-manifest-unknown with: image_name: 'file-server' package_dependencies: | From 09ac90bef4d03336973ced0a581defe96fa78006 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 21 Sep 2023 18:34:46 +0200 Subject: [PATCH 152/192] Verify image build fix --- .github/workflows/fileserver-container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 222ca507..97349840 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -1,4 +1,5 @@ name: Build file-server container + on: workflow_dispatch: inputs: From 92c0f97abb671efc9f23b7eb6d210a1895dbeff8 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 21 Sep 2023 19:04:46 +0200 Subject: [PATCH 153/192] Remove debug-related stuff --- .github/workflows/debug-image-pull.yaml | 29 ---------------------- .github/workflows/fileserver-container.yml | 8 ------ 2 files changed, 37 deletions(-) delete mode 100644 .github/workflows/debug-image-pull.yaml diff --git a/.github/workflows/debug-image-pull.yaml b/.github/workflows/debug-image-pull.yaml deleted file mode 100644 index ad8ddaa0..00000000 --- a/.github/workflows/debug-image-pull.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: Debug image pull from ghcr.io -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - pull: - name: Pull image - runs-on: ubuntu-latest - permissions: - packages: read - contents: read - steps: - - name: Login to temporary registry - id: login-ghcr - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: "Pull temporary image" - run: | - docker pull ghcr.io/flowforge/file-server:main diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 97349840..1670c6dd 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -67,10 +67,6 @@ jobs: with: mask-password: 'true' - - name: "[DEBUG] logout from ghcr registry" - run: | - docker logout ghcr.io - - name: Login to temporary registry id: login-ghcr uses: docker/login-action@v3 @@ -79,10 +75,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: "[DEBUG] Pull temporary image" - run: | - docker pull ${{ needs.build-multi-architecture.outputs.image }} - - name: Push image to ECR uses: akhilerm/tag-push-action@v2.1.0 with: From 80781c735693d1dc3373e04ccdcf8aea0d7818de Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 22 Sep 2023 13:33:43 +0200 Subject: [PATCH 154/192] Deploy via reusable workflow --- .github/workflows/fileserver-container.yml | 78 +++------------------- 1 file changed, 9 insertions(+), 69 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 1670c6dd..23c398dc 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -42,77 +42,17 @@ jobs: deploy-stage: name: Deploy to staging environment - environment: stage - runs-on: ubuntu-latest needs: build-multi-architecture - permissions: - packages: read - contents: read - steps: - - name: Set unique image tag - id: set-image-tag - run: | - echo "image_tag=nightly-$(date +%Y%m%d%H%m%S)" >> $GITHUB_ENV - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.STAGING_AWS_ID }} - aws-secret-access-key: ${{ secrets.STAGING_AWS_KEY }} - aws-region: eu-west-1 - - - name: Login to AWS ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - with: - mask-password: 'true' - - - name: Login to temporary registry - id: login-ghcr - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push image to ECR - uses: akhilerm/tag-push-action@v2.1.0 - with: - src: ${{ needs.build-multi-architecture.outputs.image }} - dst: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/file-server:nightly - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/file-server:${{ env.image_tag }} - - - name: Configure AWS credentials for EKS interaction - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.STAGING_AWS_ID }} - aws-secret-access-key: ${{ secrets.STAGING_AWS_KEY }} - aws-region: eu-west-1 - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/K8sAdmin - role-duration-seconds: 1200 - - - name: Setup kubectl - uses: azure/setup-kubectl@v3 - with: - version: 'v1.23.4' - - - name: Configure kubectl - run: - aws eks update-kubeconfig --region eu-west-1 --name ${{ secrets.EKS_CLUSTER_NAME }} - - - name: Update image and wait for deployment to finish - id: update-image - timeout-minutes: 5 - run: | - kubectl -n default set image deployment/flowforge-file file-storage=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/file-server:${{ env.image_tag }} - kubectl -n default rollout status deployment/flowforge-file + uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@feat-eks-deployment-workflow + with: + environment: staging + image_name: ${{ needs.build-multi-architecture.outputs.image }} + secrets: + aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} + aws_secret_access_key: ${{ secrets.STAGING_AWS_KEY }} + temporary_registry_token: ${{ secrets.GITHUB_TOKEN }}} + eks_cluster_name: ${{ secrets.EKS_CLUSTER_NAME }}} - - name: Rollback failed deployment - if: ${{ failure() && steps.update-image.conclusion == 'failure'}} - run: | - kubectl -n default rollout undo deployment/flowforge-file - kubectl -n default rollout status deployment/flowforge-file deploy-prod: if: false From aed52b58310b5adc5949adf4ef524768a5e827fa Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 22 Sep 2023 13:36:39 +0200 Subject: [PATCH 155/192] Fix reusable build workflows references --- .github/workflows/fileserver-container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 23c398dc..abda355b 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -20,7 +20,7 @@ on: jobs: build: name: Build single-architecture container images - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@fix-image-manifest-unknown + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@37ba5163c6caad3537d6f7292dbd9102465ad810 with: image_name: 'file-server' package_dependencies: | @@ -34,7 +34,7 @@ jobs: build-multi-architecture: name: Build multi-architecture container image needs: build - uses: flowforge/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@9d655b572fffebb432b259566b68be1a7aff7ed5 + uses: flowforge/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@37ba5163c6caad3537d6f7292dbd9102465ad810 with: image_name: 'file-server' secrets: From a0c672818da3bdffb2e47560d58dc79cd520bf18 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 22 Sep 2023 13:41:18 +0200 Subject: [PATCH 156/192] Test deployment --- .github/workflows/fileserver-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index abda355b..ebe77aed 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -1,5 +1,4 @@ name: Build file-server container - on: workflow_dispatch: inputs: From 5c1b757e2647a1f56aa48d0a4da9df1818f0da6a Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 22 Sep 2023 13:42:59 +0200 Subject: [PATCH 157/192] Fix deploy workflow input name --- .github/workflows/fileserver-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index ebe77aed..c845d35e 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -45,7 +45,7 @@ jobs: uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@feat-eks-deployment-workflow with: environment: staging - image_name: ${{ needs.build-multi-architecture.outputs.image }} + image: ${{ needs.build-multi-architecture.outputs.image }} secrets: aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_secret_access_key: ${{ secrets.STAGING_AWS_KEY }} From 75f560fd1212c013b393ae3a8ae0dfe6c7048d20 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 22 Sep 2023 13:48:36 +0200 Subject: [PATCH 158/192] Typos fix --- .github/workflows/fileserver-container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index c845d35e..2dd2137b 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -49,8 +49,8 @@ jobs: secrets: aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_secret_access_key: ${{ secrets.STAGING_AWS_KEY }} - temporary_registry_token: ${{ secrets.GITHUB_TOKEN }}} - eks_cluster_name: ${{ secrets.EKS_CLUSTER_NAME }}} + temporary_registry_token: ${{ secrets.GITHUB_TOKEN }} + eks_cluster_name: ${{ secrets.EKS_CLUSTER_NAME }} deploy-prod: From a5541415bb7e843b152aa3925e6e211196f5763f Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 22 Sep 2023 14:18:01 +0200 Subject: [PATCH 159/192] Fix environment name --- .github/workflows/fileserver-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 2dd2137b..2894cd1d 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -44,7 +44,7 @@ jobs: needs: build-multi-architecture uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@feat-eks-deployment-workflow with: - environment: staging + environment: stage image: ${{ needs.build-multi-architecture.outputs.image }} secrets: aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} From 5acc26b4f7c0ac3d96d076b48df6f17fa854e886 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 22 Sep 2023 15:44:45 +0200 Subject: [PATCH 160/192] Adjust stage deployment parameters --- .github/workflows/fileserver-container.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 2894cd1d..ebdfb51f 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -45,6 +45,9 @@ jobs: uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@feat-eks-deployment-workflow with: environment: stage + service_name: 'file-server' + deployment_name: 'flowforge-file' + container_name: 'file-storage' image: ${{ needs.build-multi-architecture.outputs.image }} secrets: aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} From fff73c3e56c7f717709ca7e31240cab31aed965b Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 22 Sep 2023 16:14:23 +0200 Subject: [PATCH 161/192] Add production deployment parameters --- .github/workflows/fileserver-container.yml | 89 ++++------------------ 1 file changed, 13 insertions(+), 76 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index ebdfb51f..fa340246 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -57,81 +57,18 @@ jobs: deploy-prod: - if: false + if: github.ref_name == 'main' name: Deploy to production environment - environment: production - runs-on: ubuntu-latest needs: [build-multi-architecture, deploy-stage] - permissions: - packages: read - contents: read - steps: - - name: Set unique image tag - id: set-image-tag - run: | - echo "image_tag=nightly-$(date +%Y%m%d%H%m%S)" >> $GITHUB_ENV - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.PRODUCTION_AWS_ID }} - aws-secret-access-key: ${{ secrets.PRODUCTION_AWS_KEY }} - aws-region: eu-west-1 - - - name: Login to AWS ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - with: - mask-password: 'true' - - - name: Login to temporary registry - id: login-ghcr - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: "[DEBUG] Pull temporary image" - run: - docker pull ${{ needs.build-multi-architecture.outputs.image }} - - - name: Push image to ECR - uses: akhilerm/tag-push-action@v2.1.0 - with: - src: ${{ needs.build-multi-architecture.outputs.image }} - dst: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/file-server:nightly - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/file-server:${{ env.image_tag }} - - - name: Configure AWS credentials for EKS interaction - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.STAGING_AWS_ID }} - aws-secret-access-key: ${{ secrets.STAGING_AWS_KEY }} - aws-region: eu-west-1 - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/K8sAdmin - role-duration-seconds: 1200 - - - name: Setup kubectl - uses: azure/setup-kubectl@v3 - with: - version: 'v1.23.4' - - - name: Configure kubectl - run: - aws eks update-kubeconfig --region eu-west-1 --name ${{ secrets.EKS_CLUSTER_NAME }} - - - name: Update image and wait for deployment to finish - id: update-image - timeout-minutes: 5 - run: | - kubectl -n default set image deployment/flowforge-file file-storage=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/file-server:${{ env.image_tag }} - kubectl -n default rollout status deployment/flowforge-file - - - name: Rollback failed deployment - if: ${{ failure() && steps.update-image.conclusion == 'failure'}} - run: | - kubectl -n default rollout undo deployment/flowforge-file - kubectl -n default rollout status deployment/flowforge-file - \ No newline at end of file + uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@feat-eks-deployment-workflow + with: + environment: poduction + service_name: 'file-server' + deployment_name: 'flowforge-file' + container_name: 'file-storage' + image: ${{ needs.build-multi-architecture.outputs.image }} + secrets: + aws_access_key_id: ${{ secrets.PRODUCTION_AWS_ID }} + aws_secret_access_key: ${{ secrets.PRODUCTION_AWS_KEY }} + temporary_registry_token: ${{ secrets.GITHUB_TOKEN }} + eks_cluster_name: ${{ secrets.EKS_CLUSTER_NAME }} From 0d485ff0a9f9b0ac500ef08c6ecc7f03b24e324f Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 22 Sep 2023 16:42:52 +0200 Subject: [PATCH 162/192] Environment name typo fix --- .github/workflows/fileserver-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index fa340246..11b33f55 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -62,7 +62,7 @@ jobs: needs: [build-multi-architecture, deploy-stage] uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@feat-eks-deployment-workflow with: - environment: poduction + environment: production service_name: 'file-server' deployment_name: 'flowforge-file' container_name: 'file-storage' From dfb15f9c4ba1e7cecac7bcd3bc661fed6b86f837 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 22 Sep 2023 16:43:55 +0200 Subject: [PATCH 163/192] Apply new deployment approach to flowforge app --- .github/workflows/flowforge-container.yml | 132 +++++++--------------- 1 file changed, 41 insertions(+), 91 deletions(-) diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 58f0ee16..aa41fc43 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -12,16 +12,15 @@ on: push: branches: - main - # pull_request: - # branches: - # - main + pull_request: + branches: + - main jobs: - build-stage: - name: Build stage image - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 + build: + name: Build single-architecture container images + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@37ba5163c6caad3537d6f7292dbd9102465ad810 with: - environment: 'stage' image_name: 'forge-k8s' package_dependencies: | @flowforge/flowforge @@ -30,95 +29,46 @@ jobs: npm_registry_url: ${{ vars.PUBLIC_NPM_REGISTRY_URL }} secrets: npm_registry_auth_token: ${{ secrets.NPM_PUBLISH_TOKEN }} - aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} - aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} - - build-prod: - name: Build production image - if: false - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 + temporary_registry_token: ${{ secrets.GITHUB_TOKEN }} + + build-multi-architecture: + name: Build multi-architecture container image + needs: build + uses: flowforge/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@37ba5163c6caad3537d6f7292dbd9102465ad810 with: - environment: 'production' image_name: 'forge-k8s' - package_dependencies: | - @flowforge/flowforge - @flowforge/kubernetes - build_context: 'flowforge-container' - npm_registry_url: ${{ vars.PUBLIC_NPM_REGISTRY_URL }} secrets: - npm_registry_auth_token: ${{ secrets.NPM_PUBLISH_TOKEN }} - aws_access_key_id: ${{ secrets.PRODUCTION_AWS_ID }} - aws_access_key_secret: ${{ secrets.PRODUCTION_AWS_KEY }} + temporary_registry_token: ${{ secrets.GITHUB_TOKEN }} deploy-stage: name: Deploy to staging environment - environment: stage - runs-on: ubuntu-latest - needs: build-stage - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.STAGING_AWS_ID }} - aws-secret-access-key: ${{ secrets.STAGING_AWS_KEY }} - aws-region: eu-west-1 - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/K8sAdmin - role-duration-seconds: 1200 - - - name: Setup kubectl - uses: azure/setup-kubectl@v3 - with: - version: 'v1.23.4' - - - name: Configure kubectl - run: - aws eks update-kubeconfig --region eu-west-1 --name ${{ secrets.EKS_CLUSTER_NAME }} - - - name: Update image and wait for deployment to finish - timeout-minutes: 5 - run: | - kubectl -n default set image deployment/flowforge forge=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/${{ needs.build-stage.outputs.image }} - kubectl -n default rollout status deployment/flowforge - - - name: Rollback failed deployment - if: failure() - run: | - kubectl -n default rollout undo deployment/flowforge - kubectl -n default rollout status deployment/flowforge - + needs: build-multi-architecture + uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@feat-eks-deployment-workflow + with: + environment: stage + service_name: 'forge-k8s' + deployment_name: flowforge + container_name: forge + image: ${{ needs.build-multi-architecture.outputs.image }} + secrets: + aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} + aws_secret_access_key: ${{ secrets.STAGING_AWS_KEY }} + temporary_registry_token: ${{ secrets.GITHUB_TOKEN }} + eks_cluster_name: ${{ secrets.EKS_CLUSTER_NAME }} + deploy-prod: - if: false + if: github.ref_name == 'main' name: Deploy to production environment - environment: production - runs-on: ubuntu-latest - needs: [build-prod, deploy-stage] - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.PRODUCTION_AWS_ID }} - aws-secret-access-key: ${{ secrets.PRODUCTION_AWS_KEY }} - aws-region: eu-west-1 - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/K8sAdmin - role-duration-seconds: 1200 - - - name: Setup kubectl - uses: azure/setup-kubectl@v3 - with: - version: 'v1.23.4' - - - name: Configure kubectl - run: - aws eks update-kubeconfig --region eu-west-1 --name ${{ secrets.EKS_CLUSTER_NAME }} - - - name: Update image and wait for deployment to finish - timeout-minutes: 5 - run: | - kubectl -n default set image deployment/flowforge forge=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/flowforge/${{ needs.build-prod.outputs.image }} - kubectl -n default rollout status deployment/flowforge - - - name: Rollback failed deployment - if: failure() - run: | - kubectl -n default rollout undo deployment/flowforge - kubectl -n default rollout status deployment/flowforge + needs: [build-multi-architecture, deploy-stage] + uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@feat-eks-deployment-workflow + with: + environment: production + service_name: 'forge-k8s' + deployment_name: flowforge + container_name: forge + image: ${{ needs.build-multi-architecture.outputs.image }} + secrets: + aws_access_key_id: ${{ secrets.PRODUCTION_AWS_ID }} + aws_secret_access_key: ${{ secrets.PRODUCTION_AWS_KEY }} + temporary_registry_token: ${{ secrets.GITHUB_TOKEN }} + eks_cluster_name: ${{ secrets.EKS_CLUSTER_NAME }} From 320f636ed526f0dc5204e909274f6beaf8f9e3e3 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 22 Sep 2023 16:51:39 +0200 Subject: [PATCH 164/192] Rename workflows --- .github/workflows/fileserver-container.yml | 2 +- .github/workflows/flowforge-container.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 11b33f55..1ff8a430 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -1,4 +1,4 @@ -name: Build file-server container +name: File-server - build and deploy on: workflow_dispatch: inputs: diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index aa41fc43..26952409 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -1,4 +1,4 @@ -name: Build Flowforge container +name: Flowforge - build and deploy on: workflow_dispatch: inputs: From fd82e67ac444eb2bd5fa4e5be40861aebab85b31 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 25 Sep 2023 07:26:48 +0200 Subject: [PATCH 165/192] Update reusable workflow repo reference --- .github/workflows/fileserver-container.yml | 9 ++++----- .github/workflows/flowforge-container.yml | 8 ++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 1ff8a430..2889437f 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -19,7 +19,7 @@ on: jobs: build: name: Build single-architecture container images - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@37ba5163c6caad3537d6f7292dbd9102465ad810 + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: image_name: 'file-server' package_dependencies: | @@ -33,7 +33,7 @@ jobs: build-multi-architecture: name: Build multi-architecture container image needs: build - uses: flowforge/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@37ba5163c6caad3537d6f7292dbd9102465ad810 + uses: flowforge/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: image_name: 'file-server' secrets: @@ -42,7 +42,7 @@ jobs: deploy-stage: name: Deploy to staging environment needs: build-multi-architecture - uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@feat-eks-deployment-workflow + uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: environment: stage service_name: 'file-server' @@ -55,12 +55,11 @@ jobs: temporary_registry_token: ${{ secrets.GITHUB_TOKEN }} eks_cluster_name: ${{ secrets.EKS_CLUSTER_NAME }} - deploy-prod: if: github.ref_name == 'main' name: Deploy to production environment needs: [build-multi-architecture, deploy-stage] - uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@feat-eks-deployment-workflow + uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: environment: production service_name: 'file-server' diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 26952409..f135c343 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -19,7 +19,7 @@ on: jobs: build: name: Build single-architecture container images - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@37ba5163c6caad3537d6f7292dbd9102465ad810 + uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: image_name: 'forge-k8s' package_dependencies: | @@ -34,7 +34,7 @@ jobs: build-multi-architecture: name: Build multi-architecture container image needs: build - uses: flowforge/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@37ba5163c6caad3537d6f7292dbd9102465ad810 + uses: flowforge/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: image_name: 'forge-k8s' secrets: @@ -43,7 +43,7 @@ jobs: deploy-stage: name: Deploy to staging environment needs: build-multi-architecture - uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@feat-eks-deployment-workflow + uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: environment: stage service_name: 'forge-k8s' @@ -60,7 +60,7 @@ jobs: if: github.ref_name == 'main' name: Deploy to production environment needs: [build-multi-architecture, deploy-stage] - uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@feat-eks-deployment-workflow + uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: environment: production service_name: 'forge-k8s' From 913d7d284700838fac4c6e66ab1e69c7278809ec Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Mon, 25 Sep 2023 07:38:54 +0200 Subject: [PATCH 166/192] Rollback temporary change --- .github/workflows/nodered-container.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index 54b68629..50b47bea 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -12,9 +12,9 @@ on: push: branches: - main - # pull_request: - # branches: - # - main + pull_request: + branches: + - main jobs: build-stage-302: From 1994ee820daa69bdf620db1d20573aa8cb13990e Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Tue, 26 Sep 2023 10:18:04 +0200 Subject: [PATCH 167/192] Enable concurrency in deployment pipelines --- .github/workflows/fileserver-container.yml | 4 ++++ .github/workflows/flowforge-container.yml | 4 ++++ .github/workflows/nodered-container.yml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 2889437f..db2c7e8b 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -16,6 +16,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + jobs: build: name: Build single-architecture container images diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index f135c343..2ef94f95 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -16,6 +16,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + jobs: build: name: Build single-architecture container images diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index 50b47bea..f418b06f 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -16,6 +16,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + jobs: build-stage-302: uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 From 20ee25e64bc484f2543d07c7b987a41406adcab6 Mon Sep 17 00:00:00 2001 From: Steve-Mcl Date: Thu, 28 Sep 2023 13:26:02 +0100 Subject: [PATCH 168/192] Update CHANGELOG for 1.12.0 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d2e50bd..8d455461 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +#### 1.12.0: Release + + - Enable concurrency in deployment pipelines (#182) @ppawlowski + - Deploy multiarch image from temporary registry (#179) @ppawlowski + - Only config file-server if needed (#180) @hardillb + - Build container images using packages from public npm registry (#177) @ppawlowski + - Pin reusable workflow to commit SHA (#175) @ppawlowski + #### 1.11.0: Release - Disable production containers build (#167) @ppawlowski From ef77a92b653aab847b5d23b8a455be57916ed5cd Mon Sep 17 00:00:00 2001 From: Steve-Mcl Date: Thu, 28 Sep 2023 13:26:02 +0100 Subject: [PATCH 169/192] Update version to 1.12.0 --- VERSION | 2 +- file-server/package.json | 4 ++-- flowforge-container/package.json | 6 +++--- helm/flowforge/Chart.yaml | 4 ++-- node-red-container/package.json | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/VERSION b/VERSION index 1cac385c..0eed1a29 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.11.0 +1.12.0 diff --git a/file-server/package.json b/file-server/package.json index f42e2197..054a9701 100644 --- a/file-server/package.json +++ b/file-server/package.json @@ -1,9 +1,9 @@ { "name": "@flowforge/file-server-container", - "version": "1.11.0", + "version": "1.12.0", "private": true, "dependencies": { - "@flowforge/file-server": "^1.11.0" + "@flowforge/file-server": "^1.12.0" }, "license": "Apache-2.0" } diff --git a/flowforge-container/package.json b/flowforge-container/package.json index c694dc9f..29ec92cb 100644 --- a/flowforge-container/package.json +++ b/flowforge-container/package.json @@ -1,14 +1,14 @@ { "name": "@flowforge/flowforge-k8s", - "version": "1.11.0", + "version": "1.12.0", "description": "FlowForge in Docker", "private": true, "author": { "name": "FlowForge Inc." }, "dependencies": { - "@flowforge/flowforge": "^1.11.0", - "@flowforge/kubernetes": "^1.11.0", + "@flowforge/flowforge": "^1.12.0", + "@flowforge/kubernetes": "^1.12.0", "pg": "^8.7.1", "pg-hstore": "^2.3.4" }, diff --git a/helm/flowforge/Chart.yaml b/helm/flowforge/Chart.yaml index c041d084..4916adfe 100644 --- a/helm/flowforge/Chart.yaml +++ b/helm/flowforge/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "flowforge" -version: "1.11.0" +version: "1.12.0" description: "FlowForge" type: "application" home: "https://flowforge.com" @@ -16,4 +16,4 @@ dependencies: maintainers: - name: "FlowForge Inc" url: "https://flowforge.com" -appVersion: "1.11.0" +appVersion: "1.12.0" diff --git a/node-red-container/package.json b/node-red-container/package.json index 90c369ce..e4ae24a2 100644 --- a/node-red-container/package.json +++ b/node-red-container/package.json @@ -1,7 +1,7 @@ { "name": "node-red-project", "description": "A Node-RED Project", - "version": "1.11.0", + "version": "1.12.0", "private": true, "dependencies": {} } From 53f115327a03b24a3764281ebb724b460ab89362 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 28 Sep 2023 17:49:20 +0200 Subject: [PATCH 170/192] Reusabne workflow reference name change --- .github/workflows/build-containers.yml | 20 ++++++++++---------- .github/workflows/fileserver-container.yml | 8 ++++---- .github/workflows/flowforge-container.yml | 8 ++++---- .github/workflows/nodered-container.yml | 6 +++--- .github/workflows/project-automation.yml | 2 +- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index a086686a..dd774404 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -9,7 +9,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - repository: 'flowforge/helm' + repository: 'flowfuse/helm' path: 'helm' # sets options for Docker build # will tag contianers with GH tag name @@ -22,7 +22,7 @@ jobs: flavor: | latest=true images: | - flowforge/forge-k8s + flowfuse/forge-k8s # sets up multi platform emulators - name: Setup QEMU uses: docker/setup-qemu-action@v1 @@ -50,14 +50,14 @@ jobs: repository: flowforge/forge-k8s username: flowforge password: ${{ secrets.DOCKER_HUB_PASSWORD }} - readme-filepath: ./helm/flowforge-container/README.md + readme-filepath: ./helm/flowfuse-container/README.md build_nodered_container: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: - repository: 'flowforge/helm' + repository: 'flowfuse/helm' path: 'helm' - name: Docker Meta Data id: meta @@ -99,7 +99,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - repository: 'flowforge/helm' + repository: 'flowfuse/helm' path: 'helm' - name: Docker Meta Data id: meta @@ -134,7 +134,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - repository: 'flowforge/helm' + repository: 'flowfuse/helm' path: 'helm' - name: Docker Meta Data id: meta @@ -169,7 +169,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - repository: 'flowforge/helm' + repository: 'flowfuse/helm' path: 'helm' - name: Docker Meta Data id: meta @@ -220,7 +220,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 with: - repository: 'flowforge/helm' + repository: 'flowfuse/helm' fetch-depth: 0 path: 'helm' - name: Package @@ -229,9 +229,9 @@ jobs: cd helm git checkout gh-pages mv ../flowforge-*.tgz . - helm repo index --url https://flowforge.github.io/helm . + helm repo index --url https://flowfuse.github.io/helm . git add . - git -c user.name='FlowForge Build Bot' -c user.email='noreply@flowforge.com' commit -m "Add ${{ github.ref_name }} helm chart" + git -c user.name='FlowFuse Build Bot' -c user.email='noreply@flowfuse.com' commit -m "Add ${{ github.ref_name }} helm chart" git push origin - name: Resync Maintenance if: ${{ endsWith(github.ref, '.0') }} diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index db2c7e8b..aca5a4ac 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -23,7 +23,7 @@ concurrency: jobs: build: name: Build single-architecture container images - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/build_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: image_name: 'file-server' package_dependencies: | @@ -37,7 +37,7 @@ jobs: build-multi-architecture: name: Build multi-architecture container image needs: build - uses: flowforge/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: image_name: 'file-server' secrets: @@ -46,7 +46,7 @@ jobs: deploy-stage: name: Deploy to staging environment needs: build-multi-architecture - uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: environment: stage service_name: 'file-server' @@ -63,7 +63,7 @@ jobs: if: github.ref_name == 'main' name: Deploy to production environment needs: [build-multi-architecture, deploy-stage] - uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: environment: production service_name: 'file-server' diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 2ef94f95..04082ece 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -23,7 +23,7 @@ concurrency: jobs: build: name: Build single-architecture container images - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/build_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: image_name: 'forge-k8s' package_dependencies: | @@ -38,7 +38,7 @@ jobs: build-multi-architecture: name: Build multi-architecture container image needs: build - uses: flowforge/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: image_name: 'forge-k8s' secrets: @@ -47,7 +47,7 @@ jobs: deploy-stage: name: Deploy to staging environment needs: build-multi-architecture - uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: environment: stage service_name: 'forge-k8s' @@ -64,7 +64,7 @@ jobs: if: github.ref_name == 'main' name: Deploy to production environment needs: [build-multi-architecture, deploy-stage] - uses: flowforge/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 with: environment: production service_name: 'forge-k8s' diff --git a/.github/workflows/nodered-container.yml b/.github/workflows/nodered-container.yml index f418b06f..70c5cb9d 100644 --- a/.github/workflows/nodered-container.yml +++ b/.github/workflows/nodered-container.yml @@ -22,7 +22,7 @@ concurrency: jobs: build-stage-302: - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 + uses: flowfuse/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 with: environment: 'stage' image_name: 'node-red' @@ -36,7 +36,7 @@ jobs: aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} build-stage-223: - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 + uses: flowfuse/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 with: environment: 'stage' image_name: 'node-red' @@ -51,7 +51,7 @@ jobs: aws_access_key_id: ${{ secrets.STAGING_AWS_ID }} aws_access_key_secret: ${{ secrets.STAGING_AWS_KEY }} build-stage-310: - uses: flowforge/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 + uses: flowfuse/github-actions-workflows/.github/workflows/build_container_image.yml@e3e734b910af78371b2c9a1c6856446d17421f50 with: environment: 'stage' image_name: 'node-red' diff --git a/.github/workflows/project-automation.yml b/.github/workflows/project-automation.yml index ab8ccf7e..4d09e567 100644 --- a/.github/workflows/project-automation.yml +++ b/.github/workflows/project-automation.yml @@ -5,6 +5,6 @@ on: - opened jobs: add_to_product_board: - uses: flowforge/.github/.github/workflows/project-automation.yml@main + uses: flowfuse/.github/.github/workflows/project-automation.yml@main secrets: token: ${{ secrets.PROJECT_ACCESS_TOKEN }} From b7f1594ef028a8e232e0d675978c99a3541630f6 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 28 Sep 2023 18:07:07 +0200 Subject: [PATCH 171/192] Test lowercase fix --- .github/workflows/fileserver-container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index aca5a4ac..60d1e541 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -23,7 +23,7 @@ concurrency: jobs: build: name: Build single-architecture container images - uses: flowfuse/github-actions-workflows/.github/workflows/build_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/build_container_image.yml@fix-lower-repo-owner with: image_name: 'file-server' package_dependencies: | @@ -37,7 +37,7 @@ jobs: build-multi-architecture: name: Build multi-architecture container image needs: build - uses: flowfuse/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@fix-lower-repo-owner with: image_name: 'file-server' secrets: From 5d8dcf99e54c62f547d30572a90a8a4f57048ea2 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 28 Sep 2023 18:31:17 +0200 Subject: [PATCH 172/192] Bump container build reusable workflow version --- .github/workflows/fileserver-container.yml | 8 ++++---- .github/workflows/flowforge-container.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/fileserver-container.yml b/.github/workflows/fileserver-container.yml index 60d1e541..1dbfea1b 100644 --- a/.github/workflows/fileserver-container.yml +++ b/.github/workflows/fileserver-container.yml @@ -23,7 +23,7 @@ concurrency: jobs: build: name: Build single-architecture container images - uses: flowfuse/github-actions-workflows/.github/workflows/build_container_image.yml@fix-lower-repo-owner + uses: flowfuse/github-actions-workflows/.github/workflows/build_container_image.yml@7808b849474ae9bc8dfc7f6db19e66d241018c94 with: image_name: 'file-server' package_dependencies: | @@ -37,7 +37,7 @@ jobs: build-multi-architecture: name: Build multi-architecture container image needs: build - uses: flowfuse/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@fix-lower-repo-owner + uses: flowfuse/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@7808b849474ae9bc8dfc7f6db19e66d241018c94 with: image_name: 'file-server' secrets: @@ -46,7 +46,7 @@ jobs: deploy-stage: name: Deploy to staging environment needs: build-multi-architecture - uses: flowfuse/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/deploy_container_image.yml@7808b849474ae9bc8dfc7f6db19e66d241018c94 with: environment: stage service_name: 'file-server' @@ -63,7 +63,7 @@ jobs: if: github.ref_name == 'main' name: Deploy to production environment needs: [build-multi-architecture, deploy-stage] - uses: flowfuse/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/deploy_container_image.yml@7808b849474ae9bc8dfc7f6db19e66d241018c94 with: environment: production service_name: 'file-server' diff --git a/.github/workflows/flowforge-container.yml b/.github/workflows/flowforge-container.yml index 04082ece..62a48dbd 100644 --- a/.github/workflows/flowforge-container.yml +++ b/.github/workflows/flowforge-container.yml @@ -23,7 +23,7 @@ concurrency: jobs: build: name: Build single-architecture container images - uses: flowfuse/github-actions-workflows/.github/workflows/build_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/build_container_image.yml@7808b849474ae9bc8dfc7f6db19e66d241018c94 with: image_name: 'forge-k8s' package_dependencies: | @@ -38,7 +38,7 @@ jobs: build-multi-architecture: name: Build multi-architecture container image needs: build - uses: flowfuse/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/merge_multiarch_images.yml@7808b849474ae9bc8dfc7f6db19e66d241018c94 with: image_name: 'forge-k8s' secrets: @@ -47,7 +47,7 @@ jobs: deploy-stage: name: Deploy to staging environment needs: build-multi-architecture - uses: flowfuse/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/deploy_container_image.yml@7808b849474ae9bc8dfc7f6db19e66d241018c94 with: environment: stage service_name: 'forge-k8s' @@ -64,7 +64,7 @@ jobs: if: github.ref_name == 'main' name: Deploy to production environment needs: [build-multi-architecture, deploy-stage] - uses: flowfuse/github-actions-workflows/.github/workflows/deploy_container_image.yml@cc8efbe5e4897e997d3b170a970a1afca9410273 + uses: flowfuse/github-actions-workflows/.github/workflows/deploy_container_image.yml@7808b849474ae9bc8dfc7f6db19e66d241018c94 with: environment: production service_name: 'forge-k8s' From 18d0cf866332d69e7b1a15375c8e05a2d49419e7 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 28 Sep 2023 18:32:43 +0200 Subject: [PATCH 173/192] Fix image registry name --- .github/workflows/build-containers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index dd774404..771fb9e8 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -22,7 +22,7 @@ jobs: flavor: | latest=true images: | - flowfuse/forge-k8s + flowforge/forge-k8s # sets up multi platform emulators - name: Setup QEMU uses: docker/setup-qemu-action@v1 From 5f7c4d05eb43c2f0df530ff31c1beaf0ebbb9129 Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Thu, 28 Sep 2023 18:34:40 +0200 Subject: [PATCH 174/192] Fix directory rename --- .github/workflows/build-containers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 771fb9e8..29708507 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -50,7 +50,7 @@ jobs: repository: flowforge/forge-k8s username: flowforge password: ${{ secrets.DOCKER_HUB_PASSWORD }} - readme-filepath: ./helm/flowfuse-container/README.md + readme-filepath: ./helm/flowforge-container/README.md build_nodered_container: runs-on: ubuntu-latest From 5337de866b482a20b4854a037548e8ad10c08d96 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Thu, 28 Sep 2023 18:36:21 +0100 Subject: [PATCH 175/192] Bump to v1.12.1 --- CHANGELOG.md | 4 ++++ VERSION | 2 +- flowforge-container/package.json | 4 ++-- helm/flowforge/Chart.yaml | 4 ++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d455461..984d9ea8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +#### 1.12.1: Release + + - Bump to FlowForge v1.12.1 @hardillb + #### 1.12.0: Release - Enable concurrency in deployment pipelines (#182) @ppawlowski diff --git a/VERSION b/VERSION index 0eed1a29..f8f4f03b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.12.0 +1.12.1 diff --git a/flowforge-container/package.json b/flowforge-container/package.json index 29ec92cb..58ebfc2d 100644 --- a/flowforge-container/package.json +++ b/flowforge-container/package.json @@ -1,13 +1,13 @@ { "name": "@flowforge/flowforge-k8s", - "version": "1.12.0", + "version": "1.12.1", "description": "FlowForge in Docker", "private": true, "author": { "name": "FlowForge Inc." }, "dependencies": { - "@flowforge/flowforge": "^1.12.0", + "@flowforge/flowforge": "^1.12.1", "@flowforge/kubernetes": "^1.12.0", "pg": "^8.7.1", "pg-hstore": "^2.3.4" diff --git a/helm/flowforge/Chart.yaml b/helm/flowforge/Chart.yaml index 4916adfe..912a03f3 100644 --- a/helm/flowforge/Chart.yaml +++ b/helm/flowforge/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "flowforge" -version: "1.12.0" +version: "1.12.1" description: "FlowForge" type: "application" home: "https://flowforge.com" @@ -16,4 +16,4 @@ dependencies: maintainers: - name: "FlowForge Inc" url: "https://flowforge.com" -appVersion: "1.12.0" +appVersion: "1.12.1" From b2c8103a9d17b38f6c6f6665e26df24c2bd33a9f Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Fri, 29 Sep 2023 09:22:48 +0100 Subject: [PATCH 176/192] Bump versions for 1.12.2 --- CHANGELOG.md | 4 ++++ VERSION | 2 +- flowforge-container/package.json | 4 ++-- helm/flowforge/Chart.yaml | 4 ++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 984d9ea8..b80f262d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +#### 1.12.2: Release + + - Bump to FlowForge v1.12.2 @hardillb + #### 1.12.1: Release - Bump to FlowForge v1.12.1 @hardillb diff --git a/VERSION b/VERSION index f8f4f03b..6b89d58f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.12.1 +1.12.2 diff --git a/flowforge-container/package.json b/flowforge-container/package.json index 58ebfc2d..fdd04e44 100644 --- a/flowforge-container/package.json +++ b/flowforge-container/package.json @@ -1,13 +1,13 @@ { "name": "@flowforge/flowforge-k8s", - "version": "1.12.1", + "version": "1.12.2", "description": "FlowForge in Docker", "private": true, "author": { "name": "FlowForge Inc." }, "dependencies": { - "@flowforge/flowforge": "^1.12.1", + "@flowforge/flowforge": "^1.12.2", "@flowforge/kubernetes": "^1.12.0", "pg": "^8.7.1", "pg-hstore": "^2.3.4" diff --git a/helm/flowforge/Chart.yaml b/helm/flowforge/Chart.yaml index 912a03f3..0130820a 100644 --- a/helm/flowforge/Chart.yaml +++ b/helm/flowforge/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "flowforge" -version: "1.12.1" +version: "1.12.2" description: "FlowForge" type: "application" home: "https://flowforge.com" @@ -16,4 +16,4 @@ dependencies: maintainers: - name: "FlowForge Inc" url: "https://flowforge.com" -appVersion: "1.12.1" +appVersion: "1.12.2" From 76e4b202c2fab80799d46caa4306e6d61b7ff6cb Mon Sep 17 00:00:00 2001 From: Pez Cuckow Date: Tue, 3 Oct 2023 12:33:41 +0200 Subject: [PATCH 177/192] Add support for configuring sentry telemetry --- helm/flowforge/README.md | 3 +++ helm/flowforge/templates/configmap.yaml | 16 +++++++++++++++- helm/flowforge/values.schema.json | 14 ++++++++++++++ test/customizations.yml | 4 ++++ test/unit/configmap_spec.js | 18 ++++++++++++++++++ 5 files changed, 54 insertions(+), 1 deletion(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 5caf1afd..756ee6e0 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -81,6 +81,9 @@ Enables FlowForge Telemetry - `forge.telemetry.posthog.apikey` enables posthog logging if set (not default) - `forge.telemetry.posthog.apiurl` sets posthog target host (default `https://app.posthog.com`) - `forge.telemetry.posthog.capture_pageview` (default `true`) + - `forge.telemetry.sentry.frontend_dsn` enables sentry reporting if set + - `forge.telemetry.sentry.backend_dsn` enables sentry reporting if set + - `forge.telemetry.sentry.production_mode` late limit reporting (default `true`) ### Support diff --git a/helm/flowforge/templates/configmap.yaml b/helm/flowforge/templates/configmap.yaml index 36c013b3..c813f3e0 100644 --- a/helm/flowforge/templates/configmap.yaml +++ b/helm/flowforge/templates/configmap.yaml @@ -129,7 +129,7 @@ data: http: info telemetry: enabled: {{ .Values.forge.telemetry.enabled }} - {{ if or (.Values.forge.telemetry.plausible) (.Values.forge.telemetry.posthog) }} + {{ if or (.Values.forge.telemetry.plausible) (.Values.forge.telemetry.posthog) (hasKey .Values.forge.telemetry.sentry "frontend_dsn") }} frontend: {{ if .Values.forge.telemetry.plausible -}} plausible: @@ -147,7 +147,21 @@ data: capture_pageview: true {{ end }} {{ end -}} + {{ if and (hasKey .Values.forge.telemetry "sentry") (hasKey .Values.forge.telemetry.sentry "frontend_dsn") -}} + sentry: + dsn: {{ .Values.forge.telemetry.sentry.frontend_dsn}} + {{ if hasKey .Values.forge.telemetry.dsn "production_mode" }} + production_mode: {{ .Values.forge.telemetry.entry.production_mode }} + {{ else }} + production_mode: true + {{ end }} + {{ end -}} {{- end }} + {{ if and (hasKey .Values.forge.telemetry "sentry") (hasKey .Values.forge.telemetry.sentry "backend_dsn") -}} + backend: + sentry: + dsn: {{ .Values.forge.telemetry.sentry.backend_dsn}} + {{ end -}} {{- if .Values.forge.support.enabled }} support: enabled: true diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index fba76c46..d14b15c0 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -242,6 +242,20 @@ "required": [ "apikey" ] + }, + "sentry": { + "type": "object", + "properties": { + "frontend_dsn":{ + "type": "string" + }, + "backend_dsn": { + "type": "string" + }, + "production_mode": { + "type": "boolean" + } + }, } } }, diff --git a/test/customizations.yml b/test/customizations.yml index ba68a3d7..6f75dc45 100644 --- a/test/customizations.yml +++ b/test/customizations.yml @@ -43,6 +43,10 @@ forge: posthog: capture_pageview: false apikey: phc_fdlksajfdfadfsafsaf + sentry: + production_mode: false + frontend_dsn: https://sentry.io/flowforge/flowforge-frontend + backend_dsn: https://sentry.io/flowforge/flowforge-backend support: enabled: true hubspot: 12345678 diff --git a/test/unit/configmap_spec.js b/test/unit/configmap_spec.js index 88771d0d..9454ae2f 100644 --- a/test/unit/configmap_spec.js +++ b/test/unit/configmap_spec.js @@ -103,4 +103,22 @@ describe('Examine Config Maps', function () { }) }) }) + + describe('customizations.yml', async function () { + let yml + beforeEach(function () { + const d = configMaps.filter(doc => doc.metadata.name === 'flowforge-config')[0] + yml = yaml.parse(d.data['customizations.ym']) + }) + it('has sentry telemetry', function () { + yml.telemetry.sentry.should.have.property('production_mode') + yml.telemetry.sentry.production_mode.should.equal(false) + + yml.telemetry.sentry.should.have.property('frontend_dsn') + yml.telemetry.sentry.frontend_dsn.should.equal('https://sentry.io/flowforge/flowforge-frontend') + + yml.telemetry.sentry.should.have.property('backend_dsn') + yml.telemetry.sentry.frontend_dsn.should.equal('https://sentry.io/flowforge/flowforge-backend') + }) + }) }) From 26a6b91b4552af657c6ef1ee2c59820ab8cee280 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 3 Oct 2023 12:54:48 +0100 Subject: [PATCH 178/192] Fix tests --- helm/flowforge/templates/configmap.yaml | 4 ++-- helm/flowforge/values.schema.json | 2 +- test/customizations.yml | 4 ++-- test/unit/configmap_spec.js | 15 ++++++++------- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/helm/flowforge/templates/configmap.yaml b/helm/flowforge/templates/configmap.yaml index c813f3e0..0cdab502 100644 --- a/helm/flowforge/templates/configmap.yaml +++ b/helm/flowforge/templates/configmap.yaml @@ -150,8 +150,8 @@ data: {{ if and (hasKey .Values.forge.telemetry "sentry") (hasKey .Values.forge.telemetry.sentry "frontend_dsn") -}} sentry: dsn: {{ .Values.forge.telemetry.sentry.frontend_dsn}} - {{ if hasKey .Values.forge.telemetry.dsn "production_mode" }} - production_mode: {{ .Values.forge.telemetry.entry.production_mode }} + {{ if hasKey .Values.forge.telemetry.sentry "production_mode" }} + production_mode: {{ .Values.forge.telemetry.sentry.production_mode }} {{ else }} production_mode: true {{ end }} diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index d14b15c0..4c34cdc7 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -255,7 +255,7 @@ "production_mode": { "type": "boolean" } - }, + } } } }, diff --git a/test/customizations.yml b/test/customizations.yml index 6f75dc45..2fd1259b 100644 --- a/test/customizations.yml +++ b/test/customizations.yml @@ -45,8 +45,8 @@ forge: apikey: phc_fdlksajfdfadfsafsaf sentry: production_mode: false - frontend_dsn: https://sentry.io/flowforge/flowforge-frontend - backend_dsn: https://sentry.io/flowforge/flowforge-backend + frontend_dsn: 'https://sentry.io/flowforge/flowforge-frontend' + backend_dsn: 'https://sentry.io/flowforge/flowforge-backend' support: enabled: true hubspot: 12345678 diff --git a/test/unit/configmap_spec.js b/test/unit/configmap_spec.js index 9454ae2f..decddf8b 100644 --- a/test/unit/configmap_spec.js +++ b/test/unit/configmap_spec.js @@ -108,17 +108,18 @@ describe('Examine Config Maps', function () { let yml beforeEach(function () { const d = configMaps.filter(doc => doc.metadata.name === 'flowforge-config')[0] - yml = yaml.parse(d.data['customizations.ym']) + yml = yaml.parse(d.data['flowforge.yml']) }) it('has sentry telemetry', function () { - yml.telemetry.sentry.should.have.property('production_mode') - yml.telemetry.sentry.production_mode.should.equal(false) + console.log(yml.telemetry) + yml.telemetry.frontend.sentry.should.have.property('production_mode') + yml.telemetry.frontend.sentry.production_mode.should.equal(false) - yml.telemetry.sentry.should.have.property('frontend_dsn') - yml.telemetry.sentry.frontend_dsn.should.equal('https://sentry.io/flowforge/flowforge-frontend') + yml.telemetry.frontend.sentry.should.have.property('dsn') + yml.telemetry.frontend.sentry.dsn.should.equal('https://sentry.io/flowforge/flowforge-frontend') - yml.telemetry.sentry.should.have.property('backend_dsn') - yml.telemetry.sentry.frontend_dsn.should.equal('https://sentry.io/flowforge/flowforge-backend') + yml.telemetry.backend.sentry.should.have.property('dsn') + yml.telemetry.backend.sentry.dsn.should.equal('https://sentry.io/flowforge/flowforge-backend') }) }) }) From 92b5f8377919e7c1ab7f5538cda8791e122de4f6 Mon Sep 17 00:00:00 2001 From: Pez Cuckow Date: Tue, 3 Oct 2023 14:23:29 +0200 Subject: [PATCH 179/192] Update helm/flowforge/README.md Co-authored-by: Ben Hardill --- helm/flowforge/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 756ee6e0..86fbac02 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -83,7 +83,7 @@ Enables FlowForge Telemetry - `forge.telemetry.posthog.capture_pageview` (default `true`) - `forge.telemetry.sentry.frontend_dsn` enables sentry reporting if set - `forge.telemetry.sentry.backend_dsn` enables sentry reporting if set - - `forge.telemetry.sentry.production_mode` late limit reporting (default `true`) + - `forge.telemetry.sentry.production_mode` rate limit reporting (default `true`) ### Support From 5800b0155a76722d3d1d791f00626caa3ed8a303 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 3 Oct 2023 13:36:11 +0100 Subject: [PATCH 180/192] Enable verbose http logging for file-server --- helm/flowforge/templates/file-storage.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helm/flowforge/templates/file-storage.yml b/helm/flowforge/templates/file-storage.yml index 4c940c99..3cb0a58e 100644 --- a/helm/flowforge/templates/file-storage.yml +++ b/helm/flowforge/templates/file-storage.yml @@ -26,6 +26,9 @@ data: options: {{ toYaml .Values.forge.fileStore.context.options | indent 8 -}} {{- end }} + logging: + level: info + http: info --- {{ if eq .Values.forge.fileStore.type "localfs" }} apiVersion: v1 From cffcadf721be71a67f6b7bc2a0be3886a82dc39a Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 3 Oct 2023 13:40:16 +0100 Subject: [PATCH 181/192] Add tests --- test/unit/configmap_spec.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/unit/configmap_spec.js b/test/unit/configmap_spec.js index 88771d0d..df4efd94 100644 --- a/test/unit/configmap_spec.js +++ b/test/unit/configmap_spec.js @@ -101,6 +101,12 @@ describe('Examine Config Maps', function () { const cm = configMaps.filter(s => s.metadata.name === 'flowforge-file-config') cm.should.have.length(1) }) + it('http logging enabled', function () { + const d = configMaps.filter(s => s.metadata.name === 'flowforge-file-config')[0] + fsYML = yaml.parse(d.data['flowforge-storage.yml']) + fsYML.should.have.property('logging') + fsYML.logging.should.have.property('http') + }) }) }) }) From 7847e9a35795caa9b6b80f22e940622672f22a60 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Thu, 5 Oct 2023 10:00:01 +0100 Subject: [PATCH 182/192] Increase broker acl cache to 5mins --- helm/flowforge/templates/broker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/flowforge/templates/broker.yaml b/helm/flowforge/templates/broker.yaml index 7161d910..c9eb0179 100644 --- a/helm/flowforge/templates/broker.yaml +++ b/helm/flowforge/templates/broker.yaml @@ -19,7 +19,7 @@ data: auth_opt_hasher bcrypt auth_opt_cache true auth_opt_auth_cache_seconds 30 - auth_opt_acl_cache_seconds 90 + auth_opt_acl_cache_seconds 300 auth_opt_auth_jitter_second 3 auth_opt_acl_jitter_seconds 5 From 34408ad6ea3e2f42f7d1a79d701b10b24db9d8ee Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Thu, 5 Oct 2023 10:07:12 +0100 Subject: [PATCH 183/192] Update helm/flowforge/templates/broker.yaml Co-authored-by: Nick O'Leary --- helm/flowforge/templates/broker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/flowforge/templates/broker.yaml b/helm/flowforge/templates/broker.yaml index c9eb0179..075738f0 100644 --- a/helm/flowforge/templates/broker.yaml +++ b/helm/flowforge/templates/broker.yaml @@ -18,7 +18,7 @@ data: auth_opt_backends http auth_opt_hasher bcrypt auth_opt_cache true - auth_opt_auth_cache_seconds 30 + auth_opt_auth_cache_seconds 120 auth_opt_acl_cache_seconds 300 auth_opt_auth_jitter_second 3 auth_opt_acl_jitter_seconds 5 From 52185d1d49ddb4d96cf1fd8c773d688debd37e9c Mon Sep 17 00:00:00 2001 From: ZJ van de Weg Date: Thu, 5 Oct 2023 11:25:08 +0200 Subject: [PATCH 184/192] config: Add Prometheus config The application now features a [/metrics endpoint][1] for Prometheus to scrape. Since merging, the configuration file structure has been updated since through [another PR][2]. This change requires both changes to be in the application and afterwards allows enabling the endpoint and an engineer to configure scraping of the data. [1]: https://github.com/FlowFuse/flowfuse/pull/2889 [2]: https://github.com/FlowFuse/flowfuse/pull/2893 --- helm/flowforge/README.md | 1 + helm/flowforge/templates/configmap.yaml | 6 ++++-- helm/flowforge/values.schema.json | 13 +++++++++++++ helm/flowforge/values.yaml | 3 +++ test/customizations.yml | 3 +++ test/unit/configmap_spec.js | 8 +++++++- 6 files changed, 31 insertions(+), 3 deletions(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 86fbac02..16bc33c1 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -84,6 +84,7 @@ Enables FlowForge Telemetry - `forge.telemetry.sentry.frontend_dsn` enables sentry reporting if set - `forge.telemetry.sentry.backend_dsn` enables sentry reporting if set - `forge.telemetry.sentry.production_mode` rate limit reporting (default `true`) + - `forge.telemetry.backend.prometheus.enabled` enables the `/metrics` endpoint on the forge app for scraping by Prometheus ### Support diff --git a/helm/flowforge/templates/configmap.yaml b/helm/flowforge/templates/configmap.yaml index 0cdab502..9c00b972 100644 --- a/helm/flowforge/templates/configmap.yaml +++ b/helm/flowforge/templates/configmap.yaml @@ -157,11 +157,13 @@ data: {{ end }} {{ end -}} {{- end }} - {{ if and (hasKey .Values.forge.telemetry "sentry") (hasKey .Values.forge.telemetry.sentry "backend_dsn") -}} backend: + {{ if and (hasKey .Values.forge.telemetry "sentry") (hasKey .Values.forge.telemetry.sentry "backend_dsn") -}} sentry: dsn: {{ .Values.forge.telemetry.sentry.backend_dsn}} - {{ end -}} + {{- end }} + prometheus: + enabled: {{ .Values.forge.telemetry.backend.prometheus.enabled }} {{- if .Values.forge.support.enabled }} support: enabled: true diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index 4c34cdc7..d45171d3 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -256,6 +256,19 @@ "type": "boolean" } } + }, + "backend": { + "type": "object", + "properties": { + "prometheus": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + } + } + } } } }, diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index 8ad6d937..746850a6 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -16,6 +16,9 @@ forge: role: management telemetry: enabled: true + backend: + prometheus: + enabled: false broker: enabled: false fileStore: diff --git a/test/customizations.yml b/test/customizations.yml index 2fd1259b..f1e9c441 100644 --- a/test/customizations.yml +++ b/test/customizations.yml @@ -47,6 +47,9 @@ forge: production_mode: false frontend_dsn: 'https://sentry.io/flowforge/flowforge-frontend' backend_dsn: 'https://sentry.io/flowforge/flowforge-backend' + backend: + prometheus: + enabled: true support: enabled: true hubspot: 12345678 diff --git a/test/unit/configmap_spec.js b/test/unit/configmap_spec.js index c7afca9d..b44c11ad 100644 --- a/test/unit/configmap_spec.js +++ b/test/unit/configmap_spec.js @@ -116,8 +116,8 @@ describe('Examine Config Maps', function () { const d = configMaps.filter(doc => doc.metadata.name === 'flowforge-config')[0] yml = yaml.parse(d.data['flowforge.yml']) }) + it('has sentry telemetry', function () { - console.log(yml.telemetry) yml.telemetry.frontend.sentry.should.have.property('production_mode') yml.telemetry.frontend.sentry.production_mode.should.equal(false) @@ -127,5 +127,11 @@ describe('Examine Config Maps', function () { yml.telemetry.backend.sentry.should.have.property('dsn') yml.telemetry.backend.sentry.dsn.should.equal('https://sentry.io/flowforge/flowforge-backend') }) + + describe('using Prometheus', function () { + it('has prometheus enabled', function() { + yml.telemetry.backend.prometheus.enabled.should.equal(true) + }) + }) }) }) From a628fa57ee56b1eb42e8186897a540197e9aa584 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Thu, 5 Oct 2023 16:35:04 +0100 Subject: [PATCH 185/192] Bump for 1.12.3 release --- CHANGELOG.md | 4 ++++ VERSION | 2 +- flowforge-container/package.json | 4 ++-- helm/flowforge/Chart.yaml | 4 ++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b80f262d..9c73541a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +#### 1.12.3: Relase + + - Bump to FlowForge + #### 1.12.2: Release - Bump to FlowForge v1.12.2 @hardillb diff --git a/VERSION b/VERSION index 6b89d58f..81f36323 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.12.2 +1.12.3 diff --git a/flowforge-container/package.json b/flowforge-container/package.json index fdd04e44..d58f35ec 100644 --- a/flowforge-container/package.json +++ b/flowforge-container/package.json @@ -1,13 +1,13 @@ { "name": "@flowforge/flowforge-k8s", - "version": "1.12.2", + "version": "1.12.3", "description": "FlowForge in Docker", "private": true, "author": { "name": "FlowForge Inc." }, "dependencies": { - "@flowforge/flowforge": "^1.12.2", + "@flowforge/flowforge": "^1.12.3", "@flowforge/kubernetes": "^1.12.0", "pg": "^8.7.1", "pg-hstore": "^2.3.4" diff --git a/helm/flowforge/Chart.yaml b/helm/flowforge/Chart.yaml index 0130820a..17888011 100644 --- a/helm/flowforge/Chart.yaml +++ b/helm/flowforge/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "flowforge" -version: "1.12.2" +version: "1.12.3" description: "FlowForge" type: "application" home: "https://flowforge.com" @@ -16,4 +16,4 @@ dependencies: maintainers: - name: "FlowForge Inc" url: "https://flowforge.com" -appVersion: "1.12.2" +appVersion: "1.12.3" From b0ff99ccc2a0ca93125dcce6e6ceeaf42ea9c98e Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Thu, 5 Oct 2023 16:42:36 +0100 Subject: [PATCH 186/192] Update CHANGELOG.md --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c73541a..a4d7458e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ #### 1.12.3: Relase - - Bump to FlowForge + - Update to FlowFuse v1.12.3 + - config: Add Prometheus config (#191) @ZJvandeWeg + - Increase broker acl cache to 5mins (#190) @hardillb + - Add support for configuring sentry telemetry (#188) @Pezmc + - Enable file server http logging (#189) @hardillb #### 1.12.2: Release From 42125c2816e6d5dc6bc6a6b3c79d70e3c1dd696c Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 6 Oct 2023 09:34:27 +0200 Subject: [PATCH 187/192] Add prometheus scrape annotations --- helm/flowforge/templates/deployment.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/helm/flowforge/templates/deployment.yaml b/helm/flowforge/templates/deployment.yaml index f10a14a3..58def172 100644 --- a/helm/flowforge/templates/deployment.yaml +++ b/helm/flowforge/templates/deployment.yaml @@ -13,6 +13,12 @@ spec: metadata: labels: app: flowforge + annotations: + {{- if .Values.forge.telemetry.backend.prometheus.enabled }} + prometheus.io/scrape: "true" + prometheus.io/port: "3000" + prometheus.io/path: "/metrics" + {{- end }} spec: serviceAccountName: flowforge securityContext: From e6fbdaee69e0649b1c58c302b24eedb4b30a4bbf Mon Sep 17 00:00:00 2001 From: Piotr Pawlowski Date: Fri, 6 Oct 2023 16:31:42 +0200 Subject: [PATCH 188/192] Fix annotations location --- helm/flowforge/templates/deployment.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/helm/flowforge/templates/deployment.yaml b/helm/flowforge/templates/deployment.yaml index 58def172..8d27da35 100644 --- a/helm/flowforge/templates/deployment.yaml +++ b/helm/flowforge/templates/deployment.yaml @@ -13,12 +13,12 @@ spec: metadata: labels: app: flowforge - annotations: - {{- if .Values.forge.telemetry.backend.prometheus.enabled }} - prometheus.io/scrape: "true" - prometheus.io/port: "3000" - prometheus.io/path: "/metrics" - {{- end }} + annotations: + {{- if .Values.forge.telemetry.backend.prometheus.enabled }} + prometheus.io/scrape: "true" + prometheus.io/port: "3000" + prometheus.io/path: "/metrics" + {{- end }} spec: serviceAccountName: flowforge securityContext: From c1eddbbe0252d33b57769e4981aedcf121d8b20c Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Wed, 11 Oct 2023 10:28:17 +0100 Subject: [PATCH 189/192] Add SENTRY_ENV to deployment --- helm/flowforge/README.md | 5 +++-- helm/flowforge/templates/deployment.yaml | 4 ++++ helm/flowforge/values.schema.json | 3 +++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 16bc33c1..631426e1 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -81,9 +81,10 @@ Enables FlowForge Telemetry - `forge.telemetry.posthog.apikey` enables posthog logging if set (not default) - `forge.telemetry.posthog.apiurl` sets posthog target host (default `https://app.posthog.com`) - `forge.telemetry.posthog.capture_pageview` (default `true`) - - `forge.telemetry.sentry.frontend_dsn` enables sentry reporting if set - - `forge.telemetry.sentry.backend_dsn` enables sentry reporting if set + - `forge.telemetry.sentry.frontend_dsn` enables sentry reporting if set (default unset) + - `forge.telemetry.sentry.backend_dsn` enables sentry reporting if set (default unset) - `forge.telemetry.sentry.production_mode` rate limit reporting (default `true`) + - `forge.telemetry.sentry.environment` set as SENTRY_ENV environment variable (default unset) - `forge.telemetry.backend.prometheus.enabled` enables the `/metrics` endpoint on the forge app for scraping by Prometheus ### Support diff --git a/helm/flowforge/templates/deployment.yaml b/helm/flowforge/templates/deployment.yaml index 8d27da35..089ff2c8 100644 --- a/helm/flowforge/templates/deployment.yaml +++ b/helm/flowforge/templates/deployment.yaml @@ -58,6 +58,10 @@ spec: - name: NODE_EXTRA_CA_CERTS value: /usr/local/ssl-certs/chain.pem {{- end }} + {{- if .Values.forge.telemetry.sentry.environment }} + - name: SENTRY_ENV + valeu: {{ .Values.forge.telemetry.sentry.environment }} + {{- end }} volumeMounts: - name: config mountPath: /usr/src/forge/etc diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index d45171d3..9c45efe8 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -254,6 +254,9 @@ }, "production_mode": { "type": "boolean" + }, + "environment": { + "type": "string" } } }, From 1777b204974b22eac0de1648468ba7fa38ac697d Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Wed, 11 Oct 2023 10:41:21 +0100 Subject: [PATCH 190/192] Update helm/flowforge/templates/deployment.yaml Co-authored-by: Pez Cuckow --- helm/flowforge/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/flowforge/templates/deployment.yaml b/helm/flowforge/templates/deployment.yaml index 089ff2c8..94e3c6d5 100644 --- a/helm/flowforge/templates/deployment.yaml +++ b/helm/flowforge/templates/deployment.yaml @@ -60,7 +60,7 @@ spec: {{- end }} {{- if .Values.forge.telemetry.sentry.environment }} - name: SENTRY_ENV - valeu: {{ .Values.forge.telemetry.sentry.environment }} + value: {{ .Values.forge.telemetry.sentry.environment }} {{- end }} volumeMounts: - name: config From 7995a320d907dc86c7d5a3a7a0b690380c730c27 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Wed, 11 Oct 2023 10:41:45 +0100 Subject: [PATCH 191/192] Update helm/flowforge/README.md Co-authored-by: Pez Cuckow --- helm/flowforge/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 631426e1..fc50c990 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -84,7 +84,7 @@ Enables FlowForge Telemetry - `forge.telemetry.sentry.frontend_dsn` enables sentry reporting if set (default unset) - `forge.telemetry.sentry.backend_dsn` enables sentry reporting if set (default unset) - `forge.telemetry.sentry.production_mode` rate limit reporting (default `true`) - - `forge.telemetry.sentry.environment` set as SENTRY_ENV environment variable (default unset) + - `forge.telemetry.sentry.environment` set SENTRY_ENV environment variable, which overrides NODE_ENV for the reported environment (default unset) - `forge.telemetry.backend.prometheus.enabled` enables the `/metrics` endpoint on the forge app for scraping by Prometheus ### Support From e2b9787ecf8dac6a092470b80b725f48a5e113d7 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Fri, 13 Oct 2023 13:04:11 +0100 Subject: [PATCH 192/192] Bump to 1.12.4 --- CHANGELOG.md | 9 ++++++++- VERSION | 2 +- flowforge-container/package.json | 2 +- helm/flowforge/Chart.yaml | 4 ++-- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4d7458e..f8c93780 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,11 @@ -#### 1.12.3: Relase +#### 1.12.4: Release + + - Update to FlowFuse v1.12.4 + - Add SENTRY_ENV to deployment (#195) @hardillb + - Fix annotations location (#194) @ppawlowski + - Add prometheus-related pod annotations (#193) @ppawlowski + +#### 1.12.3: Release - Update to FlowFuse v1.12.3 - config: Add Prometheus config (#191) @ZJvandeWeg diff --git a/VERSION b/VERSION index 81f36323..89c881bc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.12.3 +1.12.4 diff --git a/flowforge-container/package.json b/flowforge-container/package.json index d58f35ec..a665a517 100644 --- a/flowforge-container/package.json +++ b/flowforge-container/package.json @@ -7,7 +7,7 @@ "name": "FlowForge Inc." }, "dependencies": { - "@flowforge/flowforge": "^1.12.3", + "@flowforge/flowforge": "^1.12.4", "@flowforge/kubernetes": "^1.12.0", "pg": "^8.7.1", "pg-hstore": "^2.3.4" diff --git a/helm/flowforge/Chart.yaml b/helm/flowforge/Chart.yaml index 17888011..af8ed870 100644 --- a/helm/flowforge/Chart.yaml +++ b/helm/flowforge/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "flowforge" -version: "1.12.3" +version: "1.12.4" description: "FlowForge" type: "application" home: "https://flowforge.com" @@ -16,4 +16,4 @@ dependencies: maintainers: - name: "FlowForge Inc" url: "https://flowforge.com" -appVersion: "1.12.3" +appVersion: "1.12.4"