diff --git a/javascript/package-lock.json b/javascript/package-lock.json index 7fcb57abf..c6d27ba97 100644 --- a/javascript/package-lock.json +++ b/javascript/package-lock.json @@ -36,7 +36,7 @@ "eslint-plugin-prettier": "^5.0.1", "husky": "^8.0.3", "prettier": "^3.0.3", - "prettier-plugin-organize-imports": "^3.2.3" + "prettier-plugin-organize-imports": "^3.2.4" }, "engines": { "node": ">=18" @@ -5118,9 +5118,9 @@ } }, "node_modules/prettier-plugin-organize-imports": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/prettier-plugin-organize-imports/-/prettier-plugin-organize-imports-3.2.3.tgz", - "integrity": "sha512-KFvk8C/zGyvUaE3RvxN2MhCLwzV6OBbFSkwZ2OamCrs9ZY4i5L77jQ/w4UmUr+lqX8qbaqVq6bZZkApn+IgJSg==", + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/prettier-plugin-organize-imports/-/prettier-plugin-organize-imports-3.2.4.tgz", + "integrity": "sha512-6m8WBhIp0dfwu0SkgfOxJqh+HpdyfqSSLfKKRZSFbDuEQXDDndb8fTpRWkUrX/uBenkex3MgnVk0J3b3Y5byog==", "dev": true, "peerDependencies": { "@volar/vue-language-plugin-pug": "^1.0.4", diff --git a/javascript/package.json b/javascript/package.json index 947d89f5b..7785a4743 100644 --- a/javascript/package.json +++ b/javascript/package.json @@ -50,7 +50,7 @@ "eslint-plugin-prettier": "^5.0.1", "husky": "^8.0.3", "prettier": "^3.0.3", - "prettier-plugin-organize-imports": "^3.2.3" + "prettier-plugin-organize-imports": "^3.2.4" }, "workspaces": [ "packages/utils", diff --git a/javascript/packages/orchestrator/src/configGenerator.ts b/javascript/packages/orchestrator/src/configGenerator.ts index 5a2c5f5dc..976a9440d 100644 --- a/javascript/packages/orchestrator/src/configGenerator.ts +++ b/javascript/packages/orchestrator/src/configGenerator.ts @@ -242,7 +242,9 @@ export async function generateNetworkSpec( if (config.parachains && config.parachains.length) { for (const parachain of config.parachains) { - const para: CHAIN = whichChain(parachain.chain || ""); + const para: CHAIN = whichChain( + parachain.chain || parachain.chain_spec_path || "", + ); let computedStatePath, computedStateCommand, diff --git a/javascript/packages/orchestrator/src/spawner.ts b/javascript/packages/orchestrator/src/spawner.ts index f631ef0de..b3e6857c9 100644 --- a/javascript/packages/orchestrator/src/spawner.ts +++ b/javascript/packages/orchestrator/src/spawner.ts @@ -86,7 +86,9 @@ export const spawnNode = async ( const isAssetHubPolkadot = parachain && (parachain.chain?.includes("statemint") || - parachain.chain?.includes("asset-hub-polkadot")); + parachain.chain?.includes("asset-hub-polkadot") || + parachain.chainSpecPath?.includes("statemint") || + parachain.chainSpecPath?.includes("asset-hub-polkadot")); const keystoreFiles = await generateKeystoreFiles( node, nodeFilesPath, diff --git a/monitoring/go.mod b/monitoring/go.mod index 743d00f65..8642759d1 100644 --- a/monitoring/go.mod +++ b/monitoring/go.mod @@ -4,6 +4,7 @@ go 1.19 require ( github.com/prometheus/client_golang v1.15.1 + k8s.io/apimachinery v0.27.1 k8s.io/client-go v0.27.1 ) @@ -22,7 +23,6 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -33,12 +33,11 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/net v0.8.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/term v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.30.0 // indirect @@ -46,7 +45,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.27.1 // indirect - k8s.io/apimachinery v0.27.1 // indirect k8s.io/klog/v2 v2.90.1 // indirect k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect diff --git a/monitoring/go.sum b/monitoring/go.sum index 76a55cd58..a75be764c 100644 --- a/monitoring/go.sum +++ b/monitoring/go.sum @@ -24,6 +24,7 @@ github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTr github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -54,10 +55,9 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -66,8 +66,10 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= @@ -80,6 +82,9 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk= +github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= @@ -90,8 +95,8 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -100,6 +105,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -121,8 +127,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= @@ -136,15 +142,15 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -155,6 +161,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -184,6 +191,7 @@ google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cn google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/scripts/jobs_status_exporter/Dockerfile b/scripts/jobs_status_exporter/Dockerfile new file mode 100644 index 000000000..1f15a3217 --- /dev/null +++ b/scripts/jobs_status_exporter/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.10-slim + +WORKDIR /usr/src/app + +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY metrics.py . + +CMD ["python", "./metrics.py"] diff --git a/scripts/jobs_status_exporter/deployment.yaml b/scripts/jobs_status_exporter/deployment.yaml new file mode 100644 index 000000000..bd2b2fe0d --- /dev/null +++ b/scripts/jobs_status_exporter/deployment.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metrics-deployment + labels: + app: metrics +spec: + replicas: 1 + selector: + matchLabels: + app: metrics + template: + metadata: + labels: + app: metrics + spec: + containers: + - name: metrics + image: emamihe/gitlab-runner-metrics:1.0 + ports: + - containerPort: 8000 + name: http-metrics + env: + - name: GITLAB_PRIVATE_TOKEN + valueFrom: + secretKeyRef: + name: gitlab-token + key: token + - name: RUNNER_ID + value: "RkrwHxX5" diff --git a/scripts/jobs_status_exporter/main.py b/scripts/jobs_status_exporter/main.py new file mode 100644 index 000000000..c947e9129 --- /dev/null +++ b/scripts/jobs_status_exporter/main.py @@ -0,0 +1,58 @@ +import requests +import os +from flask import Flask, Response +from prometheus_client import Gauge, generate_latest, REGISTRY + +GITLAB_URL = "https://gitlab.parity.io" +PROJECT_IDS = os.environ.get("GITLAB_PROJECT_IDS", "").split(",") +PRIVATE_TOKEN = os.environ.get("PRIVATE_TOKEN", "") +TARGET_JOB_NAME = os.environ.get("JOB_NAME", "") + +HEADERS = { + "Private-Token": PRIVATE_TOKEN +} + +app = Flask(__name__) + +job_status_gauge = Gauge('gitlab_pipeline_job_status', + 'Number of targeted jobs per status', + ['status', 'project_id']) + +def get_all_pages(url): + response = requests.get(url, headers=HEADERS) + response.raise_for_status() + + data = response.json() + yield from data + + while 'next' in response.links.keys(): + response = requests.get(response.links['next']['url'], headers=HEADERS) + response.raise_for_status() + data = response.json() + yield from data + +@app.route('/metrics', methods=['GET']) +def metrics(): + for project_id in PROJECT_IDS: + statistics = { + 'canceled': 0, + 'success': 0, + 'pending': 0, + 'running': 0, + 'failed': 0 + } + + url = f"{GITLAB_URL}/api/v4/projects/{project_id}/pipelines" + for pipeline in get_all_pages(url): + jobs_url = f"{GITLAB_URL}/api/v4/projects/{project_id}/pipelines/{pipeline['id']}/jobs" + for job in get_all_pages(jobs_url): + if job['name'] == TARGET_JOB_NAME and job['status'] in statistics: + statistics[job['status']] += 1 + + for status, count in statistics.items(): + job_status_gauge.labels(status=status, project_id=project_id).set(count) + + return Response(generate_latest(REGISTRY), mimetype="text/plain") + +if __name__ == "__main__": + app.run(host='0.0.0.0', port=8000) diff --git a/scripts/jobs_status_exporter/metrics.py b/scripts/jobs_status_exporter/metrics.py new file mode 100644 index 000000000..252495271 --- /dev/null +++ b/scripts/jobs_status_exporter/metrics.py @@ -0,0 +1,48 @@ +import os +import requests +import time +from prometheus_client import start_http_server, Gauge + +GITLAB_API_ENDPOINT = 'https://gitlab.parity.io/api/v4/runners/{}/jobs' +GITLAB_PRIVATE_TOKEN = os.getenv('GITLAB_PRIVATE_TOKEN') +RUNNER_ID = os.getenv('RUNNER_ID') + +if not GITLAB_PRIVATE_TOKEN or not RUNNER_ID: + raise EnvironmentError('The environment variables GITLAB_PRIVATE_TOKEN and RUNNER_ID must be set.') + +status_gauges = { + 'created': Gauge('gitlab_runner_jobs_created', 'Number of created jobs'), + 'pending': Gauge('gitlab_runner_jobs_pending', 'Number of pending jobs'), + 'running': Gauge('gitlab_runner_jobs_running', 'Number of running jobs'), + 'failed': Gauge('gitlab_runner_jobs_failed', 'Number of failed jobs'), + 'success': Gauge('gitlab_runner_jobs_success', 'Number of successful jobs'), + 'canceled': Gauge('gitlab_runner_jobs_canceled', 'Number of canceled jobs'), + 'skipped': Gauge('gitlab_runner_jobs_skipped', 'Number of skipped jobs'), + 'manual': Gauge('gitlab_runner_jobs_manual', 'Number of manual jobs'), +} + +def fetch_jobs_by_runner(runner_id): + """Fetch jobs from a specific GitLab Runner and update Prometheus metrics.""" + headers = {'PRIVATE-TOKEN': GITLAB_PRIVATE_TOKEN} + response = requests.get(GITLAB_API_ENDPOINT.format(runner_id), headers=headers) + response.raise_for_status() + jobs = response.json() + + for gauge in status_gauges.values(): + gauge.set(0) + + for job in jobs: + status = job.get('status') + if status in status_gauges: + status_gauges[status].inc() + +def main(): + start_http_server(8000) + print("Metrics server running on port 8000") + + while True: + fetch_jobs_by_runner(RUNNER_ID) + time.sleep(60) + +if __name__ == '__main__': + main() diff --git a/scripts/jobs_status_exporter/requirements.txt b/scripts/jobs_status_exporter/requirements.txt new file mode 100644 index 000000000..007b272e3 --- /dev/null +++ b/scripts/jobs_status_exporter/requirements.txt @@ -0,0 +1,3 @@ +Flask==2.0.1 +requests==2.26.0 +prometheus_client==0.11.0 diff --git a/scripts/jobs_status_exporter/runner_id b/scripts/jobs_status_exporter/runner_id new file mode 100644 index 000000000..5798a3632 --- /dev/null +++ b/scripts/jobs_status_exporter/runner_id @@ -0,0 +1 @@ +175886 diff --git a/scripts/jobs_status_exporter/service-monitor.yaml b/scripts/jobs_status_exporter/service-monitor.yaml new file mode 100644 index 000000000..8a9794161 --- /dev/null +++ b/scripts/jobs_status_exporter/service-monitor.yaml @@ -0,0 +1,13 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: metrics-servicemonitor + labels: + app: metrics +spec: + selector: + matchLabels: + app: metrics + endpoints: + - port: http-metrics + interval: 15s diff --git a/scripts/jobs_status_exporter/service.yaml b/scripts/jobs_status_exporter/service.yaml new file mode 100644 index 000000000..bb70f09fb --- /dev/null +++ b/scripts/jobs_status_exporter/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: metrics-service + labels: + app: metrics +spec: + selector: + app: metrics + ports: + - port: 8000 + targetPort: http-metrics