From bf2e26cc246383881bd1718b99808c60d560529a Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 5 Aug 2024 14:19:57 +0400 Subject: [PATCH 01/26] fix(util): add missed new line at the end of package.json --- src/if-env/util/npm.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/if-env/util/npm.ts b/src/if-env/util/npm.ts index f02898399..c5f272255 100644 --- a/src/if-env/util/npm.ts +++ b/src/if-env/util/npm.ts @@ -79,6 +79,7 @@ export const updatePackageJsonDependencies = async ( packageJsonPath, JSON.stringify(parsedPackageJson, null, 2) ); + await fs.appendFile(packageJsonPath, '\n'); }; /** From 5dc924c0e975f1aeaff4d0abb5fcbe28d884591a Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 5 Aug 2024 14:24:02 +0400 Subject: [PATCH 02/26] test(mocks): add mock function for fs.appendFunction --- src/__mocks__/fs/index.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/__mocks__/fs/index.ts b/src/__mocks__/fs/index.ts index 0f37a4291..323c7693a 100644 --- a/src/__mocks__/fs/index.ts +++ b/src/__mocks__/fs/index.ts @@ -81,6 +81,11 @@ export const writeFile = async (pathToFile: string, content: string) => { } } else if (pathToFile.includes('package.json-npm')) { const updatedPath = pathToFile.replace('-npm', ''); + // try { + // await fsAsync.appendFile(updatedPath, '\n'); + // } catch (error) { + // console.log('---error', error); + // } const fileContent = await fsAsync.readFile(updatedPath, 'utf8'); expect(content).toBe(fileContent); @@ -108,6 +113,9 @@ export const writeFile = async (pathToFile: string, content: string) => { } }; +export const appendFile = (file: string, appendContent: string) => + `${file}${appendContent}`; + export const stat = async (filePath: string) => { if (filePath === 'true') { return true; From 7d106933d680560b4de5a749b9c7a008521a2d63 Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 5 Aug 2024 16:40:20 +0400 Subject: [PATCH 03/26] test(mocks): remove unneseccary comment --- src/__mocks__/fs/index.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/__mocks__/fs/index.ts b/src/__mocks__/fs/index.ts index 323c7693a..1f0ce0b7f 100644 --- a/src/__mocks__/fs/index.ts +++ b/src/__mocks__/fs/index.ts @@ -81,11 +81,6 @@ export const writeFile = async (pathToFile: string, content: string) => { } } else if (pathToFile.includes('package.json-npm')) { const updatedPath = pathToFile.replace('-npm', ''); - // try { - // await fsAsync.appendFile(updatedPath, '\n'); - // } catch (error) { - // console.log('---error', error); - // } const fileContent = await fsAsync.readFile(updatedPath, 'utf8'); expect(content).toBe(fileContent); From 96d5aeb229c8528a37b1bf1055c082982fb42176 Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 5 Aug 2024 19:03:21 +0400 Subject: [PATCH 04/26] docs(builtins): update plugins readme file to be relevant to phased execution --- src/if-run/builtins/README.md | 224 ++---------------- src/if-run/builtins/coefficient/README.md | 5 +- src/if-run/builtins/copy-param/README.md | 3 +- src/if-run/builtins/csv-lookup/README.md | 3 +- src/if-run/builtins/divide/README.md | 5 +- src/if-run/builtins/exponent/README.md | 5 +- src/if-run/builtins/interpolation/README.md | 6 +- .../builtins/mock-observations/README.md | 5 +- src/if-run/builtins/multiply/README.md | 5 +- src/if-run/builtins/regex/README.md | 5 +- src/if-run/builtins/sci-embodied/README.md | 3 +- src/if-run/builtins/sci/README.md | 4 +- src/if-run/builtins/shell/README.md | 6 +- src/if-run/builtins/subtract/README.md | 5 +- src/if-run/builtins/sum/README.md | 5 +- 15 files changed, 50 insertions(+), 239 deletions(-) diff --git a/src/if-run/builtins/README.md b/src/if-run/builtins/README.md index ccacf83e5..e05110887 100644 --- a/src/if-run/builtins/README.md +++ b/src/if-run/builtins/README.md @@ -4,7 +4,6 @@ There are three built-in features of IF: - time-sync - CSV exporter -- groupby On this page, you can find the documentation for each of these three builtins. @@ -242,11 +241,12 @@ tree: children: child: # an advanced grouping node pipeline: - - teads-curve - - sci-e - - sci-embodied - - sci-o - - time-sync + compute: + - teads-curve + - sci-e + - sci-embodied + - sci-o + - time-sync config: teads-curve: cpu/thermal-design-power: 65 @@ -307,20 +307,19 @@ The CSV above is generated from the following yaml. The `carbon` metric is extra ```yaml tree: pipeline: - - mock-observations - - group-by - - cloud-metadata - - time-sync - - watttime - - teads-curve - - operational-carbon + observe: + - mock-observations + regroup: + - cloud/region + - name + compute: + - cloud-metadata + - time-sync + - watttime + - teads-curve + - operational-carbon defaults: grid/carbon-intensity: 500 - config: - group-by: - group: - - cloud/region - - name children: westus3: children: @@ -634,192 +633,3 @@ tree: ### CSV and aggregation The CSV representation of the output data is helpful for intuiting how the aggregation procedure works. What we refer to as "horizontal" aggregation is really an aggregation of the _rows_ of the CSV. You can replicate the IF aggregation function by summing the cells in each row of the CSV. Similarly, what we refer to as "vertical" aggregation can be replicated by summing the _columns_ in the CSV representation (this is not _exactly_ accurate because you have to skip summing both parent nodes and their children, both of which are represented in the CSV, but it is true conceptually). - -## Groupby - -Groupby is an IF plugin that reorganizes a tree according to keys provided by the user. This allows users to regroup their observations according to various properties of their application. For example, the following manifest file contains a flat array of observations. This is how you might expect data to arrive from an importer plugin, maybe one that hits a metrics API for a cloud service. - -```yaml -name: if-demo -description: demo pipeline -graph: - children: - my-app: - pipeline: - - group-by - - teads-curve - config: - group-by: - - cloud/region - - instance-type - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 99 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 23 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 12 - - timestamp: 2023-07-06T00:00 # note this time restarts at the start timstamp - duration: 300 - instance-type: B1 - cloud/region: uk-west - cpu-util: 11 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: B1 - cloud/region: uk-west - cpu-util: 67 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: B1 - cloud/region: uk-west - cpu-util: 1 -``` - -However, each observation contains an `instance-type` field that varies between observations. There are two instance types being represented in this array of observations. This means there are duplicate entries for the same timestamp in this array. This is the problem that `group-by` solves. You provide `instance-type` as a key to the `group-by` plugin and it extracts the data belonging to the different instances and separates them into independent arrays. The above example would be restructured so that instance types `A1` and `B1` have their own data, as follows: - -```yaml -graph: - children: - my-app: - pipeline: - # - group-by - - teads-curve - config: - group-by: - groups: - - cloud/region - - instance-type - children: - A1: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 99 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 23 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 12 - B1: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 11 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 67 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 1 -``` - -### Using `group-by` - -To use `group-by`, you have to initialize it as a plugin and invoke it in a pipeline. - -The initialization looks as follows: - -```yaml -initialize: -plugins: -group-by: - path: 'builtin' - method: GroupBy -``` - -You then have to provide config defining which keys to group by in each component. This is done at the component level (i.e. not global config). -For example: - -```yaml -tree: - children: - my-app: - pipeline: - - group-by - config: - group-by: - group: - - cloud/region - - instance-type -``` - -In the example above, the plugin would regroup the input data for the specific component by `cloud/region` and by `instance-type`. - -Assuming the values `A1` and `B1` are found for `instance-type` and the values `uk-east` and `uk-west` are found for `cloud/region`, the result of `group-by` would look similar to the following: - -```yaml -tree: - children: - my-app: - pipeline: - - group-by - config: - group-by: - groups: - - cloud/region - - instance-type - children: - uk-west: - children: - A1: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 99 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 23 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 12 - uk-east: - children: - B1: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 11 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 67 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 1 -``` - -This reorganized data can then be used to feed the rest of a computation pipeline. diff --git a/src/if-run/builtins/coefficient/README.md b/src/if-run/builtins/coefficient/README.md index a80f4e4fb..903a15496 100644 --- a/src/if-run/builtins/coefficient/README.md +++ b/src/if-run/builtins/coefficient/README.md @@ -97,9 +97,8 @@ tree: children: child: pipeline: - - coefficient - config: - coefficient: + compute: + - coefficient inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/copy-param/README.md b/src/if-run/builtins/copy-param/README.md index 309a83fde..2f76c98da 100644 --- a/src/if-run/builtins/copy-param/README.md +++ b/src/if-run/builtins/copy-param/README.md @@ -107,7 +107,8 @@ tree: children: child-1: pipeline: - - copy-param + compute: + - copy-param inputs: - timestamp: '2023-12-12T00:00:00.000Z' original: 'hello' diff --git a/src/if-run/builtins/csv-lookup/README.md b/src/if-run/builtins/csv-lookup/README.md index 31bdcdb17..434e234a2 100644 --- a/src/if-run/builtins/csv-lookup/README.md +++ b/src/if-run/builtins/csv-lookup/README.md @@ -135,7 +135,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/divide/README.md b/src/if-run/builtins/divide/README.md index b17925dff..abb534c8f 100644 --- a/src/if-run/builtins/divide/README.md +++ b/src/if-run/builtins/divide/README.md @@ -89,9 +89,8 @@ tree: children: child: pipeline: - - divide - config: - divide: + compute: + - divide inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/exponent/README.md b/src/if-run/builtins/exponent/README.md index b4f4d7d17..fddb55ffa 100644 --- a/src/if-run/builtins/exponent/README.md +++ b/src/if-run/builtins/exponent/README.md @@ -90,9 +90,8 @@ tree: children: child: pipeline: - - exponent - config: - exponent: + compute: + - exponent inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/interpolation/README.md b/src/if-run/builtins/interpolation/README.md index a6d897f32..243b70e70 100644 --- a/src/if-run/builtins/interpolation/README.md +++ b/src/if-run/builtins/interpolation/README.md @@ -131,7 +131,8 @@ tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 @@ -159,7 +160,8 @@ tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/mock-observations/README.md b/src/if-run/builtins/mock-observations/README.md index 91f418ccb..53bd89924 100644 --- a/src/if-run/builtins/mock-observations/README.md +++ b/src/if-run/builtins/mock-observations/README.md @@ -62,7 +62,7 @@ const result = await mockObservations.execute([]); ### manifest Example -IF users will typically call the plugin as part of a pipeline defined in a `manifest` file. In this case, instantiating the plugin is handled by `ie` and does not have to be done explicitly by the user. The following is an example `manifest` that calls `mock-observation`: +IF users will typically call the plugin as part of a pipeline defined in a `manifest` file. In this case, instantiating the plugin is handled by `if-run` and does not have to be done explicitly by the user. The following is an example `manifest` that calls `mock-observation`: ```yaml name: mock-observation-demo @@ -96,7 +96,8 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: ``` diff --git a/src/if-run/builtins/multiply/README.md b/src/if-run/builtins/multiply/README.md index 86a65aecb..dc6cde7c7 100644 --- a/src/if-run/builtins/multiply/README.md +++ b/src/if-run/builtins/multiply/README.md @@ -87,9 +87,8 @@ tree: children: child: pipeline: - - multiply - config: - multiply: + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/regex/README.md b/src/if-run/builtins/regex/README.md index d9f382274..2c6a4cd88 100644 --- a/src/if-run/builtins/regex/README.md +++ b/src/if-run/builtins/regex/README.md @@ -82,9 +82,8 @@ tree: children: child: pipeline: - - regex - config: - regex: + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/sci-embodied/README.md b/src/if-run/builtins/sci-embodied/README.md index ed9c1841e..a4c304c6d 100644 --- a/src/if-run/builtins/sci-embodied/README.md +++ b/src/if-run/builtins/sci-embodied/README.md @@ -102,7 +102,8 @@ tree: children: child: pipeline: - - sci-embodied # duration & config -> embodied + compute: + - sci-embodied # duration & config -> embodied defaults: device/emissions-embodied: 1533.120 # gCO2eq device/expected-lifespan: 3 # 3 years in seconds diff --git a/src/if-run/builtins/sci/README.md b/src/if-run/builtins/sci/README.md index d0b08c9c5..31b15afed 100644 --- a/src/if-run/builtins/sci/README.md +++ b/src/if-run/builtins/sci/README.md @@ -82,8 +82,8 @@ tree: children: child: pipeline: - - sci - config: + compute: + - sci inputs: - timestamp: 2023-07-06T00:00 carbon: 5 diff --git a/src/if-run/builtins/shell/README.md b/src/if-run/builtins/shell/README.md index 5e529f922..d114adb3b 100644 --- a/src/if-run/builtins/shell/README.md +++ b/src/if-run/builtins/shell/README.md @@ -91,7 +91,8 @@ tree: children: child: pipeline: - - sampler + compute: + - sampler inputs: - timestamp: 2023-07-06T00:00 duration: 1 # Secs @@ -117,7 +118,8 @@ tree: children: child: pipeline: - - sampler + compute: + - sampler inputs: - timestamp: 2023-07-06T00:00 duration: 1 # Secs diff --git a/src/if-run/builtins/subtract/README.md b/src/if-run/builtins/subtract/README.md index 89a186ab3..776526c7e 100644 --- a/src/if-run/builtins/subtract/README.md +++ b/src/if-run/builtins/subtract/README.md @@ -87,9 +87,8 @@ tree: children: child: pipeline: - - subtract - config: - subtract: + compute: + - subtract inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/sum/README.md b/src/if-run/builtins/sum/README.md index 014111c6c..998c54c11 100644 --- a/src/if-run/builtins/sum/README.md +++ b/src/if-run/builtins/sum/README.md @@ -97,9 +97,8 @@ tree: children: child: pipeline: - - sum - config: - sum: + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 From 4e27cde0138c7d63cc94f12a0e57472b6b5acd4e Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 5 Aug 2024 19:12:07 +0400 Subject: [PATCH 05/26] fix(manifests): update some manifests names --- ...-group.yml => failure-invalid-regroup.yml} | 5 +- ...roup.yaml => failure-invalid-regroup.yaml} | 46 +++++++++---------- 2 files changed, 25 insertions(+), 26 deletions(-) rename manifests/examples/features/regroup/{failure-invalid-config-group.yml => failure-invalid-regroup.yml} (91%) rename manifests/outputs/features/{failure-invalid-config-group.yaml => failure-invalid-regroup.yaml} (64%) diff --git a/manifests/examples/features/regroup/failure-invalid-config-group.yml b/manifests/examples/features/regroup/failure-invalid-regroup.yml similarity index 91% rename from manifests/examples/features/regroup/failure-invalid-config-group.yml rename to manifests/examples/features/regroup/failure-invalid-regroup.yml index 5d331ba05..a121d82b5 100644 --- a/manifests/examples/features/regroup/failure-invalid-config-group.yml +++ b/manifests/examples/features/regroup/failure-invalid-regroup.yml @@ -1,13 +1,12 @@ name: regroup -description: failure when `config->group-by->group` is not an array +description: failure when `regroup` is not an array initialize: plugins: {} tree: children: my-app: pipeline: - regroup: - cloud/region + regroup: cloud/region inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/features/failure-invalid-config-group.yaml b/manifests/outputs/features/failure-invalid-regroup.yaml similarity index 64% rename from manifests/outputs/features/failure-invalid-config-group.yaml rename to manifests/outputs/features/failure-invalid-regroup.yaml index 1ed31b06d..b3f3e722c 100644 --- a/manifests/outputs/features/failure-invalid-config-group.yaml +++ b/manifests/outputs/features/failure-invalid-regroup.yaml @@ -1,31 +1,31 @@ -name: groupby -description: failure when `config->group-by->group` is not an array +name: regroup +description: failure when `regroup` is not an array initialize: plugins: {} execution: status: fail command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/groupby/failure-invalid-config-group.yml -o - manifests/outputs/plugins/groupby/failure-invalid-config-group + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/features/regroup/failure-invalid-regroup.yml -o + manifests/outputs/features/failure-invalid-regroup.yaml environment: - if-version: 0.4.0 + if-version: 0.5.0 os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-02T19:11:23.166Z (UTC) + os-version: 13.6.7 + node-version: 18.20.0 + date-time: 2024-08-05T14:53:00.560Z (UTC) dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.16' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -34,6 +34,7 @@ execution: - fixpack@4.0.0 - gts@5.2.0 - husky@8.0.3 + - if-eco-ci-plugin@v0.0.1 extraneous -> file:../../if-eco-ci-plugin - jest@29.7.0 - js-yaml@4.1.0 - lint-staged@15.2.2 @@ -45,7 +46,7 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 error: >- InputValidationError: "groups" parameter is expected array, received string. Error code: invalid_type. @@ -53,8 +54,7 @@ tree: children: my-app: pipeline: - regroup: - cloud/region + regroup: cloud/region inputs: - timestamp: 2023-07-06T00:00 duration: 300 From 83a0778e5b03f97bb74835970646cc1089091821 Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 5 Aug 2024 19:26:16 +0400 Subject: [PATCH 06/26] test(mocks): update mock manifest example --- src/__mocks__/fs/index.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/__mocks__/fs/index.ts b/src/__mocks__/fs/index.ts index 0f37a4291..bb13a39b8 100644 --- a/src/__mocks__/fs/index.ts +++ b/src/__mocks__/fs/index.ts @@ -52,8 +52,9 @@ cpu-cores-available,cpu-cores-utilized,cpu-manufacturer,cpu-model-name,cpu-tdp,g tree: children: front-end: - pipeline: - - boavizta-cpu + pipeline: + compute: + - boavizta-cpu config: boavizta-cpu: core-units: 24 From b7f433ae246a71f64e87d4295ed46a8341c3adf3 Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 5 Aug 2024 19:27:48 +0400 Subject: [PATCH 07/26] test(mocks): update mock-manifest.yaml --- src/__mocks__/mock-manifest.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/__mocks__/mock-manifest.yaml b/src/__mocks__/mock-manifest.yaml index 64e00e301..a69009d1e 100644 --- a/src/__mocks__/mock-manifest.yaml +++ b/src/__mocks__/mock-manifest.yaml @@ -57,7 +57,8 @@ tree: children: child: pipeline: - - memory-energy-from-memory-util + compute: + - memory-energy-from-memory-util config: null inputs: - timestamp: 2023-12-12T00:00:00.000Z From 949a34a2e15f186071062df93385d2d9588e4790 Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 5 Aug 2024 19:29:35 +0400 Subject: [PATCH 08/26] fix(manifests): remove group-by from plugins --- .../features/regroup/failure-missing-cloud-instance-type.yml | 3 --- manifests/examples/features/regroup/success.yml | 3 --- 2 files changed, 6 deletions(-) diff --git a/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml b/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml index 0b02593fd..8b8b44faf 100644 --- a/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml +++ b/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml @@ -2,9 +2,6 @@ name: regroup description: initialize: plugins: - group-by: - path: "builtin" - method: GroupBy tree: children: my-app: diff --git a/manifests/examples/features/regroup/success.yml b/manifests/examples/features/regroup/success.yml index 4677e8edb..6863dc0d6 100644 --- a/manifests/examples/features/regroup/success.yml +++ b/manifests/examples/features/regroup/success.yml @@ -2,9 +2,6 @@ name: regroup description: successful path initialize: plugins: - group-by: - path: "builtin" - method: GroupBy tree: children: my-app: From 557de44184059c53bd53b82b79cc428e133bbabd Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 5 Aug 2024 19:31:10 +0400 Subject: [PATCH 09/26] fix(config): update if-env template manifest --- src/if-env/config/env-template.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/if-env/config/env-template.yml b/src/if-env/config/env-template.yml index ba64c88dc..dc801cbc1 100644 --- a/src/if-env/config/env-template.yml +++ b/src/if-env/config/env-template.yml @@ -14,7 +14,8 @@ tree: children: # add a chile for each distinct component you want to measure child: pipeline: # the pipeline is an ordered list of plugins you want to execute - - memory-energy-from-memory-util # must match the name in initialize! + compute: + - memory-energy-from-memory-util # must match the name in initialize! config: # any plugin specific, node-level config inputs: - timestamp: 2023-12-12T00:00:00.000Z # ISO 8061 string From 5c592489ec3501b7300d0dd890af041f84f0849a Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 5 Aug 2024 19:32:59 +0400 Subject: [PATCH 10/26] docs(doc): update migration guide readme file --- Refactor-migration-guide.md | 70 ++++++++++--------------------------- 1 file changed, 18 insertions(+), 52 deletions(-) diff --git a/Refactor-migration-guide.md b/Refactor-migration-guide.md index f8da3d8b8..e29a80d0e 100644 --- a/Refactor-migration-guide.md +++ b/Refactor-migration-guide.md @@ -125,24 +125,23 @@ There have also been some changes to the structure of manifest files. Some of th - **Node level config** - We have also introduced the concept of node-level config. This is designed for pluin configuration that might vary between components in the tree. For example, for each child in the tree you might wish to use the `groupby` plugin to group the outputs according to a different set of keys. + We have also introduced the concept of node-level config. This is designed for pluin configuration that might vary between components in the tree. For example, for each child in the tree you might wish to use the `regroup` feature to group the outputs according to a different set of keys. ```yaml tree: children: child-1: pipeline: - - teads-curve - - sci-e - - sci-embodied - - sci-o - - time-sync - - sci - config: - group-by: - group: - - region - - cloud/instance-type + compute: + - teads-curve + - sci-e + - sci-embodied + - sci-o + - time-sync + - sci + regroup: + - region + - cloud/instance-type ``` - **Defaults** @@ -156,12 +155,13 @@ There have also been some changes to the structure of manifest files. Some of th children: child-1: pipeline: - - teads-curve - - sci-e - - sci-embodied - - sci-o - - time-sync - - sci + compute: + - teads-curve + - sci-e + - sci-embodied + - sci-o + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -215,40 +215,6 @@ aggregation: type: 'both' ``` -### Groupby - -Groupby allows you to regroup your outputs according to keys you define. For example, maybe you want to group your outputs by region (show me all the outputs for applications run in `uk-south` etc). Groupby _is_ a plugin that needs to be initialized in the manifest. - -You can initialize the plugin as follows: - -```yaml -initialize: - plugins: - 'group-by': - path: builtin - method: GroupBy -``` - -Then you configure groupby for each component in the node level config. In the following example we will regroup the outputs by the `region`: - -```yaml -tree: - children: - child-1: - pipeline: - - teads-curve - - sci-e - - sci-embodied - - sci-o - - time-sync - - group-by - - sci - config: - group-by: - group: - - region -``` - ### Exhaust We have introduced `exhaust` as an IF feature. This is a wrapper around export plugins and it allows community contributors to create plugins for exporting to different formats. From 7e6f61b122baf66b56d84dab26756548739aaff7 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 6 Aug 2024 10:57:31 +0400 Subject: [PATCH 11/26] feat(.github): brake down worflow to multiple jobs --- .github/workflows/release-commit.yml | 38 +++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-commit.yml b/.github/workflows/release-commit.yml index efcf8726e..8ffa665f7 100644 --- a/.github/workflows/release-commit.yml +++ b/.github/workflows/release-commit.yml @@ -8,7 +8,7 @@ env: PRE_RELEASE: ${{ github.event.release.prerelease}} jobs: - release-commit-pr: + unit-tests: runs-on: ubuntu-latest steps: - name: Checkout code @@ -28,9 +28,45 @@ jobs: - name: Run unit tests run: npm test + integration-tests: + runs-on: ubuntu-latest + needs: unit-tests + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 18 + registry-url: https://registry.npmjs.org/ + + - name: Install dependencies + run: npm ci + - name: Run integration tests run: npm run if-check -- -d manifests/outputs + publish: + runs-on: ubuntu-latest + needs: integration-tests + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 18 + registry-url: https://registry.npmjs.org/ + + - name: Install dependencies + run: npm ci + - name: Initialize git user email run: git config --global user.email "${{ vars.RELEASE_USER_EMAIL }}" From c5814ef54640ec4ae78af3d161c1fa93192e5925 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Tue, 6 Aug 2024 11:16:32 +0400 Subject: [PATCH 12/26] fix(manifests): convert if diff outputs to phased exec --- .../pipeline-with-aggregate.yaml | 62 +++++++++-------- .../outputs-if-diff/pipeline-with-mocks.yaml | 66 +++++++++---------- 2 files changed, 62 insertions(+), 66 deletions(-) diff --git a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml index e227dd7ef..ee14e4e4a 100644 --- a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml +++ b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml @@ -237,22 +237,21 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -590,22 +589,21 @@ tree: carbon: 0.04532668505834602 child-2: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 diff --git a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml index 39d7fef1b..a050da896 100644 --- a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml +++ b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml @@ -324,23 +324,22 @@ tree: children: child-1: pipeline: - - mock-observations - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - instance-type + regroup: + - cloud/region + - instance-type + compute: + - mock-observations + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -678,23 +677,22 @@ tree: carbon: '*' child-2: pipeline: - - mock-observations - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - instance-type + compute: + - mock-observations + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 From 90e7fe3bbb4a755f3a8d64675b1dd4670d345b32 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Tue, 6 Aug 2024 11:31:11 +0400 Subject: [PATCH 13/26] fix(manifests): convert nesting output to phased exec --- manifests/outputs/pipelines/nesting.yaml | 69 ++++++++++++------------ 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/manifests/outputs/pipelines/nesting.yaml b/manifests/outputs/pipelines/nesting.yaml index 83ed612d5..5e26ae900 100644 --- a/manifests/outputs/pipelines/nesting.yaml +++ b/manifests/outputs/pipelines/nesting.yaml @@ -609,17 +609,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 @@ -976,17 +977,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 @@ -1341,17 +1343,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 From 0e753d0ad2b71421dbfb4da9bd4d4679bc1a5533 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Tue, 6 Aug 2024 15:43:23 +0400 Subject: [PATCH 14/26] fix(manifests): move mock observations to observe --- .../pipelines/outputs-if-diff/pipeline-with-mocks.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml index a050da896..983e7a638 100644 --- a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml +++ b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml @@ -324,11 +324,12 @@ tree: children: child-1: pipeline: + observe: + - mock-observations regroup: - cloud/region - instance-type compute: - - mock-observations - interpolate - cpu-factor-to-wattage - wattage-times-duration @@ -677,11 +678,12 @@ tree: carbon: '*' child-2: pipeline: + observe: + - mock-observations regroup: - cloud/region - instance-type compute: - - mock-observations - interpolate - cpu-factor-to-wattage - wattage-times-duration @@ -692,7 +694,7 @@ tree: - operational-carbon - sum-carbon - time-sync - - sci + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 From 237f44b709ac683444281a5c0e7909278eae6fd9 Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 6 Aug 2024 18:10:40 +0400 Subject: [PATCH 15/26] fix(config): update `REGROUP_ERROR` message --- src/if-run/config/strings.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/if-run/config/strings.ts b/src/if-run/config/strings.ts index 24835f8ed..e5e6e449b 100644 --- a/src/if-run/config/strings.ts +++ b/src/if-run/config/strings.ts @@ -24,8 +24,7 @@ export const STRINGS = { METRIC_MISSING: (metric: string, index: number) => `Aggregation metric ${metric} is not found in inputs[${index}].`, INVALID_GROUP_KEY: (key: string) => `Invalid group ${key}.`, - REGROUP_ERROR: - 'Regroup phase is not an array or should contain at least one key.', + REGROUP_ERROR: 'not an array or should contain at least one key', INVALID_EXHAUST_PLUGIN: (pluginName: string) => `Invalid exhaust plugin: ${pluginName}.`, UNKNOWN_PARAM: (name: string) => From a6abeed714f2845c5bc7304047a845fd3b4af7f1 Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 6 Aug 2024 18:12:20 +0400 Subject: [PATCH 16/26] fix(lib): fix regroup validation --- src/if-run/lib/regroup.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/if-run/lib/regroup.ts b/src/if-run/lib/regroup.ts index eba87a4d0..affd51307 100644 --- a/src/if-run/lib/regroup.ts +++ b/src/if-run/lib/regroup.ts @@ -44,12 +44,13 @@ export const Regroup = (inputs: PluginParams[], groups: string[]) => { /** * Validates groups array. */ - const validateGroups = (groups: string[]) => { - const inputData = {groups}; + const validateGroups = (regroup: string[]) => { + const inputData = {regroup}; const validationSchema = z.record( z.string(), - z.array(z.string()).min(1, REGROUP_ERROR) + z.array(z.string(), {message: REGROUP_ERROR}).min(1) ); + validate(validationSchema, inputData); return groups; From 98bdee7bebb696ec242ee5dbba6f630ce857e95f Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 6 Aug 2024 18:14:15 +0400 Subject: [PATCH 17/26] test(lib): update error messages --- src/__tests__/if-run/lib/regroup.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/__tests__/if-run/lib/regroup.test.ts b/src/__tests__/if-run/lib/regroup.test.ts index 9f0b373ff..67ff72e53 100644 --- a/src/__tests__/if-run/lib/regroup.test.ts +++ b/src/__tests__/if-run/lib/regroup.test.ts @@ -86,7 +86,7 @@ describe('lib/regroup: ', () => { expect(error).toBeInstanceOf(InputValidationError); expect(error).toEqual( new InputValidationError( - '"groups" parameter is required. Error code: invalid_type.' + '"regroup" parameter is not an array or should contain at least one key. Error code: invalid_type.' ) ); } @@ -135,7 +135,7 @@ describe('lib/regroup: ', () => { expect(error).toBeInstanceOf(InputValidationError); expect(error).toEqual( new InputValidationError( - '"groups" parameter is regroup phase is not an array or should contain at least one key.. Error code: too_small.' + '"regroup" parameter is array must contain at least 1 element(s). Error code: too_small.' ) ); } From 8c41f70d3db843ed91af8a92db6305699e304ac2 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 17:52:29 +0400 Subject: [PATCH 18/26] fix(manifests): remove `method` from aggregation section --- manifests/examples/pipelines/nesting.yml | 26 +- .../pipeline-with-aggregate.yaml | 18 +- .../outputs-if-diff/pipeline-with-mocks.yaml | 655 +++++++++--------- .../pipelines/pipeline-with-aggregate.yml | 9 +- .../pipelines/pipeline-with-mocks.yml | 3 +- .../bugs/aggregation-error-wrong-metric.yaml | 3 +- .../aggregate-failure-invalid-metrics.yaml | 3 +- ...gate-failure-missing-metric-in-inputs.yaml | 3 +- .../features/aggregate-horizontal.yaml | 9 +- .../outputs/features/aggregate-vertical.yaml | 9 +- manifests/outputs/features/aggregate.yaml | 9 +- manifests/outputs/pipelines/nesting.yaml | 234 ++++--- 12 files changed, 519 insertions(+), 462 deletions(-) diff --git a/manifests/examples/pipelines/nesting.yml b/manifests/examples/pipelines/nesting.yml index 185c715e0..baf12a688 100644 --- a/manifests/examples/pipelines/nesting.yml +++ b/manifests/examples/pipelines/nesting.yml @@ -6,8 +6,7 @@ tags: category: on-premise aggregation: metrics: - "carbon": - method: sum + - carbon type: "both" initialize: plugins: @@ -139,6 +138,21 @@ initialize: - carbon-operational - carbon-embodied output-parameter: carbon + parameter-metadata: + inputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + carbon-embodied: + description: Embodied carbon footprint + unit: gCO2eq + aggregation-method: sum + outputs: + carbon: + description: Total carbon footprint + unit: gCO2eq + aggregation-method: sum time-sync: method: TimeSync path: "builtin" @@ -169,10 +183,10 @@ initialize: unit: seconds description: time reserved for a component aggregation-method: avg - network/energy: - description: 'Energy consumed by the Network of the component' - unit: 'kWh' - aggregation-method: 'sum' + network/energy: + description: "Energy consumed by the Network of the component" + unit: "kWh" + aggregation-method: "sum" tree: children: child-0: diff --git a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml index ee14e4e4a..c1f92da04 100644 --- a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml +++ b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml @@ -3,8 +3,7 @@ description: a full pipeline with the aggregate feature enabled tags: null aggregation: metrics: - carbon: - method: sum + - carbon type: both initialize: plugins: @@ -159,6 +158,21 @@ initialize: - carbon-operational - carbon-embodied output-parameter: carbon + parameter-metadata: + inputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + carbon-embodied: + description: Embodied carbon footprint + unit: gCO2eq + aggregation-method: sum + outputs: + carbon: + description: Total carbon footprint + unit: gCO2eq + aggregation-method: sum time-sync: path: builtin method: TimeSync diff --git a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml index 983e7a638..77496bf23 100644 --- a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml +++ b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml @@ -3,8 +3,7 @@ description: a full pipeline seeded with data from mock-observations feature tags: null aggregation: metrics: - carbon: - method: sum + - carbon type: both initialize: plugins: @@ -52,7 +51,7 @@ initialize: - 10 - 50 - 100 - 'y': + "y": - 0.12 - 0.32 - 0.75 @@ -247,8 +246,8 @@ initialize: path: builtin method: TimeSync global-config: - start-time: '2023-12-12T00:00:00.000Z' - end-time: '2023-12-12T00:01:00.000Z' + start-time: "2023-12-12T00:00:00.000Z" + end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true parameter-metadata: @@ -285,20 +284,20 @@ execution: environment: if-version: 0.5.0 os: macOS - os-version: '14.5' + os-version: "14.5" node-version: 18.14.2 date-time: 2024-07-30T05:35:42.937Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.16' - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.16" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -340,7 +339,7 @@ tree: - operational-carbon - sum-carbon - time-sync - - sci + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -350,36 +349,36 @@ tree: vcpus-total: 8 vcpus-allocated: 1 inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 10 requests: 30 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west requests: 40 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west requests: 30 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west cpu/utilization: 15 requests: 50 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -388,21 +387,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:05.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:05.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -411,21 +410,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:10.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:10.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -434,21 +433,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:15.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:15.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -457,21 +456,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:20.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:20.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -480,21 +479,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:25.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:25.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -503,21 +502,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:30.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:30.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -526,21 +525,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:35.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:35.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -549,21 +548,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:40.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:40.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -572,21 +571,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:45.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:45.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -595,21 +594,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:50.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:50.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -618,21 +617,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:55.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:55.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -641,21 +640,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:01:00.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:01:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 - cpu/utilization: '*' + cpu/utilization: "*" requests: 0.5 cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -664,18 +663,18 @@ tree: device/expected-lifespan: 1576800 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.0000020256215119228817 - carbon-operational: '*' - carbon: '*' - sci: '*' + carbon-operational: "*" + carbon: "*" + sci: "*" aggregated: - carbon: '*' + carbon: "*" child-2: pipeline: observe: @@ -694,7 +693,7 @@ tree: - operational-carbon - sum-carbon - time-sync - - sci + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -704,34 +703,34 @@ tree: vcpus-total: 8 vcpus-allocated: 1 inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 cpu/utilization: 30 cloud/instance-type: A1 cloud/region: uk-west requests: 30 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 28 cloud/instance-type: A1 cloud/region: uk-west requests: 40 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 40 cloud/instance-type: A1 cloud/region: uk-west requests: 50 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cpu/utilization: 33 cloud/instance-type: A1 cloud/region: uk-west requests: 60 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -742,19 +741,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:05.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:05.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -765,19 +764,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:10.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:10.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -788,19 +787,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:15.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:15.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -811,19 +810,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:20.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:20.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -834,19 +833,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:25.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:25.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -857,19 +856,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:30.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:30.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -880,19 +879,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:35.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:35.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -903,19 +902,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:40.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:40.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -926,19 +925,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:45.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:45.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -949,19 +948,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:50.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:50.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -972,19 +971,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:55.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:55.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -995,19 +994,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:01:00.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:01:00.000Z" duration: 1 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 0.5 @@ -1018,57 +1017,57 @@ tree: device/expected-lifespan: 1576800 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" aggregated: - carbon: '*' + carbon: "*" outputs: - - carbon: '*' - timestamp: '2023-12-12T00:00:00.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:00.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:05.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:05.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:10.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:10.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:15.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:15.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:20.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:20.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:25.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:25.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:30.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:30.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:35.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:35.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:40.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:40.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:45.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:45.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:50.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:50.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:55.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:55.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:01:00.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:01:00.000Z" duration: 1 aggregated: - carbon: '*' + carbon: "*" diff --git a/manifests/examples/pipelines/pipeline-with-aggregate.yml b/manifests/examples/pipelines/pipeline-with-aggregate.yml index dd4dffa1c..184820228 100644 --- a/manifests/examples/pipelines/pipeline-with-aggregate.yml +++ b/manifests/examples/pipelines/pipeline-with-aggregate.yml @@ -3,8 +3,7 @@ description: a full pipeline with the aggregate feature enabled tags: aggregation: metrics: - "carbon": - method: sum + - carbon type: "both" initialize: plugins: @@ -145,6 +144,12 @@ initialize: - carbon-operational - carbon-embodied output-parameter: carbon + parameter-metadata: + outputs: + carbon: + unit: gCO2eq + description: product of carbon + aggregation-method: sum "time-sync": method: TimeSync path: "builtin" diff --git a/manifests/examples/pipelines/pipeline-with-mocks.yml b/manifests/examples/pipelines/pipeline-with-mocks.yml index 58933b171..fc6f81cb0 100644 --- a/manifests/examples/pipelines/pipeline-with-mocks.yml +++ b/manifests/examples/pipelines/pipeline-with-mocks.yml @@ -3,8 +3,7 @@ description: a full pipeline seeded with data from mock-observations feature tags: aggregation: metrics: - "carbon": - method: sum + - carbon type: "both" initialize: plugins: diff --git a/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml b/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml index 313834f76..92f85c0d7 100644 --- a/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml +++ b/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml @@ -5,8 +5,7 @@ description: >- tags: null aggregation: metrics: - "dummy-param": - method: sum + - dummy-param type: both initialize: plugins: diff --git a/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml b/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml index 11e7f5865..aef8ef6c5 100644 --- a/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml +++ b/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml @@ -2,8 +2,7 @@ name: Aggregation description: Fails with invalid metric. aggregation: metrics: - "test": - method: sum + - test type: both initialize: plugins: diff --git a/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml b/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml index 0e308d509..2dfc9fe80 100644 --- a/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml +++ b/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml @@ -2,8 +2,7 @@ name: Aggregation description: Fails with missing metric in inputs. aggregation: metrics: - "cpu/utilization": - method: sum + - cpu/utilization type: both initialize: plugins: diff --git a/manifests/outputs/features/aggregate-horizontal.yaml b/manifests/outputs/features/aggregate-horizontal.yaml index 0fef8d756..d595d5bdb 100644 --- a/manifests/outputs/features/aggregate-horizontal.yaml +++ b/manifests/outputs/features/aggregate-horizontal.yaml @@ -2,8 +2,7 @@ name: Aggregation description: Apply `horizontal` aggregation aggregation: metrics: - "cpu/utilization": - method: avg + - cpu/utilization type: horizontal initialize: plugins: @@ -18,6 +17,12 @@ initialize: output: - cpu-cores-utilized - vcpus-allocated + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node diff --git a/manifests/outputs/features/aggregate-vertical.yaml b/manifests/outputs/features/aggregate-vertical.yaml index dfd604c73..9213522af 100644 --- a/manifests/outputs/features/aggregate-vertical.yaml +++ b/manifests/outputs/features/aggregate-vertical.yaml @@ -2,8 +2,7 @@ name: Aggregation description: Apply `vertical` aggregation aggregation: metrics: - "cpu/utilization": - method: avg + - cpu/utilization type: vertical initialize: plugins: @@ -18,6 +17,12 @@ initialize: output: - cpu-cores-utilized - vcpus-allocated + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node diff --git a/manifests/outputs/features/aggregate.yaml b/manifests/outputs/features/aggregate.yaml index 79736b9c0..cb9ed29dc 100644 --- a/manifests/outputs/features/aggregate.yaml +++ b/manifests/outputs/features/aggregate.yaml @@ -2,8 +2,7 @@ name: Aggregation description: Apply both `horizontal` and `vertical` aggregations aggregation: metrics: - "cpu/utilization": - method: avg + - cpu/utilization type: both initialize: plugins: @@ -18,6 +17,12 @@ initialize: output: - cpu-cores-utilized - vcpus-allocated + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node diff --git a/manifests/outputs/pipelines/nesting.yaml b/manifests/outputs/pipelines/nesting.yaml index 5e26ae900..792f41243 100644 --- a/manifests/outputs/pipelines/nesting.yaml +++ b/manifests/outputs/pipelines/nesting.yaml @@ -6,8 +6,7 @@ tags: category: on-premise aggregation: metrics: - carbon: - method: sum + - carbon type: both initialize: plugins: @@ -21,7 +20,7 @@ initialize: - 10 - 50 - 100 - 'y': + "y": - 0.12 - 0.32 - 0.75 @@ -153,12 +152,27 @@ initialize: - carbon-operational - carbon-embodied output-parameter: carbon + parameter-metadata: + inputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + carbon-embodied: + description: Embodied carbon footprint + unit: gCO2eq + aggregation-method: sum + outputs: + carbon: + description: Total carbon footprint + unit: gCO2eq + aggregation-method: sum time-sync: path: builtin method: TimeSync global-config: - start-time: '2023-12-12T00:00:00.000Z' - end-time: '2023-12-12T00:01:00.000Z' + start-time: "2023-12-12T00:00:00.000Z" + end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true parameter-metadata: @@ -196,20 +210,20 @@ execution: environment: if-version: 0.5.0 os: macOS - os-version: '14.5' + os-version: "14.5" node-version: 18.14.2 date-time: 2024-07-31T13:17:29.944Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.16' - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.16" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -256,28 +270,28 @@ tree: - time-sync - sci inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 50 network/energy: 0.000001 requests: 50 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 60 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 70 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west @@ -285,7 +299,7 @@ tree: network/energy: 0.000001 requests: 55 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 @@ -309,7 +323,7 @@ tree: carbon-operational: 0.006833333333333334 carbon: 0.0068434614408929475 sci: 0.00006983123919278518 - - timestamp: '2023-12-12T00:00:05.000Z' + - timestamp: "2023-12-12T00:00:05.000Z" duration: 5 cpu/utilization: 13 cloud/instance-type: A1 @@ -333,7 +347,7 @@ tree: carbon-operational: 0.005340277777777777 carbon: 0.005350405885337391 sci: 0.0001028924208718729 - - timestamp: '2023-12-12T00:00:10.000Z' + - timestamp: "2023-12-12T00:00:10.000Z" duration: 5 cpu/utilization: 12 cloud/instance-type: A1 @@ -357,7 +371,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.0052011003297818366 sci: 0.0001544881286073813 - - timestamp: '2023-12-12T00:00:15.000Z' + - timestamp: "2023-12-12T00:00:15.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -381,7 +395,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0005673927632489277 - - timestamp: '2023-12-12T00:00:20.000Z' + - timestamp: "2023-12-12T00:00:20.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -405,7 +419,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0005673927632489277 - - timestamp: '2023-12-12T00:00:25.000Z' + - timestamp: "2023-12-12T00:00:25.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -429,7 +443,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0005673927632489277 - - timestamp: '2023-12-12T00:00:30.000Z' + - timestamp: "2023-12-12T00:00:30.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -453,7 +467,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0005673927632489277 - - timestamp: '2023-12-12T00:00:35.000Z' + - timestamp: "2023-12-12T00:00:35.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -477,7 +491,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0005673927632489277 - - timestamp: '2023-12-12T00:00:40.000Z' + - timestamp: "2023-12-12T00:00:40.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -501,7 +515,7 @@ tree: carbon-operational: 0.0031145833333333334 carbon: 0.003120660197869102 sci: 0.0005673927632489276 - - timestamp: '2023-12-12T00:00:45.000Z' + - timestamp: "2023-12-12T00:00:45.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -525,7 +539,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:50.000Z' + - timestamp: "2023-12-12T00:00:50.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -549,7 +563,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:55.000Z' + - timestamp: "2023-12-12T00:00:55.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -573,7 +587,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:01:00.000Z' + - timestamp: "2023-12-12T00:01:00.000Z" duration: 1 cloud/instance-type: A1 cloud/region: uk-west @@ -622,28 +636,28 @@ tree: - time-sync - sci inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 50 network/energy: 0.000001 requests: 10 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 90 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 30 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west @@ -651,7 +665,7 @@ tree: network/energy: 0.000001 requests: 22 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 @@ -675,7 +689,7 @@ tree: carbon-operational: 0.006833333333333334 carbon: 0.0068434614408929475 sci: 0.0000834568468401579 - - timestamp: '2023-12-12T00:00:05.000Z' + - timestamp: "2023-12-12T00:00:05.000Z" duration: 5 cpu/utilization: 13 cloud/instance-type: A1 @@ -699,7 +713,7 @@ tree: carbon-operational: 0.005340277777777777 carbon: 0.005350405885337391 sci: 0.00015224732194049487 - - timestamp: '2023-12-12T00:00:10.000Z' + - timestamp: "2023-12-12T00:00:10.000Z" duration: 5 cpu/utilization: 12 cloud/instance-type: A1 @@ -723,7 +737,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.0052011003297818366 sci: 0.000363108733129716 - - timestamp: '2023-12-12T00:00:15.000Z' + - timestamp: "2023-12-12T00:00:15.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -747,7 +761,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0014184819081223194 - - timestamp: '2023-12-12T00:00:20.000Z' + - timestamp: "2023-12-12T00:00:20.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -771,7 +785,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0014184819081223194 - - timestamp: '2023-12-12T00:00:25.000Z' + - timestamp: "2023-12-12T00:00:25.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -795,7 +809,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0014184819081223194 - - timestamp: '2023-12-12T00:00:30.000Z' + - timestamp: "2023-12-12T00:00:30.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -819,7 +833,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0014184819081223194 - - timestamp: '2023-12-12T00:00:35.000Z' + - timestamp: "2023-12-12T00:00:35.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -843,7 +857,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0014184819081223194 - - timestamp: '2023-12-12T00:00:40.000Z' + - timestamp: "2023-12-12T00:00:40.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -867,7 +881,7 @@ tree: carbon-operational: 0.0031145833333333334 carbon: 0.003120660197869102 sci: 0.0014184819081223194 - - timestamp: '2023-12-12T00:00:45.000Z' + - timestamp: "2023-12-12T00:00:45.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -891,7 +905,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:50.000Z' + - timestamp: "2023-12-12T00:00:50.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -915,7 +929,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:55.000Z' + - timestamp: "2023-12-12T00:00:55.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -939,7 +953,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:01:00.000Z' + - timestamp: "2023-12-12T00:01:00.000Z" duration: 1 cloud/instance-type: A1 cloud/region: uk-west @@ -990,28 +1004,28 @@ tree: - time-sync - sci inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 50 network/energy: 0.000001 requests: 50 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 65 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 80 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west @@ -1019,7 +1033,7 @@ tree: network/energy: 0.000001 requests: 40 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 @@ -1043,7 +1057,7 @@ tree: carbon-operational: 0.006833333333333334 carbon: 0.0068434614408929475 sci: 0.00006709275922444067 - - timestamp: '2023-12-12T00:00:05.000Z' + - timestamp: "2023-12-12T00:00:05.000Z" duration: 5 cpu/utilization: 13 cloud/instance-type: A1 @@ -1067,7 +1081,7 @@ tree: carbon-operational: 0.005340277777777777 carbon: 0.005350405885337391 sci: 0.0000911261343001502 - - timestamp: '2023-12-12T00:00:10.000Z' + - timestamp: "2023-12-12T00:00:10.000Z" duration: 5 cpu/utilization: 12 cloud/instance-type: A1 @@ -1091,7 +1105,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.0052011003297818366 sci: 0.00014075142645028166 - - timestamp: '2023-12-12T00:00:15.000Z' + - timestamp: "2023-12-12T00:00:15.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1115,7 +1129,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:20.000Z' + - timestamp: "2023-12-12T00:00:20.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1139,7 +1153,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:25.000Z' + - timestamp: "2023-12-12T00:00:25.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1163,7 +1177,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:30.000Z' + - timestamp: "2023-12-12T00:00:30.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1187,7 +1201,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:35.000Z' + - timestamp: "2023-12-12T00:00:35.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1211,7 +1225,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:40.000Z' + - timestamp: "2023-12-12T00:00:40.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1235,7 +1249,7 @@ tree: carbon-operational: 0.0031145833333333334 carbon: 0.003120660197869102 sci: 0.0007801650494672755 - - timestamp: '2023-12-12T00:00:45.000Z' + - timestamp: "2023-12-12T00:00:45.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1259,7 +1273,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:50.000Z' + - timestamp: "2023-12-12T00:00:50.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1283,7 +1297,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:55.000Z' + - timestamp: "2023-12-12T00:00:55.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1307,7 +1321,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:01:00.000Z' + - timestamp: "2023-12-12T00:01:00.000Z" duration: 1 cloud/instance-type: A1 cloud/region: uk-west @@ -1356,28 +1370,28 @@ tree: - time-sync - sci inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 50 network/energy: 0.000001 requests: 50 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 50 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 60 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west @@ -1385,7 +1399,7 @@ tree: network/energy: 0.000001 requests: 40 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 @@ -1409,7 +1423,7 @@ tree: carbon-operational: 0.006833333333333334 carbon: 0.0068434614408929475 sci: 0.00007603846045436609 - - timestamp: '2023-12-12T00:00:05.000Z' + - timestamp: "2023-12-12T00:00:05.000Z" duration: 5 cpu/utilization: 13 cloud/instance-type: A1 @@ -1433,7 +1447,7 @@ tree: carbon-operational: 0.005340277777777777 carbon: 0.005350405885337391 sci: 0.00012081561676568304 - - timestamp: '2023-12-12T00:00:10.000Z' + - timestamp: "2023-12-12T00:00:10.000Z" duration: 5 cpu/utilization: 12 cloud/instance-type: A1 @@ -1457,7 +1471,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.0052011003297818366 sci: 0.0001832602465191587 - - timestamp: '2023-12-12T00:00:15.000Z' + - timestamp: "2023-12-12T00:00:15.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1481,7 +1495,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:20.000Z' + - timestamp: "2023-12-12T00:00:20.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1505,7 +1519,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:25.000Z' + - timestamp: "2023-12-12T00:00:25.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1529,7 +1543,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:30.000Z' + - timestamp: "2023-12-12T00:00:30.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1553,7 +1567,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:35.000Z' + - timestamp: "2023-12-12T00:00:35.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1577,7 +1591,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:40.000Z' + - timestamp: "2023-12-12T00:00:40.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1601,7 +1615,7 @@ tree: carbon-operational: 0.0031145833333333334 carbon: 0.003120660197869102 sci: 0.0007801650494672755 - - timestamp: '2023-12-12T00:00:45.000Z' + - timestamp: "2023-12-12T00:00:45.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1625,7 +1639,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:50.000Z' + - timestamp: "2023-12-12T00:00:50.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1649,7 +1663,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:55.000Z' + - timestamp: "2023-12-12T00:00:55.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1673,7 +1687,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:01:00.000Z' + - timestamp: "2023-12-12T00:01:00.000Z" duration: 1 cloud/instance-type: A1 cloud/region: uk-west @@ -1701,85 +1715,85 @@ tree: carbon: 0.04652112950279046 outputs: - carbon: 0.013686922881785895 - timestamp: '2023-12-12T00:00:00.000Z' + timestamp: "2023-12-12T00:00:00.000Z" duration: 5 - carbon: 0.010700811770674782 - timestamp: '2023-12-12T00:00:05.000Z' + timestamp: "2023-12-12T00:00:05.000Z" duration: 5 - carbon: 0.010402200659563673 - timestamp: '2023-12-12T00:00:10.000Z' + timestamp: "2023-12-12T00:00:10.000Z" duration: 5 - carbon: 0.010402200659563675 - timestamp: '2023-12-12T00:00:15.000Z' + timestamp: "2023-12-12T00:00:15.000Z" duration: 5 - carbon: 0.010402200659563675 - timestamp: '2023-12-12T00:00:20.000Z' + timestamp: "2023-12-12T00:00:20.000Z" duration: 5 - carbon: 0.010402200659563675 - timestamp: '2023-12-12T00:00:25.000Z' + timestamp: "2023-12-12T00:00:25.000Z" duration: 5 - carbon: 0.010402200659563675 - timestamp: '2023-12-12T00:00:30.000Z' + timestamp: "2023-12-12T00:00:30.000Z" duration: 5 - carbon: 0.010402200659563675 - timestamp: '2023-12-12T00:00:35.000Z' + timestamp: "2023-12-12T00:00:35.000Z" duration: 5 - carbon: 0.006241320395738204 - timestamp: '2023-12-12T00:00:40.000Z' + timestamp: "2023-12-12T00:00:40.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:00:45.000Z' + timestamp: "2023-12-12T00:00:45.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:00:50.000Z' + timestamp: "2023-12-12T00:00:50.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:00:55.000Z' + timestamp: "2023-12-12T00:00:55.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:01:00.000Z' + timestamp: "2023-12-12T00:01:00.000Z" duration: 1 aggregated: carbon: 0.09304225900558093 outputs: - carbon: 0.02737384576357179 - timestamp: '2023-12-12T00:00:00.000Z' + timestamp: "2023-12-12T00:00:00.000Z" duration: 5 - carbon: 0.021401623541349564 - timestamp: '2023-12-12T00:00:05.000Z' + timestamp: "2023-12-12T00:00:05.000Z" duration: 5 - carbon: 0.020804401319127346 - timestamp: '2023-12-12T00:00:10.000Z' + timestamp: "2023-12-12T00:00:10.000Z" duration: 5 - carbon: 0.02080440131912735 - timestamp: '2023-12-12T00:00:15.000Z' + timestamp: "2023-12-12T00:00:15.000Z" duration: 5 - carbon: 0.02080440131912735 - timestamp: '2023-12-12T00:00:20.000Z' + timestamp: "2023-12-12T00:00:20.000Z" duration: 5 - carbon: 0.02080440131912735 - timestamp: '2023-12-12T00:00:25.000Z' + timestamp: "2023-12-12T00:00:25.000Z" duration: 5 - carbon: 0.02080440131912735 - timestamp: '2023-12-12T00:00:30.000Z' + timestamp: "2023-12-12T00:00:30.000Z" duration: 5 - carbon: 0.02080440131912735 - timestamp: '2023-12-12T00:00:35.000Z' + timestamp: "2023-12-12T00:00:35.000Z" duration: 5 - carbon: 0.012482640791476408 - timestamp: '2023-12-12T00:00:40.000Z' + timestamp: "2023-12-12T00:00:40.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:00:45.000Z' + timestamp: "2023-12-12T00:00:45.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:00:50.000Z' + timestamp: "2023-12-12T00:00:50.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:00:55.000Z' + timestamp: "2023-12-12T00:00:55.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:01:00.000Z' + timestamp: "2023-12-12T00:01:00.000Z" duration: 1 aggregated: carbon: 0.18608451801116185 From 45821add239ffafc626b66290a85395bc6d30088 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:18:53 +0400 Subject: [PATCH 19/26] fix(util): update manifest schema --- src/common/util/validations.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/common/util/validations.ts b/src/common/util/validations.ts index 95d9efeff..5cdbec5db 100644 --- a/src/common/util/validations.ts +++ b/src/common/util/validations.ts @@ -3,10 +3,7 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {STRINGS} from '../../if-run/config'; -import { - AGGREGATION_METHODS, - AGGREGATION_TYPES, -} from '../../if-run/types/aggregation'; +import {AGGREGATION_TYPES} from '../../if-run/types/aggregation'; const {ManifestValidationError, InputValidationError} = ERRORS; const {VALIDATING_MANIFEST} = STRINGS; @@ -72,11 +69,7 @@ export const manifestSchema = z.object({ explain: z.record(z.string(), z.any()).optional(), aggregation: z .object({ - metrics: z.record( - z.object({ - method: z.enum(AGGREGATION_METHODS), - }) - ), + metrics: z.array(z.string()), type: z.enum(AGGREGATION_TYPES), }) .optional() From 719a239e1cea887935420046e03616cb84ced525 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:23:52 +0400 Subject: [PATCH 20/26] fix(lib): update aggregation logic without `method` --- src/if-run/lib/aggregate.ts | 21 +++++++++++++-------- src/if-run/lib/initialize.ts | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/if-run/lib/aggregate.ts b/src/if-run/lib/aggregate.ts index 508117bb3..b80d0048a 100644 --- a/src/if-run/lib/aggregate.ts +++ b/src/if-run/lib/aggregate.ts @@ -5,7 +5,7 @@ import {logger} from '../../common/util/logger'; import { AggregationParams, AggregationParamsSure, - AggregationParamsWithoutType, + AggregationMetricsWithMethod, } from '../../common/types/manifest'; import {aggregateInputsIntoOne} from '../util/aggregation-helper'; @@ -39,7 +39,7 @@ const getIthElementsFromChildren = (children: any, i: number) => { * 1. Gets the i'th element from each childrens outputs (treating children as rows and we are after a column of data). * 2. Now we just aggregate over the `ithSliceOfOutputs` the same as we did for the normal outputs. */ -const temporalAggregation = (node: any, metrics: AggregationMetric) => { +const temporalAggregation = (node: any, metrics: AggregationMetric[]) => { const outputs: PluginParams[] = []; const values: any = Object.values(node.children); @@ -63,7 +63,9 @@ const temporalAggregation = (node: any, metrics: AggregationMetric) => { * 5. Now a grouping node has it's own outputs, it can horizotnally aggregate them. */ const aggregateNode = (node: any, aggregationParams: AggregationParamsSure) => { - const metrics = aggregationParams!.metrics; + const metrics: AggregationMetric[] = aggregationParams!.metrics.map( + metric => ({[metric]: 'none'}) + ); const type = aggregationParams!.type; if (node.children) { @@ -108,12 +110,12 @@ export const aggregate = (tree: any, aggregationParams: AggregationParams) => { * Gets or stores aggregation metrics. */ export const storeAggregationMetrics = ( - aggregationParams?: AggregationParamsWithoutType + aggregationMetrics?: AggregationMetricsWithMethod ) => { - if (aggregationParams?.metrics) { + if (aggregationMetrics) { metricManager.metrics = { ...metricManager.metrics, - ...aggregationParams?.metrics, + ...aggregationMetrics, }; } @@ -146,8 +148,11 @@ export const getAggregationMethod = (unitName: string) => { memoizedLog(console.debug, CHECKING_AGGREGATION_METHOD(unitName)); const aggregationMetricsStorage = storeAggregationMetrics(); - if (aggregationMetricsStorage && `${unitName}` in aggregationMetricsStorage) { - return aggregationMetricsStorage[unitName].method; + if ( + aggregationMetricsStorage && + Object.keys(aggregationMetricsStorage).includes(unitName) + ) { + return aggregationMetricsStorage[unitName]; } memoizedLog(logger.warn, UNKNOWN_PARAM(unitName)); diff --git a/src/if-run/lib/initialize.ts b/src/if-run/lib/initialize.ts index f047be117..181e88d15 100644 --- a/src/if-run/lib/initialize.ts +++ b/src/if-run/lib/initialize.ts @@ -118,7 +118,7 @@ export const initialize = async ( Object.keys(parameters).forEach(key => { storeAggregationMetrics({ - metrics: {[key]: {method: parameters[key]['aggregation-method']}}, + [key]: parameters[key]['aggregation-method'], }); }); From 6eeba6583e891e08c35ea9bb44bf5c860ceef024 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:24:55 +0400 Subject: [PATCH 21/26] fix(util): update aggregation logic --- src/if-run/util/aggregation-helper.ts | 29 +++++---------------------- src/if-run/util/helpers.ts | 7 +++---- 2 files changed, 8 insertions(+), 28 deletions(-) diff --git a/src/if-run/util/aggregation-helper.ts b/src/if-run/util/aggregation-helper.ts index 87d3e15fc..10b1a0bdd 100644 --- a/src/if-run/util/aggregation-helper.ts +++ b/src/if-run/util/aggregation-helper.ts @@ -7,40 +7,21 @@ import {AggregationMetric, AggregationResult} from '../types/aggregation'; import {getAggregationMethod} from '../lib/aggregate'; -const {InvalidAggregationMethodError, MissingAggregationParamError} = ERRORS; -const {INVALID_AGGREGATION_METHOD, METRIC_MISSING} = STRINGS; +const {MissingAggregationParamError} = ERRORS; +const {METRIC_MISSING} = STRINGS; const {AGGREGATION_ADDITIONAL_PARAMS} = CONFIG; -/** - * Validates metrics array before applying aggregator. - * If aggregation method is `none`, then throws error. - */ -const checkIfMetricsAreValid = (metrics: AggregationMetric) => { - Object.keys(metrics).forEach(metric => { - const method = metrics[metric].method; - - if (method === 'none') { - throw new InvalidAggregationMethodError( - INVALID_AGGREGATION_METHOD(metric) - ); - } - }); -}; - /** * Aggregates child node level metrics. Validates if metric aggregation type is `none`, then rejects with error. * Appends aggregation additional params to metrics. Otherwise iterates over inputs by aggregating per given `metrics`. */ export const aggregateInputsIntoOne = ( inputs: PluginParams[], - metrics: AggregationMetric, + metrics: AggregationMetric[], isTemporal?: boolean ) => { - checkIfMetricsAreValid(metrics); - const extendedMetrics = [ - ...Object.keys(metrics), - ...AGGREGATION_ADDITIONAL_PARAMS, - ]; + const metricsKeys: string[] = metrics.map(metric => Object.keys(metric)[0]); + const extendedMetrics = [...metricsKeys, ...AGGREGATION_ADDITIONAL_PARAMS]; return inputs.reduce((acc, input, index) => { for (const metric of extendedMetrics) { diff --git a/src/if-run/util/helpers.ts b/src/if-run/util/helpers.ts index da50f22f4..2939022e2 100644 --- a/src/if-run/util/helpers.ts +++ b/src/if-run/util/helpers.ts @@ -62,10 +62,9 @@ export const storeAggregationMethods = ( const {'aggregation-method': aggregationMethod} = parameterMetadata; if (aggregationMethod) { - const metrics = { - [parameterName]: {method: aggregationMethod}, - }; - storeAggregationMetrics({metrics}); + const metrics = {[parameterName]: aggregationMethod}; + + storeAggregationMetrics(metrics); } } ); From 914b8eb8cd1b91f0797d355c9df0aa3f7d92068a Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:26:13 +0400 Subject: [PATCH 22/26] fix(types): update aggregation types --- src/common/types/manifest.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/common/types/manifest.ts b/src/common/types/manifest.ts index c28fe7194..1efab523d 100644 --- a/src/common/types/manifest.ts +++ b/src/common/types/manifest.ts @@ -1,5 +1,7 @@ import {z} from 'zod'; +import {AggregationMethodTypes} from '../../if-run/types/aggregation'; + import {manifestSchema} from '../util/validations'; export type Manifest = z.infer; @@ -9,10 +11,9 @@ export type GlobalPlugins = Manifest['initialize']['plugins']; export type PluginOptions = GlobalPlugins[string]; export type AggregationParams = Manifest['aggregation']; -export type AggregationParamsWithoutType = Omit< - Exclude, - 'type' ->; +export type AggregationMetricsWithMethod = { + [key: string]: AggregationMethodTypes; +}; export type AggregationParamsSure = Extract; From c69949718f867c005371de1d52132be46d32a550 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:27:05 +0400 Subject: [PATCH 23/26] fix(types): update aggregation types --- src/if-run/types/aggregation.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/if-run/types/aggregation.ts b/src/if-run/types/aggregation.ts index a7e895da4..c3b143a1f 100644 --- a/src/if-run/types/aggregation.ts +++ b/src/if-run/types/aggregation.ts @@ -3,7 +3,5 @@ export type AggregationResult = Record; export const AGGREGATION_TYPES = ['horizontal', 'vertical', 'both'] as const; export const AGGREGATION_METHODS = ['sum', 'avg', 'none'] as const; -export type AggregationMetric = Record< - string, - {method: 'sum' | 'avg' | 'none'} ->; +export type AggregationMethodTypes = 'sum' | 'avg' | 'none'; +export type AggregationMetric = Record; From be28ccdadd3e7304e9add7416f48a4e6da87be4e Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:28:49 +0400 Subject: [PATCH 24/26] fix(src): fix adding aggregation metric into storage --- src/if-run/index.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/if-run/index.ts b/src/if-run/index.ts index 9b8d86f6c..6d9f61a78 100644 --- a/src/if-run/index.ts +++ b/src/if-run/index.ts @@ -12,6 +12,8 @@ import {compute} from './lib/compute'; import {exhaust} from './lib/exhaust'; import {explain} from './lib/explain'; +import {AGGREGATION_METHODS} from './types/aggregation'; + import {parseIfRunProcessArgs} from './util/args'; import {andHandle} from './util/helpers'; @@ -43,7 +45,13 @@ const impactEngine = async () => { const {tree, ...context} = validateManifest(envManifest); if (context.aggregation) { - storeAggregationMetrics({metrics: context.aggregation?.metrics}); + const convertMetrics = context.aggregation?.metrics.map( + (metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + }) + ); + + storeAggregationMetrics(...convertMetrics); } const pluginStorage = await initialize(context); From 1612a6293e843b2fd7e8bae461f91c8a565c0e50 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:29:44 +0400 Subject: [PATCH 25/26] test(src): update tests accroding to changes --- .../if-run/builtins/time-sync.test.ts | 23 +++++--- src/__tests__/if-run/lib/aggregate.test.ts | 16 ++--- .../if-run/util/aggregation-helper.test.ts | 59 +++++++------------ src/__tests__/if-run/util/helpers.test.ts | 12 +--- 4 files changed, 48 insertions(+), 62 deletions(-) diff --git a/src/__tests__/if-run/builtins/time-sync.test.ts b/src/__tests__/if-run/builtins/time-sync.test.ts index 314ed06d4..38621e12f 100644 --- a/src/__tests__/if-run/builtins/time-sync.test.ts +++ b/src/__tests__/if-run/builtins/time-sync.test.ts @@ -7,6 +7,7 @@ import {storeAggregationMetrics} from '../../../if-run/lib/aggregate'; import {TimeSync} from '../../../if-run/builtins/time-sync'; import {STRINGS} from '../../../if-run/config'; +import {AGGREGATION_METHODS} from '../../../if-run/types/aggregation'; Settings.defaultZone = 'utc'; const { @@ -56,16 +57,18 @@ jest.mock('luxon', () => { describe('builtins/time-sync:', () => { beforeAll(() => { const metricStorage: AggregationParams = { - metrics: { - carbon: {method: 'sum'}, - 'cpu/utilization': {method: 'sum'}, - 'time-reserved': {method: 'avg'}, - 'resources-total': {method: 'none'}, - }, + metrics: [ + 'carbon', + 'cpu/utilization', + 'time-reserved', + 'resources-total', + ], type: 'horizontal', }; - - storeAggregationMetrics(metricStorage); + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); + storeAggregationMetrics(...convertedMetrics); }); describe('time-sync: ', () => { @@ -472,6 +475,7 @@ describe('execute(): ', () => { interval: 1, 'allow-padding': true, }; + storeAggregationMetrics({carbon: 'sum'}); const timeModel = TimeSync(basicConfig, parametersMetadata); @@ -586,6 +590,8 @@ describe('execute(): ', () => { interval: 5, 'allow-padding': true, }; + storeAggregationMetrics({'time-reserved': 'avg'}); + storeAggregationMetrics({'resources-total': 'sum'}); const timeModel = TimeSync(basicConfig, parametersMetadata); @@ -659,6 +665,7 @@ describe('execute(): ', () => { interval: 5, 'allow-padding': true, }; + storeAggregationMetrics({'resources-total': 'none'}); const timeModel = TimeSync(basicConfig, parametersMetadata); diff --git a/src/__tests__/if-run/lib/aggregate.test.ts b/src/__tests__/if-run/lib/aggregate.test.ts index 36bbc0d1f..00d9c6d0e 100644 --- a/src/__tests__/if-run/lib/aggregate.test.ts +++ b/src/__tests__/if-run/lib/aggregate.test.ts @@ -6,17 +6,19 @@ import { aggregate, storeAggregationMetrics, } from '../../../if-run/lib/aggregate'; +import {AGGREGATION_METHODS} from '../../../if-run/types/aggregation'; describe('lib/aggregate: ', () => { beforeAll(() => { const metricStorage: AggregationParams = { - metrics: { - carbon: {method: 'sum'}, - }, + metrics: ['carbon'], type: 'horizontal', }; + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); - storeAggregationMetrics(metricStorage); + storeAggregationMetrics(...convertedMetrics); }); describe('aggregate(): ', () => { @@ -60,7 +62,7 @@ describe('lib/aggregate: ', () => { }; const aggregatedTree = aggregate(tree, { - metrics: {carbon: {method: 'sum'}}, + metrics: ['carbon'], type: 'horizontal', }); const expectedAggregated = { @@ -108,7 +110,7 @@ describe('lib/aggregate: ', () => { }; const aggregatedTree = aggregate(tree, { - metrics: {carbon: {method: 'sum'}}, + metrics: ['carbon'], type: 'vertical', }); const expectedOutputs = [ @@ -169,7 +171,7 @@ describe('lib/aggregate: ', () => { }; const aggregatedTree = aggregate(tree, { - metrics: {carbon: {method: 'sum'}}, + metrics: ['carbon'], type: 'both', }); diff --git a/src/__tests__/if-run/util/aggregation-helper.test.ts b/src/__tests__/if-run/util/aggregation-helper.test.ts index df8a74d9b..f83536f39 100644 --- a/src/__tests__/if-run/util/aggregation-helper.test.ts +++ b/src/__tests__/if-run/util/aggregation-helper.test.ts @@ -4,52 +4,33 @@ import {PluginParams} from '@grnsft/if-core/types'; import {AggregationParams} from '../../../common/types/manifest'; import {aggregateInputsIntoOne} from '../../../if-run/util/aggregation-helper'; -import {AggregationMetric} from '../../../if-run/types/aggregation'; +import { + AGGREGATION_METHODS, + AggregationMetric, +} from '../../../if-run/types/aggregation'; import {storeAggregationMetrics} from '../../../if-run/lib/aggregate'; import {STRINGS} from '../../../if-run/config'; -const {InvalidAggregationMethodError, MissingAggregationParamError} = ERRORS; -const {INVALID_AGGREGATION_METHOD, METRIC_MISSING} = STRINGS; +const {MissingAggregationParamError} = ERRORS; +const {METRIC_MISSING} = STRINGS; describe('util/aggregation-helper: ', () => { beforeAll(() => { const metricStorage: AggregationParams = { - metrics: { - carbon: {method: 'sum'}, - 'cpu/number-cores': {method: 'none'}, - 'cpu/utilization': {method: 'sum'}, - }, + metrics: ['carbon', 'cpu/number-cores', 'cpu/utilization'], type: 'horizontal', }; - - storeAggregationMetrics(metricStorage); + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); + storeAggregationMetrics(...convertedMetrics); }); describe('aggregateInputsIntoOne(): ', () => { - it('throws error if aggregation method is none.', () => { - const inputs: PluginParams[] = []; - const metrics: AggregationMetric = {'cpu/number-cores': {method: 'none'}}; - const isTemporal = false; - - expect.assertions(2); - - try { - aggregateInputsIntoOne(inputs, metrics, isTemporal); - } catch (error) { - expect(error).toBeInstanceOf(InvalidAggregationMethodError); - - if (error instanceof InvalidAggregationMethodError) { - expect(error.message).toEqual( - INVALID_AGGREGATION_METHOD('cpu/number-cores') - ); - } - } - }); - it('throws error if aggregation criteria is not found in input.', () => { const inputs: PluginParams[] = [{timestamp: '', duration: 10}]; - const metrics: AggregationMetric = {'cpu/utilization': {method: 'sum'}}; + const metrics: AggregationMetric[] = [{'cpu/utilization': 'sum'}]; const isTemporal = false; expect.assertions(2); @@ -70,7 +51,7 @@ describe('util/aggregation-helper: ', () => { {timestamp: '', duration: 10, carbon: 10}, {timestamp: '', duration: 10, carbon: 20}, ]; - const metrics: AggregationMetric = {carbon: {method: 'sum'}}; + const metrics: AggregationMetric[] = [{carbon: 'sum'}]; const isTemporal = true; const expectedValue = { @@ -87,7 +68,7 @@ describe('util/aggregation-helper: ', () => { {timestamp: '', duration: 10, carbon: 10}, {timestamp: '', duration: 10, carbon: 20}, ]; - const metrics: AggregationMetric = {carbon: {method: 'sum'}}; + const metrics: AggregationMetric[] = [{carbon: 'sum'}]; const isTemporal = false; const expectedValue = { @@ -99,18 +80,20 @@ describe('util/aggregation-helper: ', () => { it('calculates average of metrics.', () => { const metricStorage: AggregationParams = { - metrics: { - 'cpu/utilization': {method: 'avg'}, - }, + metrics: ['cpu/utilization'], type: 'horizontal', }; + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); + storeAggregationMetrics(...convertedMetrics); + storeAggregationMetrics({'cpu/utilization': 'avg'}); - storeAggregationMetrics(metricStorage); const inputs: PluginParams[] = [ {timestamp: '', duration: 10, 'cpu/utilization': 10}, {timestamp: '', duration: 10, 'cpu/utilization': 90}, ]; - const metrics: AggregationMetric = {'cpu/utilization': {method: 'avg'}}; + const metrics: AggregationMetric[] = [{'cpu/utilization': 'avg'}]; const isTemporal = false; const expectedValue = { diff --git a/src/__tests__/if-run/util/helpers.test.ts b/src/__tests__/if-run/util/helpers.test.ts index c694c0a9a..796f6b201 100644 --- a/src/__tests__/if-run/util/helpers.test.ts +++ b/src/__tests__/if-run/util/helpers.test.ts @@ -242,19 +242,13 @@ describe('if-run/util/helpers: ', () => { expect(storeAggregationMetrics).toHaveBeenCalledTimes(3); expect(storeAggregationMetrics).toHaveBeenNthCalledWith(1, { - metrics: { - carbon: {method: 'sum'}, - }, + carbon: 'sum', }); expect(storeAggregationMetrics).toHaveBeenNthCalledWith(2, { - metrics: { - cpu: {method: 'avg'}, - }, + cpu: 'avg', }); expect(storeAggregationMetrics).toHaveBeenNthCalledWith(3, { - metrics: { - carbon: {method: 'none'}, - }, + carbon: 'none', }); }); From 27fce36762d13ecbaafab9233fa91f45ed89f593 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:32:39 +0400 Subject: [PATCH 26/26] fix(doc): update aggregation part in the doc --- Refactor-migration-guide.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Refactor-migration-guide.md b/Refactor-migration-guide.md index e29a80d0e..95283c841 100644 --- a/Refactor-migration-guide.md +++ b/Refactor-migration-guide.md @@ -199,9 +199,7 @@ The aggregate plugin aggregates data in two ways: first it condenses individual This is a builtin feature of IF, meaning it does not have to be initialized as a plugin. Instead, you just have to include a short config block in the top of the manifest file. There are two pieces of information required: -- `metrics`: which metrics do you want to aggregate? Every metric you provide here must exist in the output array. - - - `method`: the aggregation method for the specied metric +- `metrics`: which metrics do you want to aggregate? Every metric you provide here must exist in the output array and be described in the `parameter-metadata` of the plugin. - `type`: the options are `horizontal`, `vertical` or both. Horizontal aggregation is the type that condenses each time series into a single summary value. Vertical aggregation is aggregated across components. @@ -210,8 +208,7 @@ Here's what the config block should look like: ```yaml aggregation: metrics: - 'carbon': - method: 'sum' + - carbon type: 'both' ```