diff --git a/.github/workflows/dispatch-integration-test.yml b/.github/workflows/dispatch-integration-test.yml new file mode 100644 index 000000000..6fe4904d7 --- /dev/null +++ b/.github/workflows/dispatch-integration-test.yml @@ -0,0 +1,23 @@ +name: Run if-check on requested branch + +on: + workflow_dispatch: + +jobs: + run: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 18 + cache: 'npm' + + - name: Install dependencies + run: npm install + + - name: Run integration tests + run: npm run if-check -- -d manifests/outputs diff --git a/.github/workflows/release-commit.yml b/.github/workflows/release-commit.yml index efcf8726e..8ffa665f7 100644 --- a/.github/workflows/release-commit.yml +++ b/.github/workflows/release-commit.yml @@ -8,7 +8,7 @@ env: PRE_RELEASE: ${{ github.event.release.prerelease}} jobs: - release-commit-pr: + unit-tests: runs-on: ubuntu-latest steps: - name: Checkout code @@ -28,9 +28,45 @@ jobs: - name: Run unit tests run: npm test + integration-tests: + runs-on: ubuntu-latest + needs: unit-tests + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 18 + registry-url: https://registry.npmjs.org/ + + - name: Install dependencies + run: npm ci + - name: Run integration tests run: npm run if-check -- -d manifests/outputs + publish: + runs-on: ubuntu-latest + needs: integration-tests + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 18 + registry-url: https://registry.npmjs.org/ + + - name: Install dependencies + run: npm ci + - name: Initialize git user email run: git config --global user.email "${{ vars.RELEASE_USER_EMAIL }}" diff --git a/Refactor-migration-guide.md b/Refactor-migration-guide.md index f8da3d8b8..95283c841 100644 --- a/Refactor-migration-guide.md +++ b/Refactor-migration-guide.md @@ -125,24 +125,23 @@ There have also been some changes to the structure of manifest files. Some of th - **Node level config** - We have also introduced the concept of node-level config. This is designed for pluin configuration that might vary between components in the tree. For example, for each child in the tree you might wish to use the `groupby` plugin to group the outputs according to a different set of keys. + We have also introduced the concept of node-level config. This is designed for pluin configuration that might vary between components in the tree. For example, for each child in the tree you might wish to use the `regroup` feature to group the outputs according to a different set of keys. ```yaml tree: children: child-1: pipeline: - - teads-curve - - sci-e - - sci-embodied - - sci-o - - time-sync - - sci - config: - group-by: - group: - - region - - cloud/instance-type + compute: + - teads-curve + - sci-e + - sci-embodied + - sci-o + - time-sync + - sci + regroup: + - region + - cloud/instance-type ``` - **Defaults** @@ -156,12 +155,13 @@ There have also been some changes to the structure of manifest files. Some of th children: child-1: pipeline: - - teads-curve - - sci-e - - sci-embodied - - sci-o - - time-sync - - sci + compute: + - teads-curve + - sci-e + - sci-embodied + - sci-o + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -199,9 +199,7 @@ The aggregate plugin aggregates data in two ways: first it condenses individual This is a builtin feature of IF, meaning it does not have to be initialized as a plugin. Instead, you just have to include a short config block in the top of the manifest file. There are two pieces of information required: -- `metrics`: which metrics do you want to aggregate? Every metric you provide here must exist in the output array. - - - `method`: the aggregation method for the specied metric +- `metrics`: which metrics do you want to aggregate? Every metric you provide here must exist in the output array and be described in the `parameter-metadata` of the plugin. - `type`: the options are `horizontal`, `vertical` or both. Horizontal aggregation is the type that condenses each time series into a single summary value. Vertical aggregation is aggregated across components. @@ -210,45 +208,10 @@ Here's what the config block should look like: ```yaml aggregation: metrics: - 'carbon': - method: 'sum' + - carbon type: 'both' ``` -### Groupby - -Groupby allows you to regroup your outputs according to keys you define. For example, maybe you want to group your outputs by region (show me all the outputs for applications run in `uk-south` etc). Groupby _is_ a plugin that needs to be initialized in the manifest. - -You can initialize the plugin as follows: - -```yaml -initialize: - plugins: - 'group-by': - path: builtin - method: GroupBy -``` - -Then you configure groupby for each component in the node level config. In the following example we will regroup the outputs by the `region`: - -```yaml -tree: - children: - child-1: - pipeline: - - teads-curve - - sci-e - - sci-embodied - - sci-o - - time-sync - - group-by - - sci - config: - group-by: - group: - - region -``` - ### Exhaust We have introduced `exhaust` as an IF feature. This is a wrapper around export plugins and it allows community contributors to create plugins for exporting to different formats. diff --git a/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml b/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml index 938c126aa..5d497b948 100644 --- a/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml +++ b/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml @@ -14,7 +14,8 @@ tree: children: child: pipeline: - - coefficient + compute: + - coefficient config: sum: inputs: diff --git a/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml b/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml index c02f42cf9..1ef932df0 100644 --- a/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml +++ b/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml @@ -14,7 +14,8 @@ tree: children: child: pipeline: - - coefficient + compute: + - coefficient config: sum: inputs: diff --git a/manifests/examples/builtins/coefficient/success.yml b/manifests/examples/builtins/coefficient/success.yml index bd7d2cc94..69dd764eb 100644 --- a/manifests/examples/builtins/coefficient/success.yml +++ b/manifests/examples/builtins/coefficient/success.yml @@ -14,7 +14,8 @@ tree: children: child: pipeline: - - coefficient + compute: + - coefficient config: sum: inputs: diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml index befa110a6..21f9769b4 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml index 1cd9b609a..c9fbb8296 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml index a01156154..62de6150d 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml b/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml index 52bda60b6..84e10112b 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml index 68d3bc917..5b4e9583b 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml index 64354be35..71417932d 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml b/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml index c30e11f9c..4c2767a3a 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/success.yml b/manifests/examples/builtins/csv-lookup/region-metadata/success.yml index 64354be35..71417932d 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/success.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/success.yml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml index 52d2b660c..991bdbb8b 100644 --- a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml +++ b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml @@ -15,7 +15,8 @@ tree: children: child: pipeline: - - tdp-finder + compute: + - tdp-finder config: inputs: - timestamp: 2023-07-06T00:00 diff --git a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml index f86767455..3d433d6b2 100644 --- a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml +++ b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml @@ -15,7 +15,8 @@ tree: children: child: pipeline: - - tdp-finder + compute: + - tdp-finder config: inputs: - timestamp: 2023-07-06T00:00 diff --git a/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml b/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml index b1953f5ca..a04288e0c 100644 --- a/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml +++ b/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml @@ -15,7 +15,8 @@ tree: children: child: pipeline: - - tdp-finder + compute: + - tdp-finder config: inputs: - timestamp: 2023-07-06T00:00 diff --git a/manifests/examples/builtins/divide/failure-denominator-equal-zero.yml b/manifests/examples/builtins/divide/failure-denominator-equal-zero.yml index d26272a82..96d627aee 100644 --- a/manifests/examples/builtins/divide/failure-denominator-equal-zero.yml +++ b/manifests/examples/builtins/divide/failure-denominator-equal-zero.yml @@ -23,8 +23,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide + compute: + - cloud-metadata + - divide config: divide: defaults: diff --git a/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml b/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml index 01347d943..fef8a1a7c 100644 --- a/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml +++ b/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml @@ -23,8 +23,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide + compute: + - cloud-metadata + - divide config: divide: defaults: diff --git a/manifests/examples/builtins/divide/failure-missing-numerator.yml b/manifests/examples/builtins/divide/failure-missing-numerator.yml index eda63d49d..5645f0ecf 100644 --- a/manifests/examples/builtins/divide/failure-missing-numerator.yml +++ b/manifests/examples/builtins/divide/failure-missing-numerator.yml @@ -23,8 +23,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide + compute: + - cloud-metadata + - divide config: divide: defaults: diff --git a/manifests/examples/builtins/divide/success.yml b/manifests/examples/builtins/divide/success.yml index 02222a6f4..32b247a87 100644 --- a/manifests/examples/builtins/divide/success.yml +++ b/manifests/examples/builtins/divide/success.yml @@ -23,10 +23,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide - config: - divide: + compute: + - cloud-metadata + - divide defaults: cloud/vendor: aws cloud/instance-type: m5n.large diff --git a/manifests/examples/builtins/exponent/success.yml b/manifests/examples/builtins/exponent/success.yml index fe54f30a2..163dd1460 100644 --- a/manifests/examples/builtins/exponent/success.yml +++ b/manifests/examples/builtins/exponent/success.yml @@ -14,9 +14,8 @@ tree: children: child: pipeline: - - exponent - config: - exponent: + compute: + - exponent inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/interpolation/interpolation.yml b/manifests/examples/builtins/interpolation/interpolation.yml index 35d750e48..394946467 100644 --- a/manifests/examples/builtins/interpolation/interpolation.yml +++ b/manifests/examples/builtins/interpolation/interpolation.yml @@ -12,12 +12,12 @@ initialize: y: [0.12, 0.32, 0.75, 1.02] input-parameter: "cpu/utilization" output-parameter: "result" - tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/interpolation/success.yml b/manifests/examples/builtins/interpolation/success.yml index cce024727..394946467 100644 --- a/manifests/examples/builtins/interpolation/success.yml +++ b/manifests/examples/builtins/interpolation/success.yml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml b/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml index fc74d71a6..472f797fd 100644 --- a/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml +++ b/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml @@ -29,5 +29,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: diff --git a/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml b/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml index 0184d280a..85e3f566b 100644 --- a/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml +++ b/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml @@ -29,5 +29,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: diff --git a/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml b/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml index 72cd6c347..58545dcec 100644 --- a/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml +++ b/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml @@ -29,5 +29,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: diff --git a/manifests/examples/builtins/mock-observations/success.yml b/manifests/examples/builtins/mock-observations/success.yml index 7767a68bc..5ca2d1942 100644 --- a/manifests/examples/builtins/mock-observations/success.yml +++ b/manifests/examples/builtins/mock-observations/success.yml @@ -29,5 +29,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: diff --git a/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml b/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml index d71f9f096..988a33fc2 100644 --- a/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml +++ b/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml @@ -13,9 +13,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml b/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml index 0a70ba5b3..e6e138723 100644 --- a/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml +++ b/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml @@ -13,9 +13,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/multiply/success.yml b/manifests/examples/builtins/multiply/success.yml index 88b7bbf71..c5d53e046 100644 --- a/manifests/examples/builtins/multiply/success.yml +++ b/manifests/examples/builtins/multiply/success.yml @@ -14,9 +14,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/regex/failure-missing-input-param.yml b/manifests/examples/builtins/regex/failure-missing-input-param.yml index f6732c632..84adc0cb2 100644 --- a/manifests/examples/builtins/regex/failure-missing-input-param.yml +++ b/manifests/examples/builtins/regex/failure-missing-input-param.yml @@ -14,9 +14,8 @@ tree: children: child: pipeline: - - regex - config: - regex: + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml b/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml index d8818265a..8341d08e6 100644 --- a/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml +++ b/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml @@ -14,9 +14,8 @@ tree: children: child: pipeline: - - regex - config: - regex: + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/regex/success.yml b/manifests/examples/builtins/regex/success.yml index 8e97d4c43..425dd9e67 100644 --- a/manifests/examples/builtins/regex/success.yml +++ b/manifests/examples/builtins/regex/success.yml @@ -14,9 +14,8 @@ tree: children: child: pipeline: - - regex - config: - regex: + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml b/manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml index 6ccd41a6e..503300f84 100644 --- a/manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml +++ b/manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml @@ -10,7 +10,8 @@ tree: children: child: pipeline: - - sci-embodied # duration & config -> embodied + compute: + - sci-embodied # duration & config -> embodied defaults: device/emissions-embodied: "fail" # gCO2eq time-reserved: 3600 # 1hr in seconds diff --git a/manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml b/manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml index 26edf154c..8fd3e5784 100644 --- a/manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml +++ b/manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml @@ -10,7 +10,8 @@ tree: children: child: pipeline: - - sci-embodied # duration & config -> embodied + compute: + - sci-embodied # duration & config -> embodied defaults: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds diff --git a/manifests/examples/builtins/sci-embodied/success.yml b/manifests/examples/builtins/sci-embodied/success.yml index 219e76845..991569404 100644 --- a/manifests/examples/builtins/sci-embodied/success.yml +++ b/manifests/examples/builtins/sci-embodied/success.yml @@ -10,7 +10,8 @@ tree: children: child: pipeline: - - sci-embodied # duration & config -> embodied + compute: + - sci-embodied # duration & config -> embodied defaults: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds diff --git a/manifests/examples/builtins/sci/failure-invalid-config-value.yml b/manifests/examples/builtins/sci/failure-invalid-config-value.yml index 798e1debd..5882111dd 100644 --- a/manifests/examples/builtins/sci/failure-invalid-config-value.yml +++ b/manifests/examples/builtins/sci/failure-invalid-config-value.yml @@ -13,7 +13,8 @@ tree: children: child: pipeline: - - sci + compute: + - sci config: sci: functional-unit: 999 # factor to convert per time to per f.unit diff --git a/manifests/examples/builtins/sci/failure-missing-input-param.yml b/manifests/examples/builtins/sci/failure-missing-input-param.yml index d30f21550..0a7677261 100644 --- a/manifests/examples/builtins/sci/failure-missing-input-param.yml +++ b/manifests/examples/builtins/sci/failure-missing-input-param.yml @@ -14,7 +14,8 @@ tree: children: child: pipeline: - - sci + compute: + - sci inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/sci/success.yml b/manifests/examples/builtins/sci/success.yml index 95979fd28..85dd19db2 100644 --- a/manifests/examples/builtins/sci/success.yml +++ b/manifests/examples/builtins/sci/success.yml @@ -13,8 +13,8 @@ tree: children: child: pipeline: - - sci - config: + compute: + - sci inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/shell/failure-invalid-command.yml b/manifests/examples/builtins/shell/failure-invalid-command.yml index 2944bc3ea..e67aff103 100644 --- a/manifests/examples/builtins/shell/failure-invalid-command.yml +++ b/manifests/examples/builtins/shell/failure-invalid-command.yml @@ -12,7 +12,8 @@ tree: children: child: pipeline: - - shell + compute: + - shell inputs: - timestamp: "2023-11-02T10:35:31.820Z" duration: 3600 diff --git a/manifests/examples/builtins/shell/success.yml b/manifests/examples/builtins/shell/success.yml index b44ca2892..a26af274d 100644 --- a/manifests/examples/builtins/shell/success.yml +++ b/manifests/examples/builtins/shell/success.yml @@ -12,7 +12,8 @@ tree: children: child: pipeline: - - shell + compute: + - shell inputs: - timestamp: "2023-11-02T10:35:31.820Z" duration: 3600 diff --git a/manifests/examples/builtins/subtract/success.yml b/manifests/examples/builtins/subtract/success.yml index b29d89fd0..745ceb067 100644 --- a/manifests/examples/builtins/subtract/success.yml +++ b/manifests/examples/builtins/subtract/success.yml @@ -13,9 +13,8 @@ tree: children: child: pipeline: - - subtract - config: - subtract: + compute: + - subtract inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/sum/failure-missing-input-param.yml b/manifests/examples/builtins/sum/failure-missing-input-param.yml index 65004b679..1e963b694 100644 --- a/manifests/examples/builtins/sum/failure-missing-input-param.yml +++ b/manifests/examples/builtins/sum/failure-missing-input-param.yml @@ -13,9 +13,8 @@ tree: children: child: pipeline: - - sum - config: - sum: + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/sum/failure-missing-output-param.yml b/manifests/examples/builtins/sum/failure-missing-output-param.yml index b3c21ac3e..0248bf449 100644 --- a/manifests/examples/builtins/sum/failure-missing-output-param.yml +++ b/manifests/examples/builtins/sum/failure-missing-output-param.yml @@ -13,9 +13,8 @@ tree: children: child: pipeline: - - sum - config: - sum: + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/sum/success.yml b/manifests/examples/builtins/sum/success.yml index 4f5f7452b..366ae5580 100644 --- a/manifests/examples/builtins/sum/success.yml +++ b/manifests/examples/builtins/sum/success.yml @@ -13,9 +13,8 @@ tree: children: child: pipeline: - - sum - config: - sum: + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml b/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml index e6bc3a230..ffed32274 100644 --- a/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml +++ b/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml @@ -17,8 +17,8 @@ tree: children: child: pipeline: - - time-sync - config: + compute: + - time-sync inputs: - timestamp: '2023-12-12T00:00:00.000Z' duration: 1 @@ -31,4 +31,4 @@ tree: energy-cpu: 0.001 - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 - energy-cpu: 0.001 + energy-cpu: 0.001 \ No newline at end of file diff --git a/manifests/examples/builtins/time-sync/failure-missing-global-config.yml b/manifests/examples/builtins/time-sync/failure-missing-global-config.yml index 7b8e110fb..c5715a26a 100644 --- a/manifests/examples/builtins/time-sync/failure-missing-global-config.yml +++ b/manifests/examples/builtins/time-sync/failure-missing-global-config.yml @@ -17,8 +17,8 @@ tree: children: child: pipeline: - - time-sync - config: + compute: + - time-sync inputs: - timestamp: '2023-12-12T00:00:00.000Z' duration: 3 @@ -31,4 +31,4 @@ tree: energy-cpu: 0.001 - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 - energy-cpu: 0.001 + energy-cpu: 0.001 \ No newline at end of file diff --git a/manifests/examples/builtins/time-sync/success.yml b/manifests/examples/builtins/time-sync/success.yml index 6d1841e97..8aac13740 100644 --- a/manifests/examples/builtins/time-sync/success.yml +++ b/manifests/examples/builtins/time-sync/success.yml @@ -17,8 +17,8 @@ tree: children: child: pipeline: - - time-sync - config: + compute: + - time-sync inputs: - timestamp: '2023-12-12T00:00:00.000Z' duration: 1 diff --git a/manifests/examples/builtins/groupby/failure-invalid-config-group.yml b/manifests/examples/features/regroup/failure-invalid-regroup.yml similarity index 82% rename from manifests/examples/builtins/groupby/failure-invalid-config-group.yml rename to manifests/examples/features/regroup/failure-invalid-regroup.yml index eca1dd8a9..a121d82b5 100644 --- a/manifests/examples/builtins/groupby/failure-invalid-config-group.yml +++ b/manifests/examples/features/regroup/failure-invalid-regroup.yml @@ -1,18 +1,12 @@ -name: groupby -description: failure when `config->group-by->group` is not an array +name: regroup +description: failure when `regroup` is not an array initialize: - plugins: - group-by: - path: "builtin" - method: GroupBy + plugins: {} tree: children: my-app: pipeline: - - group-by - config: - group-by: - group: cloud/region + regroup: cloud/region inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/examples/builtins/groupby/failure-missing-cloud-instance-type.yml b/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml similarity index 84% rename from manifests/examples/builtins/groupby/failure-missing-cloud-instance-type.yml rename to manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml index 34f31266d..8b8b44faf 100644 --- a/manifests/examples/builtins/groupby/failure-missing-cloud-instance-type.yml +++ b/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml @@ -1,20 +1,14 @@ -name: groupby +name: regroup description: initialize: plugins: - group-by: - path: "builtin" - method: GroupBy tree: children: my-app: pipeline: - - group-by - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/examples/builtins/groupby/success.yml b/manifests/examples/features/regroup/success.yml similarity index 84% rename from manifests/examples/builtins/groupby/success.yml rename to manifests/examples/features/regroup/success.yml index f3f4161f6..6863dc0d6 100644 --- a/manifests/examples/builtins/groupby/success.yml +++ b/manifests/examples/features/regroup/success.yml @@ -1,20 +1,14 @@ -name: groupby +name: regroup description: successful path initialize: plugins: - group-by: - path: "builtin" - method: GroupBy tree: children: my-app: pipeline: - - group-by - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/examples/pipelines/generics.yml b/manifests/examples/pipelines/generics.yml index 4fd3fa018..f9a19a6b3 100644 --- a/manifests/examples/pipelines/generics.yml +++ b/manifests/examples/pipelines/generics.yml @@ -62,15 +62,15 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - coefficient - - multiply - config: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - coefficient + - multiply defaults: cpu/thermal-design-power: 100 vcpus-allocated: 1 diff --git a/manifests/examples/pipelines/instance-metadata.yml b/manifests/examples/pipelines/instance-metadata.yml index d79dedb8c..0a64e9ac3 100644 --- a/manifests/examples/pipelines/instance-metadata.yml +++ b/manifests/examples/pipelines/instance-metadata.yml @@ -23,8 +23,9 @@ tree: children: child: pipeline: - - cloud-instance-metadata - - extract-processor-name + compute: + - cloud-instance-metadata + - extract-processor-name inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/pipelines/nesting.yml b/manifests/examples/pipelines/nesting.yml index 9f034092f..baf12a688 100644 --- a/manifests/examples/pipelines/nesting.yml +++ b/manifests/examples/pipelines/nesting.yml @@ -6,8 +6,7 @@ tags: category: on-premise aggregation: metrics: - "carbon": - method: sum + - carbon type: "both" initialize: plugins: @@ -20,12 +19,38 @@ initialize: y: [0.12, 0.32, 0.75, 1.02] input-parameter: "cpu/utilization" output-parameter: "cpu-factor" + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg + outputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg "cpu-factor-to-wattage": method: Multiply path: builtin global-config: input-parameters: ["cpu-factor", "cpu/thermal-design-power"] output-parameter: "cpu-wattage" + parameter-metadata: + inputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu/thermal-design-power: + unit: kWh + description: thermal design power for a processor + aggregation-method: avg + outputs: + cpu-wattage: + unit: kWh + description: the energy used by the CPU + aggregation-method: sum "wattage-times-duration": method: Multiply path: builtin @@ -39,6 +64,17 @@ initialize: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw + parameter-metadata: + inputs: + cpu-wattage-times-duration: + unit: kWh + description: CPU wattage multiplied by duration + aggregation-method: sum + outputs: + cpu-energy-raw: + unit: kWh + description: Raw energy used by CPU in kWh + aggregation-method: sum "calculate-vcpu-ratio": method: Divide path: "builtin" @@ -46,6 +82,12 @@ initialize: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio + parameter-metadata: + outputs: + vcpu-ratio: + unit: none + description: Ratio of vCPUs + aggregation-method: none "correct-cpu-energy-for-vcpu-ratio": method: Divide path: "builtin" @@ -62,11 +104,32 @@ initialize: global-config: input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"] output-parameter: "carbon-operational" + parameter-metadata: + inputs: + cpu-energy-kwh: + unit: kWh + description: Corrected CPU energy in kWh + aggregation-method: sum + grid/carbon-intensity: + unit: gCO2eq/kWh + description: Carbon intensity for the grid + aggregation-method: avg + outputs: + carbon-operational: + unit: gCO2eq + description: Operational carbon footprint + aggregation-method: sum sci: path: "builtin" method: Sci global-config: functional-unit: "requests" + parameter-metadata: + inputs: + requests: + unit: none + description: expressed the final SCI value + aggregation-method: sum "sum-carbon": path: "builtin" method: Sum @@ -75,6 +138,21 @@ initialize: - carbon-operational - carbon-embodied output-parameter: carbon + parameter-metadata: + inputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + carbon-embodied: + description: Embodied carbon footprint + unit: gCO2eq + aggregation-method: sum + outputs: + carbon: + description: Total carbon footprint + unit: gCO2eq + aggregation-method: sum time-sync: method: TimeSync path: "builtin" @@ -83,6 +161,32 @@ initialize: end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true + parameter-metadata: + inputs: + timestamp: + unit: RFC3339 + description: refers to the time of occurrence of the input + aggregation-method: none + duration: + unit: seconds + description: refers to the duration of the input + aggregation-method: sum + cloud/instance-type: + unit: none + description: type of Cloud Instance name used in the cloud provider APIs + aggregation-method: none + cloud/region: + unit: none + description: region cloud instance + aggregation-method: none + time-reserved: + unit: seconds + description: time reserved for a component + aggregation-method: avg + network/energy: + description: "Energy consumed by the Network of the component" + unit: "kWh" + aggregation-method: "sum" tree: children: child-0: @@ -95,17 +199,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -145,17 +250,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -197,17 +303,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -247,17 +354,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 diff --git a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml new file mode 100644 index 000000000..c1f92da04 --- /dev/null +++ b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml @@ -0,0 +1,997 @@ +name: pipeline-with-aggregate +description: a full pipeline with the aggregate feature enabled +tags: null +aggregation: + metrics: + - carbon + type: both +initialize: + plugins: + interpolate: + path: builtin + method: Interpolation + global-config: + method: linear + x: + - 0 + - 10 + - 50 + - 100 + 'y': + - 0.12 + - 0.32 + - 0.75 + - 1.02 + input-parameter: cpu/utilization + output-parameter: cpu-factor + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg + outputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu-factor-to-wattage: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-factor + - cpu/thermal-design-power + output-parameter: cpu-wattage + parameter-metadata: + inputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu/thermal-design-power: + unit: kWh + description: thermal design power for a processor + aggregation-method: avg + outputs: + cpu-wattage: + unit: kWh + description: the energy used by the CPU + aggregation-method: sum + wattage-times-duration: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-wattage + - duration + output-parameter: cpu-wattage-times-duration + wattage-to-energy-kwh: + path: builtin + method: Divide + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + parameter-metadata: + inputs: + cpu-wattage-times-duration: + unit: kWh + description: CPU wattage multiplied by duration + aggregation-method: sum + outputs: + cpu-energy-raw: + unit: kWh + description: Raw energy used by CPU in kWh + aggregation-method: sum + calculate-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + parameter-metadata: + inputs: + vcpus-total: + unit: count + description: total number of vcpus available on a particular resource + aggregation-method: none + vcpus-allocated: + unit: count + description: number of vcpus allocated to particular resource + aggregation-method: none + outputs: + vcpu-ratio: + unit: none + description: Ratio of vCPUs + aggregation-method: none + correct-cpu-energy-for-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh + sci-embodied: + path: builtin + method: SciEmbodied + operational-carbon: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-energy-kwh + - grid/carbon-intensity + output-parameter: carbon-operational + parameter-metadata: + inputs: + cpu-energy-kwh: + unit: kWh + description: Corrected CPU energy in kWh + aggregation-method: sum + grid/carbon-intensity: + unit: gCO2eq/kWh + description: Carbon intensity for the grid + aggregation-method: avg + outputs: + carbon-operational: + unit: gCO2eq + description: Operational carbon footprint + aggregation-method: sum + sci: + path: builtin + method: Sci + global-config: + functional-unit: requests + parameter-metadata: + inputs: + requests: + unit: none + description: expressed the final SCI value + aggregation-method: sum + sum-carbon: + path: builtin + method: Sum + global-config: + input-parameters: + - carbon-operational + - carbon-embodied + output-parameter: carbon + parameter-metadata: + inputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + carbon-embodied: + description: Embodied carbon footprint + unit: gCO2eq + aggregation-method: sum + outputs: + carbon: + description: Total carbon footprint + unit: gCO2eq + aggregation-method: sum + time-sync: + path: builtin + method: TimeSync + global-config: + start-time: '2023-12-12T00:00:00.000Z' + end-time: '2023-12-12T00:01:00.000Z' + interval: 5 + allow-padding: true + parameter-metadata: + inputs: + timestamp: + unit: RFC3339 + description: refers to the time of occurrence of the input + aggregation-method: none + duration: + unit: seconds + description: refers to the duration of the input + aggregation-method: sum + cloud/instance-type: + unit: none + description: type of Cloud Instance name used in the cloud provider APIs + aggregation-method: none + cloud/region: + unit: none + description: region cloud instance + aggregation-method: none + time-reserved: + unit: seconds + description: time reserved for a component + aggregation-method: avg +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/pipeline-with-aggregate.yml -o + manifests/outputs/pipelines/pipeline-with-aggregate-1.yaml + environment: + if-version: 0.5.0 + os: macOS + os-version: '14.5' + node-version: 18.14.2 + date-time: 2024-07-31T12:41:31.920Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.16' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + status: success +tree: + children: + child-1: + pipeline: + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 10 + requests: 10 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 5 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 15 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + requests: 30 + outputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: 14 + requests: 14 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 2759.6159999999995 + time-reserved: 2880 + device/expected-lifespan: 170294400 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.3205 + cpu-wattage: 66.19999999999999 + cpu-wattage-times-duration: 203 + cpu-energy-raw: 0.0000563888888888889 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000007048611111111113 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.0056388888888888895 + carbon: 0.005649016996448503 + sci: 0.000403501214032036 + - timestamp: '2023-12-12T00:00:05.000Z' + duration: 5 + cpu/utilization: 13 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 9.571428571428571 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 1182.6925714285712 + time-reserved: 2880 + device/expected-lifespan: 72983314.28571428 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.30975 + cpu-wattage: 29.907142857142862 + cpu-wattage-times-duration: 192.25 + cpu-energy-raw: 0.00005340277777777778 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006675347222222222 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005340277777777777 + carbon: 0.005350405885337391 + sci: 0.0005589976298113692 + - timestamp: '2023-12-12T00:00:10.000Z' + duration: 5 + cpu/utilization: 12 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 8.428571428571429 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 759.2594285714285 + time-reserved: 2880 + device/expected-lifespan: 46853485.71428572 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.29900000000000004 + cpu-wattage: 18.50952380952381 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.0000064887152777777775 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005190972222222222 + carbon: 0.0052011003297818366 + sci: 0.0006170797001436077 + - timestamp: '2023-12-12T00:00:15.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + requests: 5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0010402200659563674 + - timestamp: '2023-12-12T00:00:20.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + requests: 5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0010402200659563674 + - timestamp: '2023-12-12T00:00:25.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + requests: 5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0010402200659563674 + - timestamp: '2023-12-12T00:00:30.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + requests: 5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0010402200659563674 + - timestamp: '2023-12-12T00:00:35.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + requests: 5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0010402200659563674 + - timestamp: '2023-12-12T00:00:40.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 9 + requests: 3 + cpu/thermal-design-power: 60 + grid/carbon-intensity: 480 + device/emissions-embodied: 153.312 + time-reserved: 2160.2 + device/expected-lifespan: 9460800 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.22425 + cpu-wattage: 3.7375 + cpu-wattage-times-duration: 112.125 + cpu-energy-raw: 0.000031145833333333336 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000003893229166666667 + carbon-embodied: 0.000006076864535768645 + carbon-operational: 0.0031145833333333334 + carbon: 0.003120660197869102 + sci: 0.0010402200659563674 + - timestamp: '2023-12-12T00:00:45.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: '2023-12-12T00:00:50.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: '2023-12-12T00:00:55.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: '2023-12-12T00:01:00.000Z' + duration: 1 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 1 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + aggregated: + carbon: 0.04532668505834602 + child-2: + pipeline: + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + duration: 1 + cpu/utilization: 30 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 100 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + cpu/utilization: 28 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 150 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + cpu/utilization: 40 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 110 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + cpu/utilization: 33 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 180 + outputs: + - timestamp: '2023-12-12T00:00:00.000Z' + duration: 5 + cpu/utilization: 22.8 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 220 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 2759.6159999999995 + time-reserved: 2880 + device/expected-lifespan: 170294400 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.41509999999999997 + cpu-wattage: 94.57999999999998 + cpu-wattage-times-duration: 258.9 + cpu-energy-raw: 0.00007191666666666668 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000008989583333333334 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.007191666666666666 + carbon: 0.007201794774226282 + sci: 0.00003273543079193765 + - timestamp: '2023-12-12T00:00:05.000Z' + duration: 5 + cpu/utilization: 29.6 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 92.85714285714285 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 1182.6925714285712 + time-reserved: 2880 + device/expected-lifespan: 72983314.28571428 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.48819999999999997 + cpu-wattage: 46.98428571428572 + cpu-wattage-times-duration: 308.35 + cpu-energy-raw: 0.00008565277777777778 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000010706597222222223 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.008565277777777778 + carbon: 0.008575405885337391 + sci: 0.00009235052491901808 + - timestamp: '2023-12-12T00:00:10.000Z' + duration: 5 + cpu/utilization: 30.6 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 59.14285714285714 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 759.2594285714285 + time-reserved: 2880 + device/expected-lifespan: 46853485.71428572 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.49894999999999995 + cpu-wattage: 31.31738095238095 + cpu-wattage-times-duration: 306.2 + cpu-energy-raw: 0.00008505555555555556 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000010631944444444445 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.008505555555555556 + carbon: 0.00851568366311517 + sci: 0.0001439849894729618 + - timestamp: '2023-12-12T00:00:15.000Z' + duration: 5 + cpu/utilization: 26.4 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 30 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.45380000000000004 + cpu-wattage: 9.454166666666667 + cpu-wattage-times-duration: 283.625 + cpu-energy-raw: 0.00007878472222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000009848090277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.007878472222222222 + carbon: 0.007888600329781836 + sci: 0.0002629533443260612 + - timestamp: '2023-12-12T00:00:20.000Z' + duration: 5 + cpu/utilization: 26.4 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 30 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.45380000000000004 + cpu-wattage: 9.454166666666667 + cpu-wattage-times-duration: 283.625 + cpu-energy-raw: 0.00007878472222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000009848090277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.007878472222222222 + carbon: 0.007888600329781836 + sci: 0.0002629533443260612 + - timestamp: '2023-12-12T00:00:25.000Z' + duration: 5 + cpu/utilization: 26.4 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 30 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.45380000000000004 + cpu-wattage: 9.454166666666667 + cpu-wattage-times-duration: 283.625 + cpu-energy-raw: 0.00007878472222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000009848090277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.007878472222222222 + carbon: 0.007888600329781836 + sci: 0.0002629533443260612 + - timestamp: '2023-12-12T00:00:30.000Z' + duration: 5 + cpu/utilization: 26.4 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 30 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.45380000000000004 + cpu-wattage: 9.454166666666667 + cpu-wattage-times-duration: 283.625 + cpu-energy-raw: 0.00007878472222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000009848090277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.007878472222222222 + carbon: 0.007888600329781836 + sci: 0.0002629533443260612 + - timestamp: '2023-12-12T00:00:35.000Z' + duration: 5 + cpu/utilization: 26.4 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 30 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.45380000000000004 + cpu-wattage: 9.454166666666667 + cpu-wattage-times-duration: 283.625 + cpu-energy-raw: 0.00007878472222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000009848090277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.007878472222222222 + carbon: 0.007888600329781836 + sci: 0.0002629533443260612 + - timestamp: '2023-12-12T00:00:40.000Z' + duration: 5 + cpu/utilization: 19.8 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 18 + cpu/thermal-design-power: 60 + grid/carbon-intensity: 480 + device/emissions-embodied: 153.312 + time-reserved: 2160.2 + device/expected-lifespan: 9460800 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.34035000000000004 + cpu-wattage: 5.6725 + cpu-wattage-times-duration: 170.175 + cpu-energy-raw: 0.00004727083333333333 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000005908854166666666 + carbon-embodied: 0.000006076864535768645 + carbon-operational: 0.004727083333333333 + carbon: 0.0047331601978691015 + sci: 0.00026295334432606117 + - timestamp: '2023-12-12T00:00:45.000Z' + duration: 5 + cpu/utilization: 0 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: '2023-12-12T00:00:50.000Z' + duration: 5 + cpu/utilization: 0 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: '2023-12-12T00:00:55.000Z' + duration: 5 + cpu/utilization: 0 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: '2023-12-12T00:01:00.000Z' + duration: 1 + cpu/utilization: 0 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 1 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + aggregated: + carbon: 0.06846904616945712 + outputs: + - carbon: 0.012850811770674785 + timestamp: '2023-12-12T00:00:00.000Z' + duration: 5 + - carbon: 0.013925811770674782 + timestamp: '2023-12-12T00:00:05.000Z' + duration: 5 + - carbon: 0.013716783992897007 + timestamp: '2023-12-12T00:00:10.000Z' + duration: 5 + - carbon: 0.013089700659563674 + timestamp: '2023-12-12T00:00:15.000Z' + duration: 5 + - carbon: 0.013089700659563674 + timestamp: '2023-12-12T00:00:20.000Z' + duration: 5 + - carbon: 0.013089700659563674 + timestamp: '2023-12-12T00:00:25.000Z' + duration: 5 + - carbon: 0.013089700659563674 + timestamp: '2023-12-12T00:00:30.000Z' + duration: 5 + - carbon: 0.013089700659563674 + timestamp: '2023-12-12T00:00:35.000Z' + duration: 5 + - carbon: 0.007853820395738204 + timestamp: '2023-12-12T00:00:40.000Z' + duration: 5 + - carbon: 0 + timestamp: '2023-12-12T00:00:45.000Z' + duration: 5 + - carbon: 0 + timestamp: '2023-12-12T00:00:50.000Z' + duration: 5 + - carbon: 0 + timestamp: '2023-12-12T00:00:55.000Z' + duration: 5 + - carbon: 0 + timestamp: '2023-12-12T00:01:00.000Z' + duration: 1 + aggregated: + carbon: 0.11379573122780316 diff --git a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml new file mode 100644 index 000000000..77496bf23 --- /dev/null +++ b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml @@ -0,0 +1,1073 @@ +name: pipeline-with-mocks +description: a full pipeline seeded with data from mock-observations feature +tags: null +aggregation: + metrics: + - carbon + type: both +initialize: + plugins: + mock-observations: + path: builtin + method: MockObservations + global-config: + timestamp-from: 2023-12-12T00:00 + timestamp-to: 2023-12-12T00:10 + duration: 60 + components: + - cloud/instance-type: A1 + generators: + common: + cloud/region: uk-west + randint: + cpu/utilization: + min: 1 + max: 99 + parameter-metadata: + inputs: + timestamp: + unit: RFC3339 + description: refers to the time of occurrence of the input + aggregation-method: none + duration: + unit: seconds + description: refers to the duration of the input + aggregation-method: sum + cloud/instance-type: + unit: none + description: type of Cloud Instance name used in the cloud provider APIs + aggregation-method: none + cloud/region: + unit: none + description: region cloud instance + aggregation-method: none + interpolate: + path: builtin + method: Interpolation + global-config: + method: linear + x: + - 0 + - 10 + - 50 + - 100 + "y": + - 0.12 + - 0.32 + - 0.75 + - 1.02 + input-parameter: cpu/utilization + output-parameter: cpu-factor + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg + outputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu-factor-to-wattage: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-factor + - cpu/thermal-design-power + output-parameter: cpu-wattage + parameter-metadata: + inputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu/thermal-design-power: + unit: kWh + description: thermal design power for a processor + aggregation-method: avg + outputs: + cpu-wattage: + unit: kWh + description: the energy used by the CPU + aggregation-method: sum + wattage-times-duration: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-wattage + - duration + output-parameter: cpu-wattage-times-duration + parameter-metadata: + inputs: + cpu-wattage: + unit: kWh + description: Energy used by the CPU + aggregation-method: sum + duration: + unit: seconds + description: Duration of the observation + aggregation-method: sum + outputs: + cpu-wattage-times-duration: + unit: kWh + description: CPU wattage multiplied by duration + aggregation-method: sum + wattage-to-energy-kwh: + path: builtin + method: Divide + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + parameter-metadata: + inputs: + cpu-wattage-times-duration: + unit: kWh + description: CPU wattage multiplied by duration + aggregation-method: sum + outputs: + cpu-energy-raw: + unit: kWh + description: Raw energy used by CPU in kWh + aggregation-method: sum + calculate-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + parameter-metadata: + inputs: + vcpus-total: + unit: count + description: total number of vcpus available on a particular resource + aggregation-method: none + vcpus-allocated: + unit: count + description: number of vcpus allocated to particular resource + aggregation-method: none + outputs: + vcpu-ratio: + unit: none + description: Ratio of vCPUs + aggregation-method: none + correct-cpu-energy-for-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh + parameter-metadata: + inputs: + cpu-energy-raw: + unit: kWh + description: Raw energy used by CPU in kWh + aggregation-method: sum + vcpu-ratio: + unit: none + description: Ratio of vCPUs + aggregation-method: none + outputs: + cpu-energy-kwh: + unit: kWh + description: Corrected CPU energy in kWh + aggregation-method: sum + sci-embodied: + path: builtin + method: SciEmbodied + operational-carbon: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-energy-kwh + - grid/carbon-intensity + output-parameter: carbon-operational + parameter-metadata: + inputs: + cpu-energy-kwh: + unit: kWh + description: Corrected CPU energy in kWh + aggregation-method: sum + grid/carbon-intensity: + unit: gCO2eq/kWh + description: Carbon intensity for the grid + aggregation-method: avg + outputs: + carbon-operational: + unit: gCO2eq + description: Operational carbon footprint + aggregation-method: sum + sum-carbon: + path: builtin + method: Sum + global-config: + input-parameters: + - carbon-operational + - carbon-embodied + output-parameter: carbon + parameter-metadata: + inputs: + carbon-operational: + unit: gCO2eq + description: Operational carbon footprint + aggregation-method: sum + carbon-embodied: + unit: gCO2eq + description: Embodied carbon footprint + aggregation-method: sum + outputs: + carbon: + unit: gCO2eq + description: Total carbon footprint + aggregation-method: sum + sci: + path: builtin + method: Sci + global-config: + functional-unit: requests + parameter-metadata: + inputs: + requests: + unit: none + description: expressed the final SCI value + aggregation-method: sum + outputs: + sci: + unit: none + description: Scientific Carbon Intensity + aggregation-method: none + time-sync: + path: builtin + method: TimeSync + global-config: + start-time: "2023-12-12T00:00:00.000Z" + end-time: "2023-12-12T00:01:00.000Z" + interval: 5 + allow-padding: true + parameter-metadata: + inputs: + time-reserved: + unit: seconds + description: time reserved for a component + aggregation-method: avg + outputs: + synced-time: + unit: none + description: Synced time + aggregation-method: none + group-by: + path: builtin + method: GroupBy + parameter-metadata: + inputs: + group: + unit: none + description: Group by fields + aggregation-method: none + outputs: + grouped-data: + unit: none + description: Grouped data + aggregation-method: none +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/pipeline-with-mocks.yml -o + manifests/outputs/pipelines/pipeline-with-mocks-update + environment: + if-version: 0.5.0 + os: macOS + os-version: "14.5" + node-version: 18.14.2 + date-time: 2024-07-30T05:35:42.937Z (UTC) + dependencies: + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.16" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + status: success +tree: + children: + child-1: + pipeline: + observe: + - mock-observations + regroup: + - cloud/region + - instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 10 + requests: 30 + - timestamp: "2023-12-12T00:00:01.000Z" + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 40 + - timestamp: "2023-12-12T00:00:06.000Z" + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 30 + - timestamp: "2023-12-12T00:00:13.000Z" + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + requests: 50 + outputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:05.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:10.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:15.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:20.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:25.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:30.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:35.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:40.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:45.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:50.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:55.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:01:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: "*" + requests: 0.5 + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 25.552 + time-reserved: 3600 + device/expected-lifespan: 1576800 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.0000020256215119228817 + carbon-operational: "*" + carbon: "*" + sci: "*" + aggregated: + carbon: "*" + child-2: + pipeline: + observe: + - mock-observations + regroup: + - cloud/region + - instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + duration: 1 + cpu/utilization: 30 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 30 + - timestamp: "2023-12-12T00:00:01.000Z" + duration: 5 + cpu/utilization: 28 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 40 + - timestamp: "2023-12-12T00:00:06.000Z" + duration: 7 + cpu/utilization: 40 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 50 + - timestamp: "2023-12-12T00:00:13.000Z" + duration: 30 + cpu/utilization: 33 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 60 + outputs: + - timestamp: "2023-12-12T00:00:00.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 0.5 + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 25.552 + time-reserved: 3600 + device/expected-lifespan: 1576800 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + aggregated: + carbon: "*" + outputs: + - carbon: "*" + timestamp: "2023-12-12T00:00:00.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + aggregated: + carbon: "*" diff --git a/manifests/examples/pipelines/pipeline-teads-sci.yml b/manifests/examples/pipelines/pipeline-teads-sci.yml index c07b093bd..d80d0e89e 100644 --- a/manifests/examples/pipelines/pipeline-teads-sci.yml +++ b/manifests/examples/pipelines/pipeline-teads-sci.yml @@ -79,17 +79,17 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - sci - config: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 diff --git a/manifests/examples/pipelines/pipeline-with-aggregate.yml b/manifests/examples/pipelines/pipeline-with-aggregate.yml index a833bcb9b..184820228 100644 --- a/manifests/examples/pipelines/pipeline-with-aggregate.yml +++ b/manifests/examples/pipelines/pipeline-with-aggregate.yml @@ -3,8 +3,7 @@ description: a full pipeline with the aggregate feature enabled tags: aggregation: metrics: - "carbon": - method: sum + - carbon type: "both" initialize: plugins: @@ -17,12 +16,38 @@ initialize: y: [0.12, 0.32, 0.75, 1.02] input-parameter: "cpu/utilization" output-parameter: "cpu-factor" + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg + outputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg "cpu-factor-to-wattage": method: Multiply path: builtin global-config: input-parameters: ["cpu-factor", "cpu/thermal-design-power"] output-parameter: "cpu-wattage" + parameter-metadata: + inputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu/thermal-design-power: + unit: kWh + description: thermal design power for a processor + aggregation-method: avg + outputs: + cpu-wattage: + unit: kWh + description: the energy used by the CPU + aggregation-method: sum "wattage-times-duration": method: Multiply path: builtin @@ -36,6 +61,17 @@ initialize: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw + parameter-metadata: + inputs: + cpu-wattage-times-duration: + unit: kWh + description: CPU wattage multiplied by duration + aggregation-method: sum + outputs: + cpu-energy-raw: + unit: kWh + description: Raw energy used by CPU in kWh + aggregation-method: sum "calculate-vcpu-ratio": method: Divide path: "builtin" @@ -43,6 +79,21 @@ initialize: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio + parameter-metadata: + inputs: + vcpus-total: + unit: count + description: total number of vcpus available on a particular resource + aggregation-method: none + vcpus-allocated: + unit: count + description: number of vcpus allocated to particular resource + aggregation-method: none + outputs: + vcpu-ratio: + unit: none + description: Ratio of vCPUs + aggregation-method: none "correct-cpu-energy-for-vcpu-ratio": method: Divide path: "builtin" @@ -59,11 +110,32 @@ initialize: global-config: input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"] output-parameter: "carbon-operational" + parameter-metadata: + inputs: + cpu-energy-kwh: + unit: kWh + description: Corrected CPU energy in kWh + aggregation-method: sum + grid/carbon-intensity: + unit: gCO2eq/kWh + description: Carbon intensity for the grid + aggregation-method: avg + outputs: + carbon-operational: + unit: gCO2eq + description: Operational carbon footprint + aggregation-method: sum "sci": path: "builtin" method: Sci global-config: functional-unit: requests # factor to convert per time to per f.unit + parameter-metadata: + inputs: + requests: + unit: none + description: expressed the final SCI value + aggregation-method: sum "sum-carbon": path: "builtin" method: Sum @@ -72,6 +144,12 @@ initialize: - carbon-operational - carbon-embodied output-parameter: carbon + parameter-metadata: + outputs: + carbon: + unit: gCO2eq + description: product of carbon + aggregation-method: sum "time-sync": method: TimeSync path: "builtin" @@ -80,26 +158,52 @@ initialize: end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true + parameter-metadata: + inputs: + timestamp: + unit: RFC3339 + description: refers to the time of occurrence of the input + aggregation-method: none + duration: + unit: seconds + description: refers to the duration of the input + aggregation-method: sum + cloud/instance-type: + unit: none + description: type of Cloud Instance name used in the cloud provider APIs + aggregation-method: none + cloud/region: + unit: none + description: region cloud instance + aggregation-method: none + time-reserved: + unit: seconds + description: time reserved for a component + aggregation-method: avg + outputs: + synced-time: + unit: none + description: Synced time + aggregation-method: none tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -135,22 +239,21 @@ tree: requests: 30 child-2: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 diff --git a/manifests/examples/pipelines/pipeline-with-mocks.yml b/manifests/examples/pipelines/pipeline-with-mocks.yml index 0ef54f32c..fc6f81cb0 100644 --- a/manifests/examples/pipelines/pipeline-with-mocks.yml +++ b/manifests/examples/pipelines/pipeline-with-mocks.yml @@ -3,8 +3,7 @@ description: a full pipeline seeded with data from mock-observations feature tags: aggregation: metrics: - "carbon": - method: sum + - carbon type: "both" initialize: plugins: @@ -13,9 +12,9 @@ initialize: method: MockObservations path: "builtin" global-config: - timestamp-from: 2023-12-12T00:00 - timestamp-to: 2023-12-12T00:10 - duration: 60 + timestamp-from: "2023-12-12T00:00:00.000Z" + timestamp-to: "2023-12-12T00:00:13.000Z" + duration: 30 components: - cloud/instance-type: A1 generators: @@ -43,7 +42,7 @@ initialize: description: region cloud instance unit: none aggregation-method: none - "interpolate": + interpolate: method: Interpolation path: "builtin" global-config: @@ -63,7 +62,7 @@ initialize: description: result of interpolate unit: kWh aggregation-method: avg - "cpu-factor-to-wattage": + cpu-factor-to-wattage: method: Multiply path: builtin global-config: @@ -71,29 +70,59 @@ initialize: output-parameter: "cpu-wattage" parameter-metadata: inputs: + cpu-factor: + description: result of interpolate + unit: kWh + aggregation-method: avg cpu/thermal-design-power: description: thermal design power for a processor - unit: kwh + unit: kWh aggregation-method: avg outputs: cpu-wattage: description: the energy used by the CPU - unit: kwh + unit: kWh aggregation-method: sum - "wattage-times-duration": + wattage-times-duration: method: Multiply path: builtin global-config: input-parameters: ["cpu-wattage", "duration"] output-parameter: "cpu-wattage-times-duration" - "wattage-to-energy-kwh": + parameter-metadata: + inputs: + cpu-wattage: + description: Energy used by the CPU + unit: kWh + aggregation-method: sum + duration: + description: Duration of the observation + unit: seconds + aggregation-method: sum + outputs: + cpu-wattage-times-duration: + description: CPU wattage multiplied by duration + unit: kWh + aggregation-method: sum + wattage-to-energy-kwh: method: Divide path: "builtin" global-config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw - "calculate-vcpu-ratio": + parameter-metadata: + inputs: + cpu-wattage-times-duration: + description: CPU wattage multiplied by duration + unit: kWh + aggregation-method: sum + outputs: + cpu-energy-raw: + description: Raw energy used by CPU in kWh + unit: kWh + aggregation-method: sum + calculate-vcpu-ratio: method: Divide path: "builtin" global-config: @@ -110,17 +139,37 @@ initialize: description: number of vcpus allocated to particular resource unit: count aggregation-method: none - "correct-cpu-energy-for-vcpu-ratio": + outputs: + vcpu-ratio: + description: Ratio of vCPUs + unit: none + aggregation-method: none + correct-cpu-energy-for-vcpu-ratio: method: Divide path: "builtin" global-config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh - "sci-embodied": + parameter-metadata: + inputs: + cpu-energy-raw: + description: Raw energy used by CPU in kWh + unit: kWh + aggregation-method: sum + vcpu-ratio: + description: Ratio of vCPUs + unit: none + aggregation-method: none + outputs: + cpu-energy-kwh: + description: Corrected CPU energy in kWh + unit: kWh + aggregation-method: sum + sci-embodied: path: "builtin" method: SciEmbodied - "operational-carbon": + operational-carbon: method: Multiply path: builtin global-config: @@ -128,11 +177,20 @@ initialize: output-parameter: "carbon-operational" parameter-metadata: inputs: + cpu-energy-kwh: + description: Corrected CPU energy in kWh + unit: kWh + aggregation-method: sum grid/carbon-intensity: description: Carbon intensity for the grid unit: gCO2eq/kWh aggregation-method: avg - "sum-carbon": + outputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + sum-carbon: path: "builtin" method: Sum global-config: @@ -140,7 +198,22 @@ initialize: - carbon-operational - carbon-embodied output-parameter: carbon - "sci": + parameter-metadata: + inputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + carbon-embodied: + description: Embodied carbon footprint + unit: gCO2eq + aggregation-method: sum + outputs: + carbon: + description: Total carbon footprint + unit: gCO2eq + aggregation-method: sum + sci: path: "builtin" method: Sci global-config: @@ -151,7 +224,12 @@ initialize: description: expressed the final SCI value unit: none aggregation-method: sum - "time-sync": + outputs: + sci: + description: Scientific Carbon Intensity + unit: none + aggregation-method: none + time-sync: method: TimeSync path: "builtin" global-config: @@ -159,36 +237,27 @@ initialize: end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true - parameter-metadata: - inputs: - time-reserved: - description: time reserved for a component - unit: seconds - aggregation-method: avg - "group-by": - path: builtin - method: GroupBy tree: children: child-1: pipeline: - - mock-observations - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - instance-type + observe: + - mock-observations + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -198,49 +267,25 @@ tree: vcpus-total: 8 vcpus-allocated: 1 inputs: - - timestamp: "2023-12-12T00:00:00.000Z" - cloud/instance-type: A1 - cloud/region: uk-west - duration: 1 - cpu/utilization: 10 - requests: 30 - - timestamp: "2023-12-12T00:00:01.000Z" - duration: 5 - cpu/utilization: 20 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 40 - - timestamp: "2023-12-12T00:00:06.000Z" - duration: 7 - cpu/utilization: 15 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 30 - - timestamp: "2023-12-12T00:00:13.000Z" - duration: 30 - cloud/instance-type: A1 - cloud/region: uk-west - cpu/utilization: 15 - requests: 50 child-2: pipeline: - - mock-observations - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + observe: + - mock-observations + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -250,27 +295,3 @@ tree: vcpus-total: 8 vcpus-allocated: 1 inputs: - - timestamp: "2023-12-12T00:00:00.000Z" - duration: 1 - cpu/utilization: 30 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 30 - - timestamp: "2023-12-12T00:00:01.000Z" - duration: 5 - cpu/utilization: 28 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 40 - - timestamp: "2023-12-12T00:00:06.000Z" - duration: 7 - cpu/utilization: 40 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 50 - - timestamp: "2023-12-12T00:00:13.000Z" - duration: 30 - cpu/utilization: 33 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 60 diff --git a/manifests/examples/pipelines/scenario-1.yml b/manifests/examples/pipelines/scenario-1.yml new file mode 100644 index 000000000..9ab2d20f4 --- /dev/null +++ b/manifests/examples/pipelines/scenario-1.yml @@ -0,0 +1,34 @@ +name: demo +description: demo for observe feat +tags: +initialize: + plugins: + mock-observations: + kind: plugin + method: MockObservations + path: "builtin" + global-config: + timestamp-from: 2023-07-06T00:00 + timestamp-to: 2023-07-06T00:01 + duration: 60 + components: + - cloud/instance-type: A1 + - cloud/instance-type: B1 + generators: + common: + region: uk-west + common-key: common-val + randint: + cpu/utilization: + min: 1 + max: 99 + memory/utilization: + min: 1 + max: 99 +tree: + children: + child: + pipeline: + observe: + - mock-observations + inputs: null diff --git a/manifests/examples/pipelines/scenario-2.yml b/manifests/examples/pipelines/scenario-2.yml new file mode 100644 index 000000000..569cbcb05 --- /dev/null +++ b/manifests/examples/pipelines/scenario-2.yml @@ -0,0 +1,52 @@ +name: regroup demo +description: +initialize: + plugins: + interpolate: + method: Interpolation + path: "builtin" + global-config: + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: "cpu/utilization" + output-parameter: "cpu-factor" +tree: + children: + child: + pipeline: + observe: + regroup: + - cloud/region + - cloud/instance-type + inputs: + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 99 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 23 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + - timestamp: 2023-07-06T00:00 # note this time restarts at the start timstamp + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 11 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 67 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 1 diff --git a/manifests/examples/pipelines/scenario-3.yml b/manifests/examples/pipelines/scenario-3.yml new file mode 100644 index 000000000..f5710b553 --- /dev/null +++ b/manifests/examples/pipelines/scenario-3.yml @@ -0,0 +1,86 @@ +name: groupby +description: successful path +initialize: + plugins: + "sum": + path: "builtin" + method: Sum + global-config: + input-parameters: + - cpu/energy + - network/energy + output-parameter: energy +tree: + children: + my-app: + pipeline: + observe: + regroup: + - cloud/instance-type + - cloud/region + compute: + children: + uk-west: + inputs: + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 99 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 23 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 11 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 67 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 1 + uk-east: + inputs: + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-east + cpu/utilization: 9 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-east + cpu/utilization: 23 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-east + cpu/utilization: 12 + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-east + cpu/utilization: 11 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-east + cpu/utilization: 67 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-east + cpu/utilization: 1 diff --git a/manifests/examples/pipelines/scenario-4.yml b/manifests/examples/pipelines/scenario-4.yml new file mode 100644 index 000000000..151db09be --- /dev/null +++ b/manifests/examples/pipelines/scenario-4.yml @@ -0,0 +1,46 @@ +name: demo +description: +tags: +initialize: + plugins: + "sum": + path: "builtin" + method: Sum + global-config: + input-parameters: + - cpu/energy + - network/energy + output-parameter: energy-sum + "coefficient": + path: "builtin" + method: Coefficient + global-config: + input-parameter: energy + coefficient: 2 + output-parameter: energy-doubled + "multiply": + path: "builtin" + method: Multiply + global-config: + input-parameters: ["cpu/utilization", "duration"] + output-parameter: "cpu-times-duration" +tree: + children: + child-1: + pipeline: + observe: + compute: + - sum + - coefficient + - multiply + defaults: + cpu/thermal-design-power: 100 + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + cpu/energy: 20 + network/energy: 10 + energy: 5 diff --git a/manifests/examples/pipelines/scenario-5.yml b/manifests/examples/pipelines/scenario-5.yml new file mode 100644 index 000000000..6990c33e3 --- /dev/null +++ b/manifests/examples/pipelines/scenario-5.yml @@ -0,0 +1,46 @@ +name: demo +description: +tags: +initialize: + plugins: + mock-observations: + kind: plugin + method: MockObservations + path: "builtin" + global-config: + timestamp-from: 2023-07-06T00:00 + timestamp-to: 2023-07-06T00:01 + duration: 60 + components: + - cloud/instance-type: A1 + - cloud/instance-type: B1 + generators: + common: + region: uk-west + common-key: common-val + randint: + cpu/utilization: + min: 1 + max: 99 + memory/utilization: + min: 1 + max: 99 + sum: + path: "builtin" + method: Sum + global-config: + input-parameters: + - cpu/utilization + - memory/utilization + output-parameter: util-sum +tree: + children: + child: + pipeline: + observe: + - mock-observations + regroup: + - cloud/instance-type + compute: + - sum + inputs: null diff --git a/manifests/examples/pipelines/sci.yml b/manifests/examples/pipelines/sci.yml index 927f4eaf5..949d9e7df 100644 --- a/manifests/examples/pipelines/sci.yml +++ b/manifests/examples/pipelines/sci.yml @@ -79,18 +79,18 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sum-energy-components - - embodied-carbon - - operational-carbon - - sum-carbon - - sci - config: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sum-energy-components + - embodied-carbon + - operational-carbon + - sum-carbon + - sci defaults: cpu/thermal-design-power: 100 vcpus-total: 8 diff --git a/manifests/examples/pipelines/teads-curve.yml b/manifests/examples/pipelines/teads-curve.yml index 924384c20..1679a0976 100644 --- a/manifests/examples/pipelines/teads-curve.yml +++ b/manifests/examples/pipelines/teads-curve.yml @@ -49,12 +49,13 @@ tree: children: child: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio defaults: thermal-design-power: 100 vcpus-total: 8 diff --git a/manifests/examples/pipelines/zeros.yml b/manifests/examples/pipelines/zeros.yml index da5541789..f76a2c79f 100644 --- a/manifests/examples/pipelines/zeros.yml +++ b/manifests/examples/pipelines/zeros.yml @@ -106,22 +106,21 @@ tree: children: child-1: pipeline: - - sum-zero-and-one - - sum-zero-and-zero - - subtract-one-and-zero - - subtract-zero-and-zero - - subtract-zero-and-one - - coefficient-one-times-zero - - coefficient-zero-times-one - - coefficient-zero-times-zero - - multiply-one-times-zero - - multiply-zero-times-one - - exponent-one-to-zero - - exponent-zero-to-one - - exponent-zero-to-zero - - sci - config: - defaults: + compute: + - sum-zero-and-one + - sum-zero-and-zero + - subtract-one-and-zero + - subtract-zero-and-zero + - subtract-zero-and-one + - coefficient-one-times-zero + - coefficient-zero-times-one + - coefficient-zero-times-zero + - multiply-one-times-zero + - multiply-zero-times-one + - exponent-one-to-zero + - exponent-zero-to-one + - exponent-zero-to-zero + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 diff --git a/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml b/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml index ebf55ad73..92f85c0d7 100644 --- a/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml +++ b/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml @@ -5,8 +5,7 @@ description: >- tags: null aggregation: metrics: - "dummy-param": - method: sum + - dummy-param type: both initialize: plugins: @@ -88,9 +87,6 @@ initialize: end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true - group-by: - path: builtin - method: GroupBy execution: status: fail command: >- @@ -142,21 +138,20 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -192,21 +187,20 @@ tree: requests: 100 child-2: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 diff --git a/manifests/outputs/bugs/input-error-missing-duration.yaml b/manifests/outputs/bugs/input-error-missing-duration.yaml index 310a7d0ae..1006c1c0c 100644 --- a/manifests/outputs/bugs/input-error-missing-duration.yaml +++ b/manifests/outputs/bugs/input-error-missing-duration.yaml @@ -41,7 +41,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -76,7 +75,8 @@ tree: defaults: cpu/thermal-design-power: 100 pipeline: - - interpolate + compute: + - interpolate inputs: - timestamp: 2023-07-06T00:00 cpu/utilization: 20 diff --git a/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml b/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml index 56ed12cee..d137975b5 100644 --- a/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml +++ b/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml @@ -44,7 +44,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -70,10 +69,11 @@ execution: - typescript@5.2.2 - winston@3.11.0 - zod@3.22.4 - error: "RangeError: Maximum call stack size exceeded" + error: "InputValidationError: \"duration\" parameter is number must be greater than 0. Error code: too_small." tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: null diff --git a/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml b/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml index aaf987300..fef0fa246 100644 --- a/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml +++ b/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml @@ -41,7 +41,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -76,7 +75,8 @@ tree: defaults: cpu/thermal-design-power: 100 pipeline: - - wrong-name + compute: + - wrong-name inputs: - timestamp: 2023-07-06T00:00 duration: 1 diff --git a/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml b/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml index 1fd747ce9..7a0060013 100644 --- a/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml +++ b/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml @@ -41,7 +41,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -76,8 +75,9 @@ tree: defaults: cpu/thermal-design-power: 100 pipeline: - - interpolate - - multiply + compute: + - interpolate + - multiply inputs: - timestamp: 2023-07-06T00:00 duration: 1 diff --git a/manifests/outputs/bugs/pipeline-ordering-error.yaml b/manifests/outputs/bugs/pipeline-ordering-error.yaml index 5a47d8d3f..d1bd27d27 100644 --- a/manifests/outputs/bugs/pipeline-ordering-error.yaml +++ b/manifests/outputs/bugs/pipeline-ordering-error.yaml @@ -78,7 +78,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -111,13 +110,13 @@ tree: children: child-1: pipeline: - - interpolate - - correct-cpu-energy-for-vcpu-ratio - - calculate-vcpu-ratio - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - config: null + compute: + - interpolate + - correct-cpu-energy-for-vcpu-ratio + - calculate-vcpu-ratio + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 diff --git a/manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml b/manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml index 5c93301e2..486947ebd 100644 --- a/manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml +++ b/manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml @@ -60,7 +60,8 @@ tree: children: child: pipeline: - - sci-embodied + compute: + - sci-embodied defaults: device/emissions-embodied: 1533.12 time-reserved: 3600 diff --git a/manifests/outputs/plugins/coefficient/failure-invalid-config-input-param.yaml b/manifests/outputs/builtins/coefficient/failure-invalid-config-input-param.yaml similarity index 97% rename from manifests/outputs/plugins/coefficient/failure-invalid-config-input-param.yaml rename to manifests/outputs/builtins/coefficient/failure-invalid-config-input-param.yaml index 78af0bab9..7b6051dce 100644 --- a/manifests/outputs/plugins/coefficient/failure-invalid-config-input-param.yaml +++ b/manifests/outputs/builtins/coefficient/failure-invalid-config-input-param.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -62,9 +61,8 @@ tree: children: child: pipeline: - - coefficient - config: - sum: null + compute: + - coefficient inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/coefficient/failure-output-param-is-null.yaml b/manifests/outputs/builtins/coefficient/failure-output-param-is-null.yaml similarity index 97% rename from manifests/outputs/plugins/coefficient/failure-output-param-is-null.yaml rename to manifests/outputs/builtins/coefficient/failure-output-param-is-null.yaml index 91e6d064e..847e984e7 100644 --- a/manifests/outputs/plugins/coefficient/failure-output-param-is-null.yaml +++ b/manifests/outputs/builtins/coefficient/failure-output-param-is-null.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -62,9 +61,8 @@ tree: children: child: pipeline: - - coefficient - config: - sum: null + compute: + - coefficient inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/coefficient/success.yaml b/manifests/outputs/builtins/coefficient/success.yaml similarity index 98% rename from manifests/outputs/plugins/coefficient/success.yaml rename to manifests/outputs/builtins/coefficient/success.yaml index c560d0411..14a235709 100644 --- a/manifests/outputs/plugins/coefficient/success.yaml +++ b/manifests/outputs/builtins/coefficient/success.yaml @@ -28,7 +28,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -60,8 +59,6 @@ tree: child: pipeline: - coefficient - config: - sum: null inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml similarity index 98% rename from manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml rename to manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml index d1a5f2cfb..6945e6012 100644 --- a/manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml +++ b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml @@ -66,8 +66,8 @@ tree: children: child: pipeline: - - cloud-metadata - config: null + compute: + - cloud-metadata inputs: - timestamp: 2023-07-06T00:00 cloud/vendor: aws diff --git a/manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml similarity index 97% rename from manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml rename to manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml index 79a4960a5..333d9e1ce 100644 --- a/manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml +++ b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml @@ -66,8 +66,8 @@ tree: children: child: pipeline: - - cloud-metadata - config: null + compute: + - cloud-metadata inputs: - timestamp: 2023-07-06T00:00 duration: 100 diff --git a/manifests/outputs/plugins/csv-lookup/cloud-metadata/success.yaml b/manifests/outputs/builtins/csv-lookup/cloud-metadata/success.yaml similarity index 98% rename from manifests/outputs/plugins/csv-lookup/cloud-metadata/success.yaml rename to manifests/outputs/builtins/csv-lookup/cloud-metadata/success.yaml index 098a0d1c6..37fab5816 100644 --- a/manifests/outputs/plugins/csv-lookup/cloud-metadata/success.yaml +++ b/manifests/outputs/builtins/csv-lookup/cloud-metadata/success.yaml @@ -62,8 +62,8 @@ tree: children: child: pipeline: - - cloud-metadata - config: null + compute: + - cloud-metadata inputs: - timestamp: 2023-07-06T00:00 cloud/vendor: aws diff --git a/manifests/outputs/plugins/csv-lookup/region-metadata/failure-missing-column.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-column.yaml similarity index 97% rename from manifests/outputs/plugins/csv-lookup/region-metadata/failure-missing-column.yaml rename to manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-column.yaml index 4db0b6b2f..842080a77 100644 --- a/manifests/outputs/plugins/csv-lookup/region-metadata/failure-missing-column.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-column.yaml @@ -66,7 +66,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/csv-lookup/region-metadata/failure-missing-output.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-output.yaml similarity index 97% rename from manifests/outputs/plugins/csv-lookup/region-metadata/failure-missing-output.yaml rename to manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-output.yaml index bdaa3b6ad..32ca690d9 100644 --- a/manifests/outputs/plugins/csv-lookup/region-metadata/failure-missing-output.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-output.yaml @@ -32,7 +32,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -65,7 +64,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/csv-lookup/region-metadata/success-renaming.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/success-renaming.yaml similarity index 98% rename from manifests/outputs/plugins/csv-lookup/region-metadata/success-renaming.yaml rename to manifests/outputs/builtins/csv-lookup/region-metadata/success-renaming.yaml index e7e04b535..2f6f23dcf 100644 --- a/manifests/outputs/plugins/csv-lookup/region-metadata/success-renaming.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/success-renaming.yaml @@ -33,7 +33,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -64,7 +63,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/csv-lookup/region-metadata/success.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/success.yaml similarity index 98% rename from manifests/outputs/plugins/csv-lookup/region-metadata/success.yaml rename to manifests/outputs/builtins/csv-lookup/region-metadata/success.yaml index e8d31f661..f8bd9fdba 100644 --- a/manifests/outputs/plugins/csv-lookup/region-metadata/success.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/success.yaml @@ -31,7 +31,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -62,7 +61,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/csv-lookup/tdp-finder/failure-missing-input-param.yaml b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yaml similarity index 97% rename from manifests/outputs/plugins/csv-lookup/tdp-finder/failure-missing-input-param.yaml rename to manifests/outputs/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yaml index f842a3756..ce268cd02 100644 --- a/manifests/outputs/plugins/csv-lookup/tdp-finder/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yaml @@ -32,7 +32,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -65,8 +64,8 @@ tree: children: child: pipeline: - - tdp-finder - config: null + compute: + - tdp-finder inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/plugins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml similarity index 98% rename from manifests/outputs/plugins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml rename to manifests/outputs/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml index 7ccf90ff6..28dc0b1a0 100644 --- a/manifests/outputs/plugins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml +++ b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml @@ -65,8 +65,8 @@ tree: children: child: pipeline: - - tdp-finder - config: null + compute: + - tdp-finder inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/plugins/csv-lookup/tdp-finder/success.yaml b/manifests/outputs/builtins/csv-lookup/tdp-finder/success.yaml similarity index 97% rename from manifests/outputs/plugins/csv-lookup/tdp-finder/success.yaml rename to manifests/outputs/builtins/csv-lookup/tdp-finder/success.yaml index 249d2d3f4..f085664ff 100644 --- a/manifests/outputs/plugins/csv-lookup/tdp-finder/success.yaml +++ b/manifests/outputs/builtins/csv-lookup/tdp-finder/success.yaml @@ -61,8 +61,8 @@ tree: children: child: pipeline: - - tdp-finder - config: null + compute: + - tdp-finder inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/plugins/divide/failure-invalid-config-denominator.yaml b/manifests/outputs/builtins/divide/failure-invalid-config-denominator.yaml similarity index 97% rename from manifests/outputs/plugins/divide/failure-invalid-config-denominator.yaml rename to manifests/outputs/builtins/divide/failure-invalid-config-denominator.yaml index 8b98a5a27..b1149c505 100644 --- a/manifests/outputs/plugins/divide/failure-invalid-config-denominator.yaml +++ b/manifests/outputs/builtins/divide/failure-invalid-config-denominator.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -62,9 +61,8 @@ tree: children: child: pipeline: - - divide - config: - divide: null + compute: + - divide inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/divide/failure-missing-numerator.yaml b/manifests/outputs/builtins/divide/failure-missing-numerator.yaml similarity index 98% rename from manifests/outputs/plugins/divide/failure-missing-numerator.yaml rename to manifests/outputs/builtins/divide/failure-missing-numerator.yaml index 41ac4b1ea..6c714bf18 100644 --- a/manifests/outputs/plugins/divide/failure-missing-numerator.yaml +++ b/manifests/outputs/builtins/divide/failure-missing-numerator.yaml @@ -28,7 +28,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -61,7 +60,8 @@ tree: children: child: pipeline: - - divide + compute: + - divide config: divide: null defaults: diff --git a/manifests/outputs/plugins/divide/success-denominator-equal-zero.yaml b/manifests/outputs/builtins/divide/success-denominator-equal-zero.yaml similarity index 96% rename from manifests/outputs/plugins/divide/success-denominator-equal-zero.yaml rename to manifests/outputs/builtins/divide/success-denominator-equal-zero.yaml index 2384df53b..25033c32c 100644 --- a/manifests/outputs/plugins/divide/success-denominator-equal-zero.yaml +++ b/manifests/outputs/builtins/divide/success-denominator-equal-zero.yaml @@ -39,7 +39,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -70,10 +69,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide - config: - divide: null + compute: + - cloud-metadata + - divide defaults: cloud/vendor: aws cloud/instance-type: m5n.large diff --git a/manifests/outputs/plugins/divide/success.yaml b/manifests/outputs/builtins/divide/success.yaml similarity index 96% rename from manifests/outputs/plugins/divide/success.yaml rename to manifests/outputs/builtins/divide/success.yaml index 3e43a85de..f7bab8b04 100644 --- a/manifests/outputs/plugins/divide/success.yaml +++ b/manifests/outputs/builtins/divide/success.yaml @@ -39,7 +39,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -70,10 +69,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide - config: - divide: null + compute: + - cloud-metadata + - divide defaults: cloud/vendor: aws cloud/instance-type: m5n.large diff --git a/manifests/outputs/plugins/exponent/success.yaml b/manifests/outputs/builtins/exponent/success.yaml similarity index 97% rename from manifests/outputs/plugins/exponent/success.yaml rename to manifests/outputs/builtins/exponent/success.yaml index 7ad94e5bc..8586b57f6 100644 --- a/manifests/outputs/plugins/exponent/success.yaml +++ b/manifests/outputs/builtins/exponent/success.yaml @@ -28,7 +28,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -59,9 +58,8 @@ tree: children: child: pipeline: - - exponent - config: - exponent: null + compute: + - exponent inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/interpolation/interpolation.yaml b/manifests/outputs/builtins/interpolation/interpolation.yaml similarity index 97% rename from manifests/outputs/plugins/interpolation/interpolation.yaml rename to manifests/outputs/builtins/interpolation/interpolation.yaml index 8eff7b1f1..2569b5eb4 100644 --- a/manifests/outputs/plugins/interpolation/interpolation.yaml +++ b/manifests/outputs/builtins/interpolation/interpolation.yaml @@ -38,7 +38,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -69,7 +68,8 @@ tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/interpolation/success.yaml b/manifests/outputs/builtins/interpolation/success.yaml similarity index 97% rename from manifests/outputs/plugins/interpolation/success.yaml rename to manifests/outputs/builtins/interpolation/success.yaml index cbfc92011..80def13d9 100644 --- a/manifests/outputs/plugins/interpolation/success.yaml +++ b/manifests/outputs/builtins/interpolation/success.yaml @@ -38,7 +38,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -69,7 +68,8 @@ tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/mock-observations/failure-invalid-config-cpu-range.yaml b/manifests/outputs/builtins/mock-observations/failure-invalid-config-cpu-range.yaml similarity index 97% rename from manifests/outputs/plugins/mock-observations/failure-invalid-config-cpu-range.yaml rename to manifests/outputs/builtins/mock-observations/failure-invalid-config-cpu-range.yaml index fda4ddba7..272d9940e 100644 --- a/manifests/outputs/plugins/mock-observations/failure-invalid-config-cpu-range.yaml +++ b/manifests/outputs/builtins/mock-observations/failure-invalid-config-cpu-range.yaml @@ -47,7 +47,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -80,5 +79,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: null diff --git a/manifests/outputs/plugins/mock-observations/failure-invalid-memory-utilization-range.yaml b/manifests/outputs/builtins/mock-observations/failure-invalid-memory-utilization-range.yaml similarity index 97% rename from manifests/outputs/plugins/mock-observations/failure-invalid-memory-utilization-range.yaml rename to manifests/outputs/builtins/mock-observations/failure-invalid-memory-utilization-range.yaml index cfb073fc6..e7c7e6180 100644 --- a/manifests/outputs/plugins/mock-observations/failure-invalid-memory-utilization-range.yaml +++ b/manifests/outputs/builtins/mock-observations/failure-invalid-memory-utilization-range.yaml @@ -45,7 +45,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -78,5 +77,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: null diff --git a/manifests/outputs/plugins/mock-observations/failure-missing-timestamp-from-param.yaml b/manifests/outputs/builtins/mock-observations/failure-missing-timestamp-from-param.yaml similarity index 97% rename from manifests/outputs/plugins/mock-observations/failure-missing-timestamp-from-param.yaml rename to manifests/outputs/builtins/mock-observations/failure-missing-timestamp-from-param.yaml index 184e86230..86f4afd9e 100644 --- a/manifests/outputs/plugins/mock-observations/failure-missing-timestamp-from-param.yaml +++ b/manifests/outputs/builtins/mock-observations/failure-missing-timestamp-from-param.yaml @@ -44,7 +44,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -77,5 +76,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: null diff --git a/manifests/outputs/builtins/mock-observations/success.yaml b/manifests/outputs/builtins/mock-observations/success.yaml new file mode 100644 index 000000000..bcc5ee034 --- /dev/null +++ b/manifests/outputs/builtins/mock-observations/success.yaml @@ -0,0 +1,358 @@ +name: mock-observation-demo +description: a manifest demonstrating how to use the mock observations feature +tags: null +initialize: + plugins: + mock-observations: + path: builtin + method: MockObservations + global-config: + timestamp-from: 2023-07-06T00:00 + timestamp-to: 2023-07-06T00:10 + duration: 60 + components: + - cloud/instance-type: A1 + - cloud/instance-type: B1 + generators: + common: + region: uk-west + common-key: common-val + randint: + cpu/utilization: + min: 1 + max: 99 + memory/utilization: + min: 1 + max: 99 +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/builtins/mock-observations/success.yml -o + manifests/outputs/builtins/mock-observations/success + environment: + if-version: 0.5.0 + os: macOS + os-version: '14.5' + node-version: 18.14.2 + date-time: 2024-08-02T15:04:18.262Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.16' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + status: success +tree: + children: + child: + pipeline: + observe: + - mock-observations + inputs: + - timestamp: '2023-07-06T00:00:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:01:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:02:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:03:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:04:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:05:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:06:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:07:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:08:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:09:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:00:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:01:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:02:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:03:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:04:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:05:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:06:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:07:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:08:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:09:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + outputs: + - timestamp: '2023-07-06T00:00:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:01:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:02:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:03:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:04:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:05:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:06:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:07:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:08:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:09:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:00:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:01:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:02:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:03:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:04:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:05:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:06:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:07:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:08:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:09:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' diff --git a/manifests/outputs/plugins/multiply/failure-input-parameter-is-missing.yaml b/manifests/outputs/builtins/multiply/failure-input-parameter-is-missing.yaml similarity index 97% rename from manifests/outputs/plugins/multiply/failure-input-parameter-is-missing.yaml rename to manifests/outputs/builtins/multiply/failure-input-parameter-is-missing.yaml index 8c12556fb..80738e80c 100644 --- a/manifests/outputs/plugins/multiply/failure-input-parameter-is-missing.yaml +++ b/manifests/outputs/builtins/multiply/failure-input-parameter-is-missing.yaml @@ -30,7 +30,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -63,9 +62,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: null + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/multiply/success-with-multiple-inputs.yaml b/manifests/outputs/builtins/multiply/success-with-multiple-inputs.yaml similarity index 97% rename from manifests/outputs/plugins/multiply/success-with-multiple-inputs.yaml rename to manifests/outputs/builtins/multiply/success-with-multiple-inputs.yaml index b9b2470c3..359e25a06 100644 --- a/manifests/outputs/plugins/multiply/success-with-multiple-inputs.yaml +++ b/manifests/outputs/builtins/multiply/success-with-multiple-inputs.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -60,9 +59,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: null + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/multiply/success.yaml b/manifests/outputs/builtins/multiply/success.yaml similarity index 97% rename from manifests/outputs/plugins/multiply/success.yaml rename to manifests/outputs/builtins/multiply/success.yaml index c2925f183..1f93a9140 100644 --- a/manifests/outputs/plugins/multiply/success.yaml +++ b/manifests/outputs/builtins/multiply/success.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -60,9 +59,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: null + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/regex/failure-missing-input-param.yaml b/manifests/outputs/builtins/regex/failure-missing-input-param.yaml similarity index 97% rename from manifests/outputs/plugins/regex/failure-missing-input-param.yaml rename to manifests/outputs/builtins/regex/failure-missing-input-param.yaml index 5ea08772c..cd578218f 100644 --- a/manifests/outputs/plugins/regex/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/regex/failure-missing-input-param.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -62,9 +61,8 @@ tree: children: child: pipeline: - - regex - config: - regex: null + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/regex/success.yaml b/manifests/outputs/builtins/regex/success.yaml similarity index 92% rename from manifests/outputs/plugins/regex/success.yaml rename to manifests/outputs/builtins/regex/success.yaml index 745e3cf71..22988eead 100644 --- a/manifests/outputs/plugins/regex/success.yaml +++ b/manifests/outputs/builtins/regex/success.yaml @@ -28,7 +28,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -59,9 +58,8 @@ tree: children: child: pipeline: - - regex - config: - regex: null + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 @@ -75,5 +73,4 @@ tree: Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz cpu/name: >- - Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® - E5-2673 v4 2.3 GHz, + Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz, Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz diff --git a/manifests/outputs/plugins/sci-embodied/failure-invalid-default-emission-value.yaml b/manifests/outputs/builtins/sci-embodied/failure-invalid-default-emission-value.yaml similarity index 97% rename from manifests/outputs/plugins/sci-embodied/failure-invalid-default-emission-value.yaml rename to manifests/outputs/builtins/sci-embodied/failure-invalid-default-emission-value.yaml index b1784d516..f6af79041 100644 --- a/manifests/outputs/plugins/sci-embodied/failure-invalid-default-emission-value.yaml +++ b/manifests/outputs/builtins/sci-embodied/failure-invalid-default-emission-value.yaml @@ -28,7 +28,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -61,7 +60,8 @@ tree: children: child: pipeline: - - sci-embodied + compute: + - sci-embodied defaults: device/emissions-embodied: fail time-reserved: 3600 diff --git a/manifests/outputs/plugins/sci-embodied/failure-missing-expected-lifespan.yaml b/manifests/outputs/builtins/sci-embodied/failure-missing-expected-lifespan.yaml similarity index 97% rename from manifests/outputs/plugins/sci-embodied/failure-missing-expected-lifespan.yaml rename to manifests/outputs/builtins/sci-embodied/failure-missing-expected-lifespan.yaml index 5959d2b21..43f22163e 100644 --- a/manifests/outputs/plugins/sci-embodied/failure-missing-expected-lifespan.yaml +++ b/manifests/outputs/builtins/sci-embodied/failure-missing-expected-lifespan.yaml @@ -25,7 +25,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -58,7 +57,8 @@ tree: children: child: pipeline: - - sci-embodied + compute: + - sci-embodied defaults: device/emissions-embodied: 1533.12 time-reserved: 3600 diff --git a/manifests/outputs/plugins/sci-embodied/success.yaml b/manifests/outputs/builtins/sci-embodied/success.yaml similarity index 97% rename from manifests/outputs/plugins/sci-embodied/success.yaml rename to manifests/outputs/builtins/sci-embodied/success.yaml index f7c4e8dc0..e91ab9212 100644 --- a/manifests/outputs/plugins/sci-embodied/success.yaml +++ b/manifests/outputs/builtins/sci-embodied/success.yaml @@ -24,7 +24,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -55,7 +54,8 @@ tree: children: child: pipeline: - - sci-embodied + compute: + - sci-embodied defaults: device/emissions-embodied: 1533.12 time-reserved: 3600 diff --git a/manifests/outputs/plugins/sci/failure-invalid-config-value.yaml b/manifests/outputs/builtins/sci/failure-invalid-config-value.yaml similarity index 98% rename from manifests/outputs/plugins/sci/failure-invalid-config-value.yaml rename to manifests/outputs/builtins/sci/failure-invalid-config-value.yaml index 463a855da..384fe14b1 100644 --- a/manifests/outputs/plugins/sci/failure-invalid-config-value.yaml +++ b/manifests/outputs/builtins/sci/failure-invalid-config-value.yaml @@ -26,7 +26,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -57,7 +56,8 @@ tree: children: child: pipeline: - - sci + compute: + - sci config: sci: functional-unit: 999 diff --git a/manifests/outputs/plugins/sci/failure-missing-input-param.yaml b/manifests/outputs/builtins/sci/failure-missing-input-param.yaml similarity index 98% rename from manifests/outputs/plugins/sci/failure-missing-input-param.yaml rename to manifests/outputs/builtins/sci/failure-missing-input-param.yaml index 0aed8267c..b0fbce2f8 100644 --- a/manifests/outputs/plugins/sci/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/sci/failure-missing-input-param.yaml @@ -28,7 +28,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -61,7 +60,8 @@ tree: children: child: pipeline: - - sci + compute: + - sci inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/sci/success.yml.yaml b/manifests/outputs/builtins/sci/success.yaml similarity index 98% rename from manifests/outputs/plugins/sci/success.yml.yaml rename to manifests/outputs/builtins/sci/success.yaml index e08d7e27e..9f36fc9bd 100644 --- a/manifests/outputs/plugins/sci/success.yml.yaml +++ b/manifests/outputs/builtins/sci/success.yaml @@ -26,7 +26,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -57,8 +56,8 @@ tree: children: child: pipeline: - - sci - config: null + compute: + - sci inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/shell/failure-invalid-command.yaml b/manifests/outputs/builtins/shell/failure-invalid-command.yaml similarity index 98% rename from manifests/outputs/plugins/shell/failure-invalid-command.yaml rename to manifests/outputs/builtins/shell/failure-invalid-command.yaml index ad11fecad..97d8bb6af 100644 --- a/manifests/outputs/plugins/shell/failure-invalid-command.yaml +++ b/manifests/outputs/builtins/shell/failure-invalid-command.yaml @@ -27,7 +27,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -60,7 +59,8 @@ tree: children: child: pipeline: - - shell + compute: + - shell inputs: - timestamp: "2023-11-02T10:35:31.820Z" duration: 3600 diff --git a/manifests/outputs/plugins/shell/success.yaml b/manifests/outputs/builtins/shell/success.yaml similarity index 97% rename from manifests/outputs/plugins/shell/success.yaml rename to manifests/outputs/builtins/shell/success.yaml index 0c53a27aa..b85cbfb95 100644 --- a/manifests/outputs/plugins/shell/success.yaml +++ b/manifests/outputs/builtins/shell/success.yaml @@ -26,7 +26,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -57,7 +56,8 @@ tree: children: child: pipeline: - - shell + compute: + - shell inputs: - timestamp: "2023-11-02T10:35:31.820Z" duration: 3600 diff --git a/manifests/outputs/plugins/subtract/success.yaml b/manifests/outputs/builtins/subtract/success.yaml similarity index 97% rename from manifests/outputs/plugins/subtract/success.yaml rename to manifests/outputs/builtins/subtract/success.yaml index ebf2197c5..5eb0a7bd3 100644 --- a/manifests/outputs/plugins/subtract/success.yaml +++ b/manifests/outputs/builtins/subtract/success.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -60,9 +59,8 @@ tree: children: child: pipeline: - - subtract - config: - subtract: null + compute: + - subtract inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/sum/failure-missing-input-param.yaml b/manifests/outputs/builtins/sum/failure-missing-input-param.yaml similarity index 97% rename from manifests/outputs/plugins/sum/failure-missing-input-param.yaml rename to manifests/outputs/builtins/sum/failure-missing-input-param.yaml index 85838df45..e2aaf8158 100644 --- a/manifests/outputs/plugins/sum/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/sum/failure-missing-input-param.yaml @@ -30,7 +30,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -63,9 +62,8 @@ tree: children: child: pipeline: - - sum - config: - sum: null + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/sum/failure-missing-output-param.yaml b/manifests/outputs/builtins/sum/failure-missing-output-param.yaml similarity index 97% rename from manifests/outputs/plugins/sum/failure-missing-output-param.yaml rename to manifests/outputs/builtins/sum/failure-missing-output-param.yaml index 624bf3fdf..d7a4d9302 100644 --- a/manifests/outputs/plugins/sum/failure-missing-output-param.yaml +++ b/manifests/outputs/builtins/sum/failure-missing-output-param.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -62,9 +61,8 @@ tree: children: child: pipeline: - - sum - config: - sum: null + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/sum/success.yaml b/manifests/outputs/builtins/sum/success.yaml similarity index 97% rename from manifests/outputs/plugins/sum/success.yaml rename to manifests/outputs/builtins/sum/success.yaml index 77b0a3f2d..6b5b4d973 100644 --- a/manifests/outputs/plugins/sum/success.yaml +++ b/manifests/outputs/builtins/sum/success.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -60,9 +59,8 @@ tree: children: child: pipeline: - - sum - config: - sum: null + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/time-sync/failure-config-start-later-end.yaml b/manifests/outputs/builtins/time-sync/failure-config-start-later-end.yaml similarity index 98% rename from manifests/outputs/plugins/time-sync/failure-config-start-later-end.yaml rename to manifests/outputs/builtins/time-sync/failure-config-start-later-end.yaml index 81a430414..bb5ae79fb 100644 --- a/manifests/outputs/plugins/time-sync/failure-config-start-later-end.yaml +++ b/manifests/outputs/builtins/time-sync/failure-config-start-later-end.yaml @@ -62,8 +62,8 @@ tree: children: child: pipeline: - - time-sync - config: null + compute: + - time-sync inputs: - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 diff --git a/manifests/outputs/plugins/time-sync/success.yaml b/manifests/outputs/builtins/time-sync/success.yaml similarity index 98% rename from manifests/outputs/plugins/time-sync/success.yaml rename to manifests/outputs/builtins/time-sync/success.yaml index 17e1a71d1..3ed64a0e3 100644 --- a/manifests/outputs/plugins/time-sync/success.yaml +++ b/manifests/outputs/builtins/time-sync/success.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -60,8 +59,8 @@ tree: children: child: pipeline: - - time-sync - config: null + compute: + - time-sync inputs: - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 diff --git a/manifests/outputs/divide.yaml b/manifests/outputs/divide.yaml deleted file mode 100644 index 85c681d1f..000000000 --- a/manifests/outputs/divide.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: divide -description: success path -tags: null -initialize: - plugins: - cloud-metadata: - path: builtin - method: CSVLookup - global-config: - filepath: >- - https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv - query: - instance-class: cloud/instance-type - output: - - cpu-cores-utilized - - vcpus-allocated - divide: - path: builtin - method: Divide - global-config: - numerator: vcpus-allocated - denominator: 2 - output: cpu/number-cores -execution: - command: >- - /home/joe/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /home/joe/Code/if/src/index.ts -m ./manifests/plugins/divide/success.yml -o - ./manifests/outputs/divide - environment: - if-version: 0.4.0 - os: linux - os-version: 5.15.0-113-generic - node-version: 21.4.0 - date-time: 2024-07-03T14:02:26.789Z (UTC) - dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - status: success -tree: - children: - child: - pipeline: - - cloud-metadata - - divide - config: - divide: null - defaults: - cloud/vendor: aws - cloud/instance-type: m5n.large - cpu/name: Intel® Core™ i7-1185G7 - inputs: - - timestamp: 2023-08-06T00:00 - duration: 3600 - cpu/utilization: 80 - outputs: - - timestamp: 2023-08-06T00:00 - duration: 3600 - cpu/utilization: 80 - cloud/vendor: aws - cloud/instance-type: m5n.large - cpu/name: Intel® Core™ i7-1185G7 - vcpus-allocated: 2 - cpu/number-cores: 1 diff --git a/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml b/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml index 870be308a..aef8ef6c5 100644 --- a/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml +++ b/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml @@ -2,8 +2,7 @@ name: Aggregation description: Fails with invalid metric. aggregation: metrics: - "test": - method: sum + - test type: both initialize: plugins: @@ -70,7 +69,8 @@ tree: children: application: pipeline: - - cloud-metadata + compute: + - cloud-metadata children: uk-west: children: diff --git a/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml b/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml index 4a3da85ae..2dfc9fe80 100644 --- a/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml +++ b/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml @@ -2,8 +2,7 @@ name: Aggregation description: Fails with missing metric in inputs. aggregation: metrics: - "cpu/utilization": - method: sum + - cpu/utilization type: both initialize: plugins: @@ -70,7 +69,8 @@ tree: children: application: pipeline: - - cloud-metadata + compute: + - cloud-metadata children: uk-west: children: diff --git a/manifests/outputs/features/aggregate-horizontal.yaml b/manifests/outputs/features/aggregate-horizontal.yaml index f870bb4d2..d595d5bdb 100644 --- a/manifests/outputs/features/aggregate-horizontal.yaml +++ b/manifests/outputs/features/aggregate-horizontal.yaml @@ -2,8 +2,7 @@ name: Aggregation description: Apply `horizontal` aggregation aggregation: metrics: - "cpu/utilization": - method: sum + - cpu/utilization type: horizontal initialize: plugins: @@ -18,6 +17,12 @@ initialize: output: - cpu-cores-utilized - vcpus-allocated + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -36,7 +41,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -67,7 +71,8 @@ tree: children: application: pipeline: - - cloud-metadata + compute: + - cloud-metadata children: uk-west: children: diff --git a/manifests/outputs/features/aggregate-vertical.yaml b/manifests/outputs/features/aggregate-vertical.yaml index 0fd5b170a..9213522af 100644 --- a/manifests/outputs/features/aggregate-vertical.yaml +++ b/manifests/outputs/features/aggregate-vertical.yaml @@ -2,8 +2,7 @@ name: Aggregation description: Apply `vertical` aggregation aggregation: metrics: - "cpu/utilization": - method: sum + - cpu/utilization type: vertical initialize: plugins: @@ -18,6 +17,12 @@ initialize: output: - cpu-cores-utilized - vcpus-allocated + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -36,7 +41,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -67,7 +71,8 @@ tree: children: application: pipeline: - - cloud-metadata + compute: + - cloud-metadata children: uk-west: children: diff --git a/manifests/outputs/features/aggregate.yaml b/manifests/outputs/features/aggregate.yaml index 35ab21423..cb9ed29dc 100644 --- a/manifests/outputs/features/aggregate.yaml +++ b/manifests/outputs/features/aggregate.yaml @@ -2,8 +2,7 @@ name: Aggregation description: Apply both `horizontal` and `vertical` aggregations aggregation: metrics: - "cpu/utilization": - method: sum + - cpu/utilization type: both initialize: plugins: @@ -18,6 +17,12 @@ initialize: output: - cpu-cores-utilized - vcpus-allocated + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -36,7 +41,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -67,7 +71,8 @@ tree: children: application: pipeline: - - cloud-metadata + compute: + - cloud-metadata children: uk-west: children: diff --git a/manifests/outputs/plugins/groupby/failure-invalid-config-group.yaml b/manifests/outputs/features/failure-invalid-regroup.yaml similarity index 60% rename from manifests/outputs/plugins/groupby/failure-invalid-config-group.yaml rename to manifests/outputs/features/failure-invalid-regroup.yaml index 22506aa2e..b3f3e722c 100644 --- a/manifests/outputs/plugins/groupby/failure-invalid-config-group.yaml +++ b/manifests/outputs/features/failure-invalid-regroup.yaml @@ -1,35 +1,31 @@ -name: groupby -description: failure when `config->group-by->group` is not an array +name: regroup +description: failure when `regroup` is not an array initialize: - plugins: - group-by: - path: builtin - method: GroupBy + plugins: {} execution: status: fail command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/groupby/failure-invalid-config-group.yml -o - manifests/outputs/plugins/groupby/failure-invalid-config-group + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/features/regroup/failure-invalid-regroup.yml -o + manifests/outputs/features/failure-invalid-regroup.yaml environment: - if-version: 0.4.0 + if-version: 0.5.0 os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-02T19:11:23.166Z (UTC) + os-version: 13.6.7 + node-version: 18.20.0 + date-time: 2024-08-05T14:53:00.560Z (UTC) dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.16' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -38,6 +34,7 @@ execution: - fixpack@4.0.0 - gts@5.2.0 - husky@8.0.3 + - if-eco-ci-plugin@v0.0.1 extraneous -> file:../../if-eco-ci-plugin - jest@29.7.0 - js-yaml@4.1.0 - lint-staged@15.2.2 @@ -49,18 +46,15 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 error: >- - InputValidationError: "group" parameter is expected array, received string. + InputValidationError: "groups" parameter is expected array, received string. Error code: invalid_type. tree: children: my-app: pipeline: - - group-by - config: - group-by: - group: cloud/region + regroup: cloud/region inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/plugins/groupby/failure-missing-cloud-instance-type.yaml b/manifests/outputs/features/failure-missing-cloud-instance-type.yaml similarity index 92% rename from manifests/outputs/plugins/groupby/failure-missing-cloud-instance-type.yaml rename to manifests/outputs/features/failure-missing-cloud-instance-type.yaml index e6b2ea8ea..741544461 100644 --- a/manifests/outputs/plugins/groupby/failure-missing-cloud-instance-type.yaml +++ b/manifests/outputs/features/failure-missing-cloud-instance-type.yaml @@ -1,10 +1,7 @@ name: groupby description: null initialize: - plugins: - group-by: - path: builtin - method: GroupBy + plugins: {} execution: status: fail command: >- @@ -24,7 +21,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -55,12 +51,9 @@ tree: children: my-app: pipeline: - - group-by - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/pipelines/cloud-metadata-divide.yaml b/manifests/outputs/pipelines/cloud-metadata-divide.yaml index f85725c5c..02ff19b86 100644 --- a/manifests/outputs/pipelines/cloud-metadata-divide.yaml +++ b/manifests/outputs/pipelines/cloud-metadata-divide.yaml @@ -39,7 +39,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -70,10 +69,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide - config: - divide: null + compute: + - cloud-metadata + - divide defaults: cloud/vendor: aws cloud/instance-type: m5n.large diff --git a/manifests/outputs/pipelines/generics.yaml b/manifests/outputs/pipelines/generics.yaml new file mode 100644 index 000000000..613a65f83 --- /dev/null +++ b/manifests/outputs/pipelines/generics.yaml @@ -0,0 +1,239 @@ +name: generics +description: >- + a pipeline that does arbitrary calculations using our generic arithmetic + builtins +tags: null +initialize: + plugins: + interpolate: + path: builtin + method: Interpolation + global-config: + method: linear + x: + - 0 + - 10 + - 50 + - 100 + 'y': + - 0.12 + - 0.32 + - 0.75 + - 1.02 + input-parameter: cpu/utilization + output-parameter: cpu-factor + cpu-factor-to-wattage: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-factor + - cpu/thermal-design-power + output-parameter: cpu-wattage + wattage-times-duration: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-wattage + - duration + output-parameter: cpu-wattage-times-duration + wattage-to-energy-kwh: + path: builtin + method: Divide + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + calculate-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + correct-cpu-energy-for-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh + coefficient: + path: builtin + method: Coefficient + global-config: + input-parameter: cpu-energy-kwh + coefficient: 2 + output-parameter: energy-doubled + multiply: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu/utilization + - duration + output-parameter: cpu-times-duration +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/generics.yml -o + manifests/outputs/pipelines/generics + environment: + if-version: 0.5.0 + os: macOS + os-version: '14.5' + node-version: 18.14.2 + date-time: 2024-07-17T20:30:54.004Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.10' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.22.4 + status: success +tree: + children: + child-1: + pipeline: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - coefficient + - multiply + defaults: + cpu/thermal-design-power: 100 + vcpus-allocated: 1 + vcpus-total: 8 + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 10 + energy: 5 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 10 + energy: 5 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 10 + energy: 5 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 10 + energy: 5 + outputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 10 + energy: 5 + cpu/thermal-design-power: 100 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.75 + cpu-wattage: 75 + cpu-wattage-times-duration: 75 + cpu-energy-raw: 0.000020833333333333333 + vcpu-ratio: 8 + cpu-energy-kwh: 0.0000026041666666666666 + energy-doubled: 0.000005208333333333333 + cpu-times-duration: 50 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 10 + energy: 5 + cpu/thermal-design-power: 100 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.4275 + cpu-wattage: 42.75 + cpu-wattage-times-duration: 213.75 + cpu-energy-raw: 0.000059375 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000007421875 + energy-doubled: 0.00001484375 + cpu-times-duration: 100 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 10 + energy: 5 + cpu/thermal-design-power: 100 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.37375 + cpu-wattage: 37.375 + cpu-wattage-times-duration: 261.625 + cpu-energy-raw: 0.00007267361111111111 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000009084201388888889 + energy-doubled: 0.000018168402777777778 + cpu-times-duration: 105 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 10 + energy: 5 + cpu/thermal-design-power: 100 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.37375 + cpu-wattage: 37.375 + cpu-wattage-times-duration: 1121.25 + cpu-energy-raw: 0.00031145833333333335 + vcpu-ratio: 8 + cpu-energy-kwh: 0.00003893229166666667 + energy-doubled: 0.00007786458333333334 + cpu-times-duration: 450 diff --git a/manifests/outputs/pipelines/instance-metadata.yaml b/manifests/outputs/pipelines/instance-metadata.yaml new file mode 100644 index 000000000..369d54c27 --- /dev/null +++ b/manifests/outputs/pipelines/instance-metadata.yaml @@ -0,0 +1,98 @@ +name: csv-demo +description: null +tags: null +initialize: + plugins: + cloud-instance-metadata: + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-azure-instances.csv + query: + instance-class: cloud/instance-type + output: '*' + extract-processor-name: + path: builtin + method: Regex + global-config: + parameter: cpu-model-name + match: /^([^,])+/g + output: cpu/name +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/instance-metadata.yml -o + manifests/outputs/pipelines/instance-metadata + environment: + if-version: 0.5.0 + os: macOS + os-version: '14.5' + node-version: 18.14.2 + date-time: 2024-07-19T06:31:27.411Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.10' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.22.4 + status: success +tree: + children: + child: + pipeline: + compute: + - cloud-instance-metadata + - extract-processor-name + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + cpu/energy: 0.001 + cloud/provider: gcp + cloud/region: asia-east + cloud/instance-type: Standard_A1_v2 + outputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + cpu/energy: 0.001 + cloud/provider: gcp + cloud/region: asia-east + cloud/instance-type: Standard_A1_v2 + cpu-cores-available: 52 + cpu-cores-utilized: 1 + cpu-manufacturer: Intel + cpu-model-name: >- + Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® + E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz + cpu-tdp: 205 + gpu-count: nan + gpu-model-name: nan + gpu-tdp: nan + memory-available: 2 + cpu/name: Intel® Xeon® Platinum 8272CL diff --git a/manifests/outputs/pipelines/mock-obs-time-sync.yaml b/manifests/outputs/pipelines/mock-obs-time-sync.yaml index 623305f0b..04eb881ec 100644 --- a/manifests/outputs/pipelines/mock-obs-time-sync.yaml +++ b/manifests/outputs/pipelines/mock-obs-time-sync.yaml @@ -169,219 +169,281 @@ tree: children: child-1: pipeline: - - mock-observations - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - time-sync + observe: + - mock-observations + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - time-sync defaults: cpu/thermal-design-power: 100 vcpus-total: 8 vcpus-allocated: 1 inputs: - - timestamp: "2023-12-12T00:00:00.000Z" + - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 cloud/region: uk-west - duration: 1 - cpu/utilization: 10 - - timestamp: "2023-12-12T00:00:01.000Z" - duration: 5 - cpu/utilization: 20 + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:01:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:02:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:03:00.000Z' cloud/instance-type: A1 cloud/region: uk-west - - timestamp: "2023-12-12T00:00:06.000Z" - duration: 7 - cpu/utilization: 15 + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:04:00.000Z' cloud/instance-type: A1 cloud/region: uk-west - - timestamp: "2023-12-12T00:00:13.000Z" - duration: 30 + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:05:00.000Z' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 15 + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:06:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:07:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:08:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:09:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 outputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: 14 + cpu/utilization: '*' cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.662 - cpu-wattage: 66.19999999999999 - cpu-wattage-times-duration: 203 - cpu-energy-raw: 0.0000563888888888889 - vcpu-ratio: 14.399999999999999 - cpu-energy-kwh: 0.000007048611111111113 + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' - timestamp: "2023-12-12T00:00:05.000Z" duration: 5 - cpu/utilization: 13 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.29907142857142854 - cpu-wattage: 29.907142857142862 - cpu-wattage-times-duration: 192.25 - cpu-energy-raw: 0.00005340277777777778 - vcpu-ratio: 6.171428571428571 - cpu-energy-kwh: 0.000006675347222222222 + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' - timestamp: "2023-12-12T00:00:10.000Z" duration: 5 - cpu/utilization: 12 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.1850952380952381 - cpu-wattage: 18.50952380952381 - cpu-wattage-times-duration: 186.875 - cpu-energy-raw: 0.00005190972222222222 - vcpu-ratio: 3.9619047619047616 - cpu-energy-kwh: 0.0000064887152777777775 + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' - timestamp: "2023-12-12T00:00:15.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 12 cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.06229166666666667 - cpu-wattage: 6.229166666666667 - cpu-wattage-times-duration: 186.875 - cpu-energy-raw: 0.00005190972222222223 - vcpu-ratio: 1.3333333333333333 - cpu-energy-kwh: 0.000006488715277777778 + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' - timestamp: "2023-12-12T00:00:20.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 12 cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.06229166666666667 - cpu-wattage: 6.229166666666667 - cpu-wattage-times-duration: 186.875 - cpu-energy-raw: 0.00005190972222222223 - vcpu-ratio: 1.3333333333333333 - cpu-energy-kwh: 0.000006488715277777778 + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' - timestamp: "2023-12-12T00:00:25.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 12 cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.06229166666666667 - cpu-wattage: 6.229166666666667 - cpu-wattage-times-duration: 186.875 - cpu-energy-raw: 0.00005190972222222223 - vcpu-ratio: 1.3333333333333333 - cpu-energy-kwh: 0.000006488715277777778 + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' - timestamp: "2023-12-12T00:00:30.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 12 cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.06229166666666667 - cpu-wattage: 6.229166666666667 - cpu-wattage-times-duration: 186.875 - cpu-energy-raw: 0.00005190972222222223 - vcpu-ratio: 1.3333333333333333 - cpu-energy-kwh: 0.000006488715277777778 + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' - timestamp: "2023-12-12T00:00:35.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 12 cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.06229166666666667 - cpu-wattage: 6.229166666666667 - cpu-wattage-times-duration: 186.875 - cpu-energy-raw: 0.00005190972222222223 - vcpu-ratio: 1.3333333333333333 - cpu-energy-kwh: 0.000006488715277777778 + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' - timestamp: "2023-12-12T00:00:40.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 9 - cpu/thermal-design-power: 60 + cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.037375 - cpu-wattage: 3.7375 - cpu-wattage-times-duration: 112.125 - cpu-energy-raw: 0.000031145833333333336 - vcpu-ratio: 0.8 - cpu-energy-kwh: 0.000003893229166666667 + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' - timestamp: "2023-12-12T00:00:45.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 0 - cpu/thermal-design-power: 0 + cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0 - cpu-wattage: 0 - cpu-wattage-times-duration: 0 - cpu-energy-raw: 0 - vcpu-ratio: 0 - cpu-energy-kwh: 0 + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' - timestamp: "2023-12-12T00:00:50.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 0 - cpu/thermal-design-power: 0 + cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0 - cpu-wattage: 0 - cpu-wattage-times-duration: 0 - cpu-energy-raw: 0 - vcpu-ratio: 0 - cpu-energy-kwh: 0 + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' - timestamp: "2023-12-12T00:00:55.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 0 - cpu/thermal-design-power: 0 + cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0 - cpu-wattage: 0 - cpu-wattage-times-duration: 0 - cpu-energy-raw: 0 - vcpu-ratio: 0 - cpu-energy-kwh: 0 + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' - timestamp: "2023-12-12T00:01:00.000Z" duration: 1 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 0 - cpu/thermal-design-power: 0 + cpu/thermal-design-power: 100 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0 - cpu-wattage: 0 - cpu-wattage-times-duration: 0 - cpu-energy-raw: 0 - vcpu-ratio: 0 - cpu-energy-kwh: 0 + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' diff --git a/manifests/outputs/pipelines/nesting.yaml b/manifests/outputs/pipelines/nesting.yaml new file mode 100644 index 000000000..792f41243 --- /dev/null +++ b/manifests/outputs/pipelines/nesting.yaml @@ -0,0 +1,1799 @@ +name: nesting +description: a manifest that includes nested child components +tags: + kind: web + complexity: moderate + category: on-premise +aggregation: + metrics: + - carbon + type: both +initialize: + plugins: + interpolate: + path: builtin + method: Interpolation + global-config: + method: linear + x: + - 0 + - 10 + - 50 + - 100 + "y": + - 0.12 + - 0.32 + - 0.75 + - 1.02 + input-parameter: cpu/utilization + output-parameter: cpu-factor + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg + outputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu-factor-to-wattage: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-factor + - cpu/thermal-design-power + output-parameter: cpu-wattage + parameter-metadata: + inputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu/thermal-design-power: + unit: kWh + description: thermal design power for a processor + aggregation-method: avg + outputs: + cpu-wattage: + unit: kWh + description: the energy used by the CPU + aggregation-method: sum + wattage-times-duration: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-wattage + - duration + output-parameter: cpu-wattage-times-duration + wattage-to-energy-kwh: + path: builtin + method: Divide + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + parameter-metadata: + inputs: + cpu-wattage-times-duration: + unit: kWh + description: CPU wattage multiplied by duration + aggregation-method: sum + outputs: + cpu-energy-raw: + unit: kWh + description: Raw energy used by CPU in kWh + aggregation-method: sum + calculate-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + parameter-metadata: + outputs: + vcpu-ratio: + unit: none + description: Ratio of vCPUs + aggregation-method: none + correct-cpu-energy-for-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh + sci-embodied: + path: builtin + method: SciEmbodied + operational-carbon: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-energy-kwh + - grid/carbon-intensity + output-parameter: carbon-operational + parameter-metadata: + inputs: + cpu-energy-kwh: + unit: kWh + description: Corrected CPU energy in kWh + aggregation-method: sum + grid/carbon-intensity: + unit: gCO2eq/kWh + description: Carbon intensity for the grid + aggregation-method: avg + outputs: + carbon-operational: + unit: gCO2eq + description: Operational carbon footprint + aggregation-method: sum + sci: + path: builtin + method: Sci + global-config: + functional-unit: requests + parameter-metadata: + inputs: + requests: + unit: none + description: expressed the final SCI value + aggregation-method: sum + sum-carbon: + path: builtin + method: Sum + global-config: + input-parameters: + - carbon-operational + - carbon-embodied + output-parameter: carbon + parameter-metadata: + inputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + carbon-embodied: + description: Embodied carbon footprint + unit: gCO2eq + aggregation-method: sum + outputs: + carbon: + description: Total carbon footprint + unit: gCO2eq + aggregation-method: sum + time-sync: + path: builtin + method: TimeSync + global-config: + start-time: "2023-12-12T00:00:00.000Z" + end-time: "2023-12-12T00:01:00.000Z" + interval: 5 + allow-padding: true + parameter-metadata: + inputs: + timestamp: + unit: RFC3339 + description: refers to the time of occurrence of the input + aggregation-method: none + duration: + unit: seconds + description: refers to the duration of the input + aggregation-method: sum + cloud/instance-type: + unit: none + description: type of Cloud Instance name used in the cloud provider APIs + aggregation-method: none + cloud/region: + unit: none + description: region cloud instance + aggregation-method: none + time-reserved: + unit: seconds + description: time reserved for a component + aggregation-method: avg + network/energy: + unit: kWh + description: Energy consumed by the Network of the component + aggregation-method: sum +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/nesting.yml -o + manifests/outputs/pipelines/nesting-1.yaml + environment: + if-version: 0.5.0 + os: macOS + os-version: "14.5" + node-version: 18.14.2 + date-time: 2024-07-31T13:17:29.944Z (UTC) + dependencies: + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.16" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + status: success +tree: + children: + child-0: + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-allocated: 1 + vcpus-total: 8 + pipeline: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + requests: 50 + - timestamp: "2023-12-12T00:00:01.000Z" + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 60 + - timestamp: "2023-12-12T00:00:06.000Z" + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 70 + - timestamp: "2023-12-12T00:00:13.000Z" + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + requests: 55 + outputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: 22 + network/energy: 0.0000018 + requests: 98 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 2759.6159999999995 + time-reserved: 2880 + device/expected-lifespan: 170294400 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.4065 + cpu-wattage: 109.19999999999999 + cpu-wattage-times-duration: 246 + cpu-energy-raw: 0.00006833333333333335 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000008541666666666668 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.006833333333333334 + carbon: 0.0068434614408929475 + sci: 0.00006983123919278518 + - timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + cpu/utilization: 13 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 7.714285714285713e-7 + requests: 52 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 1182.6925714285712 + time-reserved: 2880 + device/expected-lifespan: 72983314.28571428 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.30975 + cpu-wattage: 29.907142857142862 + cpu-wattage-times-duration: 192.25 + cpu-energy-raw: 0.00005340277777777778 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006675347222222222 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005340277777777777 + carbon: 0.005350405885337391 + sci: 0.0001028924208718729 + - timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + cpu/utilization: 12 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 4.952380952380952e-7 + requests: 33.666666666666664 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 759.2594285714285 + time-reserved: 2880 + device/expected-lifespan: 46853485.71428572 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 18.50952380952381 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.0000064887152777777775 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005190972222222222 + carbon: 0.0052011003297818366 + sci: 0.0001544881286073813 + - timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 9.166666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0005673927632489277 + - timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 9.166666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0005673927632489277 + - timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 9.166666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0005673927632489277 + - timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 9.166666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0005673927632489277 + - timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 9.166666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0005673927632489277 + - timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 9 + network/energy: 1.e-7 + requests: 5.5 + cpu/thermal-design-power: 60 + grid/carbon-intensity: 480 + device/emissions-embodied: 153.312 + time-reserved: 2160.2 + device/expected-lifespan: 9460800 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.22425 + cpu-wattage: 3.7375 + cpu-wattage-times-duration: 112.125 + cpu-energy-raw: 0.000031145833333333336 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000003893229166666667 + carbon-embodied: 0.000006076864535768645 + carbon-operational: 0.0031145833333333334 + carbon: 0.003120660197869102 + sci: 0.0005673927632489276 + - timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 1 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + aggregated: + carbon: 0.04652112950279046 + child-1: + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-allocated: 1 + vcpus-total: 8 + pipeline: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + requests: 10 + - timestamp: "2023-12-12T00:00:01.000Z" + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 90 + - timestamp: "2023-12-12T00:00:06.000Z" + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 30 + - timestamp: "2023-12-12T00:00:13.000Z" + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + requests: 22 + outputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: 22 + network/energy: 0.0000018 + requests: 82 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 2759.6159999999995 + time-reserved: 2880 + device/expected-lifespan: 170294400 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.4065 + cpu-wattage: 109.19999999999999 + cpu-wattage-times-duration: 246 + cpu-energy-raw: 0.00006833333333333335 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000008541666666666668 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.006833333333333334 + carbon: 0.0068434614408929475 + sci: 0.0000834568468401579 + - timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + cpu/utilization: 13 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 7.714285714285713e-7 + requests: 35.14285714285714 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 1182.6925714285712 + time-reserved: 2880 + device/expected-lifespan: 72983314.28571428 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.30975 + cpu-wattage: 29.907142857142862 + cpu-wattage-times-duration: 192.25 + cpu-energy-raw: 0.00005340277777777778 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006675347222222222 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005340277777777777 + carbon: 0.005350405885337391 + sci: 0.00015224732194049487 + - timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + cpu/utilization: 12 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 4.952380952380952e-7 + requests: 14.323809523809523 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 759.2594285714285 + time-reserved: 2880 + device/expected-lifespan: 46853485.71428572 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 18.50952380952381 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.0000064887152777777775 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005190972222222222 + carbon: 0.0052011003297818366 + sci: 0.000363108733129716 + - timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 3.6666666666666665 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0014184819081223194 + - timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 3.6666666666666665 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0014184819081223194 + - timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 3.6666666666666665 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0014184819081223194 + - timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 3.6666666666666665 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0014184819081223194 + - timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 3.6666666666666665 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0014184819081223194 + - timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 9 + network/energy: 1.e-7 + requests: 2.1999999999999997 + cpu/thermal-design-power: 60 + grid/carbon-intensity: 480 + device/emissions-embodied: 153.312 + time-reserved: 2160.2 + device/expected-lifespan: 9460800 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.22425 + cpu-wattage: 3.7375 + cpu-wattage-times-duration: 112.125 + cpu-energy-raw: 0.000031145833333333336 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000003893229166666667 + carbon-embodied: 0.000006076864535768645 + carbon-operational: 0.0031145833333333334 + carbon: 0.003120660197869102 + sci: 0.0014184819081223194 + - timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 1 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + aggregated: + carbon: 0.04652112950279046 + child-2: + children: + child-2-0: + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-allocated: 1 + vcpus-total: 8 + pipeline: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + requests: 50 + - timestamp: "2023-12-12T00:00:01.000Z" + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 65 + - timestamp: "2023-12-12T00:00:06.000Z" + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 80 + - timestamp: "2023-12-12T00:00:13.000Z" + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + requests: 40 + outputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: 22 + network/energy: 0.0000018 + requests: 102 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 2759.6159999999995 + time-reserved: 2880 + device/expected-lifespan: 170294400 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.4065 + cpu-wattage: 109.19999999999999 + cpu-wattage-times-duration: 246 + cpu-energy-raw: 0.00006833333333333335 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000008541666666666668 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.006833333333333334 + carbon: 0.0068434614408929475 + sci: 0.00006709275922444067 + - timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + cpu/utilization: 13 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 7.714285714285713e-7 + requests: 58.71428571428572 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 1182.6925714285712 + time-reserved: 2880 + device/expected-lifespan: 72983314.28571428 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.30975 + cpu-wattage: 29.907142857142862 + cpu-wattage-times-duration: 192.25 + cpu-energy-raw: 0.00005340277777777778 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006675347222222222 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005340277777777777 + carbon: 0.005350405885337391 + sci: 0.0000911261343001502 + - timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + cpu/utilization: 12 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 4.952380952380952e-7 + requests: 36.952380952380956 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 759.2594285714285 + time-reserved: 2880 + device/expected-lifespan: 46853485.71428572 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 18.50952380952381 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.0000064887152777777775 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005190972222222222 + carbon: 0.0052011003297818366 + sci: 0.00014075142645028166 + - timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 9 + network/energy: 1.e-7 + requests: 4 + cpu/thermal-design-power: 60 + grid/carbon-intensity: 480 + device/emissions-embodied: 153.312 + time-reserved: 2160.2 + device/expected-lifespan: 9460800 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.22425 + cpu-wattage: 3.7375 + cpu-wattage-times-duration: 112.125 + cpu-energy-raw: 0.000031145833333333336 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000003893229166666667 + carbon-embodied: 0.000006076864535768645 + carbon-operational: 0.0031145833333333334 + carbon: 0.003120660197869102 + sci: 0.0007801650494672755 + - timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 1 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + aggregated: + carbon: 0.04652112950279046 + child-2-1: + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-allocated: 1 + vcpus-total: 8 + pipeline: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + requests: 50 + - timestamp: "2023-12-12T00:00:01.000Z" + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 50 + - timestamp: "2023-12-12T00:00:06.000Z" + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 60 + - timestamp: "2023-12-12T00:00:13.000Z" + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + requests: 40 + outputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: 22 + network/energy: 0.0000018 + requests: 90 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 2759.6159999999995 + time-reserved: 2880 + device/expected-lifespan: 170294400 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.4065 + cpu-wattage: 109.19999999999999 + cpu-wattage-times-duration: 246 + cpu-energy-raw: 0.00006833333333333335 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000008541666666666668 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.006833333333333334 + carbon: 0.0068434614408929475 + sci: 0.00007603846045436609 + - timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + cpu/utilization: 13 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 7.714285714285713e-7 + requests: 44.28571428571428 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 1182.6925714285712 + time-reserved: 2880 + device/expected-lifespan: 72983314.28571428 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.30975 + cpu-wattage: 29.907142857142862 + cpu-wattage-times-duration: 192.25 + cpu-energy-raw: 0.00005340277777777778 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006675347222222222 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005340277777777777 + carbon: 0.005350405885337391 + sci: 0.00012081561676568304 + - timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + cpu/utilization: 12 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 4.952380952380952e-7 + requests: 28.38095238095238 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 759.2594285714285 + time-reserved: 2880 + device/expected-lifespan: 46853485.71428572 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 18.50952380952381 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.0000064887152777777775 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005190972222222222 + carbon: 0.0052011003297818366 + sci: 0.0001832602465191587 + - timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 9 + network/energy: 1.e-7 + requests: 4 + cpu/thermal-design-power: 60 + grid/carbon-intensity: 480 + device/emissions-embodied: 153.312 + time-reserved: 2160.2 + device/expected-lifespan: 9460800 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.22425 + cpu-wattage: 3.7375 + cpu-wattage-times-duration: 112.125 + cpu-energy-raw: 0.000031145833333333336 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000003893229166666667 + carbon-embodied: 0.000006076864535768645 + carbon-operational: 0.0031145833333333334 + carbon: 0.003120660197869102 + sci: 0.0007801650494672755 + - timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 1 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + aggregated: + carbon: 0.04652112950279046 + outputs: + - carbon: 0.013686922881785895 + timestamp: "2023-12-12T00:00:00.000Z" + duration: 5 + - carbon: 0.010700811770674782 + timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + - carbon: 0.010402200659563673 + timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + - carbon: 0.010402200659563675 + timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + - carbon: 0.010402200659563675 + timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + - carbon: 0.010402200659563675 + timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + - carbon: 0.010402200659563675 + timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + - carbon: 0.010402200659563675 + timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + - carbon: 0.006241320395738204 + timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + aggregated: + carbon: 0.09304225900558093 + outputs: + - carbon: 0.02737384576357179 + timestamp: "2023-12-12T00:00:00.000Z" + duration: 5 + - carbon: 0.021401623541349564 + timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + - carbon: 0.020804401319127346 + timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + - carbon: 0.02080440131912735 + timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + - carbon: 0.02080440131912735 + timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + - carbon: 0.02080440131912735 + timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + - carbon: 0.02080440131912735 + timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + - carbon: 0.02080440131912735 + timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + - carbon: 0.012482640791476408 + timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + aggregated: + carbon: 0.18608451801116185 diff --git a/manifests/outputs/pipelines/pipeline-teads-sci.yaml b/manifests/outputs/pipelines/pipeline-teads-sci.yaml new file mode 100644 index 000000000..1c2a9d005 --- /dev/null +++ b/manifests/outputs/pipelines/pipeline-teads-sci.yaml @@ -0,0 +1,276 @@ +name: pipeline-teads-sci +description: >- + a full pipeline seeded with some hardcoded input data and yielding an SCI + score +tags: null +initialize: + plugins: + interpolate: + path: builtin + method: Interpolation + global-config: + method: linear + x: + - 0 + - 10 + - 50 + - 100 + 'y': + - 0.12 + - 0.32 + - 0.75 + - 1.02 + input-parameter: cpu/utilization + output-parameter: cpu-factor + cpu-factor-to-wattage: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-factor + - cpu/thermal-design-power + output-parameter: cpu-wattage + wattage-times-duration: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-wattage + - duration + output-parameter: cpu-wattage-times-duration + wattage-to-energy-kwh: + path: builtin + method: Divide + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + calculate-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + correct-cpu-energy-for-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh + sci-embodied: + path: builtin + method: SciEmbodied + operational-carbon: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-energy-kwh + - grid/carbon-intensity + output-parameter: carbon-operational + sci: + path: builtin + method: Sci + global-config: + functional-unit: component + sum-carbon: + path: builtin + method: Sum + global-config: + input-parameters: + - carbon-operational + - carbon-embodied + output-parameter: carbon +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/pipeline-teads-sci.yml -o + manifests/outputs/pipelines/pipeline-teads-sci + environment: + if-version: 0.5.0 + os: macOS + os-version: '14.5' + node-version: 18.14.2 + date-time: 2024-07-19T06:32:50.994Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.10' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.22.4 + status: success +tree: + children: + child-1: + pipeline: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - sci + config: null + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + component: 1 + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + outputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + component: 1 + cpu-factor: 0.75 + cpu-wattage: 75 + cpu-wattage-times-duration: 75 + cpu-energy-raw: 0.000020833333333333333 + vcpu-ratio: 8 + cpu-energy-kwh: 0.0000026041666666666666 + carbon-embodied: 0.0000020256215119228817 + carbon-operational: 0.0020833333333333333 + carbon: 0.002085358954845256 + sci: 0.002085358954845256 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + component: 1 + cpu-factor: 0.4275 + cpu-wattage: 42.75 + cpu-wattage-times-duration: 213.75 + cpu-energy-raw: 0.000059375 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000007421875 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.0059375 + carbon: 0.005947628107559615 + sci: 0.005947628107559615 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + component: 1 + cpu-factor: 0.37375 + cpu-wattage: 37.375 + cpu-wattage-times-duration: 261.625 + cpu-energy-raw: 0.00007267361111111111 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000009084201388888889 + carbon-embodied: 0.00001417935058346017 + carbon-operational: 0.007267361111111111 + carbon: 0.007281540461694571 + sci: 0.007281540461694571 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + component: 1 + cpu-factor: 0.37375 + cpu-wattage: 37.375 + cpu-wattage-times-duration: 1121.25 + cpu-energy-raw: 0.00031145833333333335 + vcpu-ratio: 8 + cpu-energy-kwh: 0.00003893229166666667 + carbon-embodied: 0.00006076864535768645 + carbon-operational: 0.031145833333333334 + carbon: 0.03120660197869102 + sci: 0.03120660197869102 diff --git a/manifests/outputs/pipelines/sci.yaml b/manifests/outputs/pipelines/sci.yaml new file mode 100644 index 000000000..c4fa641c2 --- /dev/null +++ b/manifests/outputs/pipelines/sci.yaml @@ -0,0 +1,297 @@ +name: pipeline-teads-sci +description: >- + a full pipeline seeded with some hardcoded input data and yielding an SCI + score +tags: null +initialize: + plugins: + interpolate: + path: builtin + method: Interpolation + global-config: + method: linear + x: + - 0 + - 10 + - 50 + - 100 + 'y': + - 0.12 + - 0.32 + - 0.75 + - 1.02 + input-parameter: cpu/utilization + output-parameter: cpu-factor + cpu-factor-to-wattage: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-factor + - cpu/thermal-design-power + output-parameter: cpu-wattage + wattage-times-duration: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-wattage + - duration + output-parameter: cpu-wattage-times-duration + wattage-to-energy-kwh: + path: builtin + method: Divide + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + calculate-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + correct-cpu-energy-for-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu/energy + sum-energy-components: + path: builtin + method: Sum + global-config: + input-parameters: + - cpu/energy + - network/energy + output-parameter: energy + embodied-carbon: + path: builtin + method: SciEmbodied + operational-carbon: + path: builtin + method: Multiply + global-config: + input-parameters: + - energy + - grid/carbon-intensity + output-parameter: carbon-operational + sum-carbon: + path: builtin + method: Sum + global-config: + input-parameters: + - carbon-operational + - carbon-embodied + output-parameter: carbon + sci: + path: builtin + method: Sci + global-config: + functional-unit: component +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/sci.yml -o manifests/outputs/pipelines/sci + environment: + if-version: 0.5.0 + os: macOS + os-version: '14.5' + node-version: 18.14.2 + date-time: 2024-07-19T06:34:45.027Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.10' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.22.4 + status: success +tree: + children: + child-1: + pipeline: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sum-energy-components + - embodied-carbon + - operational-carbon + - sum-carbon + - sci + defaults: + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + resources-reserved: vcpus-allocated + resources-total: vcpus-total + component: 1 + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + outputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + resources-reserved: vcpus-allocated + resources-total: vcpus-total + component: 1 + cpu-factor: 0.75 + cpu-wattage: 75 + cpu-wattage-times-duration: 75 + cpu-energy-raw: 0.000020833333333333333 + vcpu-ratio: 4 + cpu/energy: 0.000005208333333333333 + energy: 0.000006208333333333333 + carbon-embodied: 0.000004051243023845763 + carbon-operational: 0.004966666666666666 + carbon: 0.004970717909690512 + sci: 0.004970717909690512 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + resources-reserved: vcpus-allocated + resources-total: vcpus-total + component: 1 + cpu-factor: 0.4275 + cpu-wattage: 42.75 + cpu-wattage-times-duration: 213.75 + cpu-energy-raw: 0.000059375 + vcpu-ratio: 4 + cpu/energy: 0.00001484375 + energy: 0.00001584375 + carbon-embodied: 0.000020256215119228814 + carbon-operational: 0.012674999999999999 + carbon: 0.012695256215119228 + sci: 0.012695256215119228 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + resources-reserved: vcpus-allocated + resources-total: vcpus-total + component: 1 + cpu-factor: 0.37375 + cpu-wattage: 37.375 + cpu-wattage-times-duration: 261.625 + cpu-energy-raw: 0.00007267361111111111 + vcpu-ratio: 4 + cpu/energy: 0.000018168402777777778 + energy: 0.000019168402777777778 + carbon-embodied: 0.00002835870116692034 + carbon-operational: 0.015334722222222222 + carbon: 0.015363080923389142 + sci: 0.015363080923389142 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + resources-reserved: vcpus-allocated + resources-total: vcpus-total + component: 1 + cpu-factor: 0.37375 + cpu-wattage: 37.375 + cpu-wattage-times-duration: 1121.25 + cpu-energy-raw: 0.00031145833333333335 + vcpu-ratio: 4 + cpu/energy: 0.00007786458333333334 + energy: 0.00007886458333333333 + carbon-embodied: 0.0001215372907153729 + carbon-operational: 0.06309166666666667 + carbon: 0.06321320395738204 + sci: 0.06321320395738204 diff --git a/manifests/outputs/pipelines/teads-curve.yaml b/manifests/outputs/pipelines/teads-curve.yaml new file mode 100644 index 000000000..26885e1c7 --- /dev/null +++ b/manifests/outputs/pipelines/teads-curve.yaml @@ -0,0 +1,188 @@ +name: carbon-intensity plugin demo +description: null +tags: null +initialize: + plugins: + interpolate: + path: builtin + method: Interpolation + global-config: + method: linear + x: + - 0 + - 10 + - 50 + - 100 + 'y': + - 0.12 + - 0.32 + - 0.75 + - 1.02 + input-parameter: cpu/utilization + output-parameter: cpu-factor + cpu-factor-to-wattage: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-factor + - thermal-design-power + output-parameter: cpu-wattage + wattage-times-duration: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-wattage + - duration + output-parameter: cpu-wattage-times-duration + wattage-to-energy-kwh: + path: builtin + method: Divide + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + calculate-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + correct-cpu-energy-for-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/teads-curve.yml -o + manifests/outputs/pipelines/teads-curve + environment: + if-version: 0.5.0 + os: macOS + os-version: '14.5' + node-version: 18.14.2 + date-time: 2024-07-19T06:35:33.728Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.10' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.22.4 + status: success +tree: + children: + child: + pipeline: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + defaults: + thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + inputs: + - timestamp: 2023-08-06T00:00 + duration: 360 + cpu/utilization: 1 + carbon: 30 + - timestamp: 2023-09-06T00:00 + duration: 360 + carbon: 30 + cpu/utilization: 10 + - timestamp: 2023-10-06T00:00 + duration: 360 + carbon: 30 + cpu/utilization: 50 + - timestamp: 2023-10-06T00:00 + duration: 360 + carbon: 30 + cpu/utilization: 100 + outputs: + - timestamp: 2023-08-06T00:00 + duration: 360 + cpu/utilization: 1 + carbon: 30 + thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + cpu-factor: 0.13999999999999999 + cpu-wattage: 13.999999999999998 + cpu-wattage-times-duration: 5039.999999999999 + cpu-energy-raw: 0.0013999999999999998 + vcpu-ratio: 4 + cpu-energy-kwh: 0.00034999999999999994 + - timestamp: 2023-09-06T00:00 + duration: 360 + carbon: 30 + cpu/utilization: 10 + thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + cpu-factor: 0.32 + cpu-wattage: 32 + cpu-wattage-times-duration: 11520 + cpu-energy-raw: 0.0032 + vcpu-ratio: 4 + cpu-energy-kwh: 0.0008 + - timestamp: 2023-10-06T00:00 + duration: 360 + carbon: 30 + cpu/utilization: 50 + thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + cpu-factor: 0.75 + cpu-wattage: 75 + cpu-wattage-times-duration: 27000 + cpu-energy-raw: 0.0075 + vcpu-ratio: 4 + cpu-energy-kwh: 0.001875 + - timestamp: 2023-10-06T00:00 + duration: 360 + carbon: 30 + cpu/utilization: 100 + thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + cpu-factor: 1.02 + cpu-wattage: 102 + cpu-wattage-times-duration: 36720 + cpu-energy-raw: 0.0102 + vcpu-ratio: 4 + cpu-energy-kwh: 0.00255 diff --git a/manifests/outputs/pipelines/zeros.yaml b/manifests/outputs/pipelines/zeros.yaml new file mode 100644 index 000000000..3efd3c0fb --- /dev/null +++ b/manifests/outputs/pipelines/zeros.yaml @@ -0,0 +1,198 @@ +name: generics +description: >- + a pipeline that does arbitrary calculations using our generic arithmetic + builtins +tags: null +initialize: + plugins: + sum-zero-and-one: + path: builtin + method: Sum + global-config: + input-parameters: + - some-value + - zero-value + output-parameter: one-plus-zero + sum-zero-and-zero: + path: builtin + method: Sum + global-config: + input-parameters: + - zero-value + - zero-value + output-parameter: zero-plus-zero + subtract-one-and-zero: + path: builtin + method: Subtract + global-config: + input-parameters: + - some-value + - zero-value + output-parameter: one-minus-zero + subtract-zero-and-zero: + path: builtin + method: Sum + global-config: + input-parameters: + - zero-value + - zero-value + output-parameter: zero-minus-zero + subtract-zero-and-one: + path: builtin + method: Subtract + global-config: + input-parameters: + - zero-value + - some-value + output-parameter: zero-minus-one + coefficient-one-times-zero: + path: builtin + method: Coefficient + global-config: + input-parameter: zero-value + coefficient: 1 + output-parameter: zero-times-one-coefficient + coefficient-zero-times-one: + path: builtin + method: Coefficient + global-config: + input-parameter: some-value + coefficient: 0 + output-parameter: one-times-zero-coefficient + coefficient-zero-times-zero: + path: builtin + method: Coefficient + global-config: + input-parameter: zero-value + coefficient: 0 + output-parameter: zero-times-zero-coefficient + multiply-one-times-zero: + path: builtin + method: Multiply + global-config: + input-parameters: + - some-value + - zero-value + output-parameter: one-times-zero + multiply-zero-times-one: + path: builtin + method: Multiply + global-config: + input-parameters: + - zero-value + - zero-value + output-parameter: zero-times-one + exponent-one-to-zero: + path: builtin + method: Exponent + global-config: + input-parameter: some-value + exponent: 0 + output-parameter: one-raised-to-zero-power + exponent-zero-to-zero: + path: builtin + method: Exponent + global-config: + input-parameter: zero-value + exponent: 0 + output-parameter: zero-raised-to-zero-power + exponent-zero-to-one: + path: builtin + method: Exponent + global-config: + input-parameter: zero-value + exponent: 1 + output-parameter: zero-raised-to-first-power + sci: + path: builtin + method: Sci + global-config: + functional-unit: zero-value +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/zeros.yml -o manifests/outputs/pipelines/zeros + environment: + if-version: 0.5.0 + os: macOS + os-version: '14.5' + node-version: 18.14.2 + date-time: 2024-07-19T06:36:00.790Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.10' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.22.4 + status: success +tree: + children: + child-1: + pipeline: + compute: + - sum-zero-and-one + - sum-zero-and-zero + - subtract-one-and-zero + - subtract-zero-and-zero + - subtract-zero-and-one + - coefficient-one-times-zero + - coefficient-zero-times-one + - coefficient-zero-times-zero + - multiply-one-times-zero + - multiply-zero-times-one + - exponent-one-to-zero + - exponent-zero-to-one + - exponent-zero-to-zero + - sci + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + duration: 1 + some-value: 1 + zero-value: 0 + carbon: 10 + outputs: + - timestamp: '2023-12-12T00:00:00.000Z' + duration: 1 + some-value: 1 + zero-value: 0 + carbon: 10 + one-plus-zero: 1 + zero-plus-zero: 0 + one-minus-zero: 1 + zero-minus-zero: 0 + zero-minus-one: -1 + zero-times-one-coefficient: 0 + one-times-zero-coefficient: 0 + zero-times-zero-coefficient: 0 + one-times-zero: 0 + zero-times-one: 0 + one-raised-to-zero-power: 1 + zero-raised-to-first-power: 0 + zero-raised-to-zero-power: 1 + sci: 10 diff --git a/manifests/outputs/plugins/mock-observations/success.yaml b/manifests/outputs/plugins/mock-observations/success.yaml deleted file mode 100644 index e92b933bf..000000000 --- a/manifests/outputs/plugins/mock-observations/success.yaml +++ /dev/null @@ -1,286 +0,0 @@ -name: mock-observation-demo -description: a manifest demonstrating how to use the mock observations feature -tags: null -initialize: - plugins: - mock-observations: - path: builtin - method: MockObservations - global-config: - timestamp-from: 2023-07-06T00:00 - timestamp-to: 2023-07-06T00:10 - duration: 60 - components: - - cloud/instance-type: A1 - - cloud/instance-type: B1 - generators: - common: - region: uk-west - common-key: common-val - randint: - cpu/utilization: - min: 1 - max: 99 - memory/utilization: - min: 1 - max: 99 -execution: - command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/mock-observations/success.yml -o - manifests/outputs/plugins/mock-observations/success - environment: - if-version: 0.4.0 - os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-02T19:15:43.108Z (UTC) - dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - status: success -tree: - children: - child: - pipeline: - - mock-observations - inputs: null - outputs: - - timestamp: "2023-07-06T00:00:00.000Z" - - timestamp: "2023-07-06T00:00:00.000Z" - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: 81 - memory/utilization: 63 - - timestamp: "2023-07-06T00:01:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:01:00.000Z" - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: 92 - memory/utilization: 63 - - timestamp: "2023-07-06T00:02:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:02:00.000Z" - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: 63 - memory/utilization: 95 - - timestamp: "2023-07-06T00:03:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:03:00.000Z" - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: 37 - memory/utilization: 13 - - timestamp: "2023-07-06T00:04:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:04:00.000Z" - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: 28 - memory/utilization: 50 - - timestamp: "2023-07-06T00:05:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:05:00.000Z" - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: 98 - memory/utilization: 29 - - timestamp: "2023-07-06T00:06:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:06:00.000Z" - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: 10 - memory/utilization: 93 - - timestamp: "2023-07-06T00:07:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:07:00.000Z" - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: 84 - memory/utilization: 37 - - timestamp: "2023-07-06T00:08:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:08:00.000Z" - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: 10 - memory/utilization: 30 - - timestamp: "2023-07-06T00:09:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:09:00.000Z" - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: 50 - memory/utilization: 10 - - timestamp: "2023-07-06T00:00:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:00:00.000Z" - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: 97 - memory/utilization: 89 - - timestamp: "2023-07-06T00:01:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:01:00.000Z" - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: 61 - memory/utilization: 83 - - timestamp: "2023-07-06T00:02:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:02:00.000Z" - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: 86 - memory/utilization: 52 - - timestamp: "2023-07-06T00:03:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:03:00.000Z" - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: 17 - memory/utilization: 60 - - timestamp: "2023-07-06T00:04:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:04:00.000Z" - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: 53 - memory/utilization: 84 - - timestamp: "2023-07-06T00:05:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:05:00.000Z" - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: 38 - memory/utilization: 58 - - timestamp: "2023-07-06T00:06:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:06:00.000Z" - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: 4 - memory/utilization: 39 - - timestamp: "2023-07-06T00:07:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:07:00.000Z" - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: 89 - memory/utilization: 52 - - timestamp: "2023-07-06T00:08:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:08:00.000Z" - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: 38 - memory/utilization: 16 - - timestamp: "2023-07-06T00:09:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:09:00.000Z" - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: "*" - memory/utilization: "*" diff --git a/manifests/outputs/plugins/regex/failure-not-matching-with-regex.yaml b/manifests/outputs/plugins/regex/failure-not-matching-with-regex.yaml deleted file mode 100644 index d07bc0484..000000000 --- a/manifests/outputs/plugins/regex/failure-not-matching-with-regex.yaml +++ /dev/null @@ -1,74 +0,0 @@ -name: regex -description: physical processor doesn't match the regex expression -tags: null -initialize: - plugins: - regex: - method: Regex - path: builtin - global-config: - parameter: physical-processor - match: ^ - output: cpu/name -execution: - status: fail - command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/regex/failure-not-matching-with-regex.yml -o - manifests/outputs/plugins/regex/failure-not-matching-with-regex - environment: - if-version: 0.4.0 - os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-02T20:24:39.241Z (UTC) - dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - error: >- - RegexMismatchError: `Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 - GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz` does - not match the /^/ regex expression -tree: - children: - child: - pipeline: - - regex - config: - regex: null - inputs: - - timestamp: 2023-08-06T00:00 - duration: 3600 - physical-processor: >- - Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® - E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz diff --git a/src/__mocks__/fs/index.ts b/src/__mocks__/fs/index.ts index 927509cba..a0520879c 100644 --- a/src/__mocks__/fs/index.ts +++ b/src/__mocks__/fs/index.ts @@ -52,8 +52,9 @@ cpu-cores-available,cpu-cores-utilized,cpu-manufacturer,cpu-model-name,cpu-tdp,g tree: children: front-end: - pipeline: - - boavizta-cpu + pipeline: + compute: + - boavizta-cpu config: boavizta-cpu: core-units: 24 @@ -108,6 +109,9 @@ export const writeFile = async (pathToFile: string, content: string) => { } }; +export const appendFile = (file: string, appendContent: string) => + `${file}${appendContent}`; + export const stat = async (filePath: string) => { if (filePath === 'true') { return true; diff --git a/src/__mocks__/mock-manifest.yaml b/src/__mocks__/mock-manifest.yaml index 64e00e301..a69009d1e 100644 --- a/src/__mocks__/mock-manifest.yaml +++ b/src/__mocks__/mock-manifest.yaml @@ -57,7 +57,8 @@ tree: children: child: pipeline: - - memory-energy-from-memory-util + compute: + - memory-energy-from-memory-util config: null inputs: - timestamp: 2023-12-12T00:00:00.000Z diff --git a/src/__tests__/if-run/builtins/group-by.test.ts b/src/__tests__/if-run/builtins/group-by.test.ts deleted file mode 100644 index b59da6836..000000000 --- a/src/__tests__/if-run/builtins/group-by.test.ts +++ /dev/null @@ -1,177 +0,0 @@ -import {ERRORS} from '@grnsft/if-core/utils'; - -import {GroupBy} from '../../../if-run/builtins/group-by'; - -import {STRINGS} from '../../../if-run/config'; - -const {InvalidGroupingError, InputValidationError, GlobalConfigError} = ERRORS; -const {MISSING_GLOBAL_CONFIG, INVALID_GROUP_BY} = STRINGS; - -describe('builtins/group-by: ', () => { - describe('GroupBy: ', () => { - const plugin = GroupBy(); - - describe('init GroupBy: ', () => { - it('initalizes object with properties.', async () => { - expect(plugin).toHaveProperty('metadata'); - expect(plugin).toHaveProperty('execute'); - }); - }); - - describe('execute(): ', () => { - it('groups inputs correctly.', () => { - const inputs = [ - { - timestamp: '2023-07-06T00:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T05:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T10:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - ]; - const config = { - group: ['region', 'cloud/instance-type'], - }; - - const expectedOutput = { - 'uk-west': { - children: { - A1: { - inputs: [ - { - 'cloud/instance-type': 'A1', - region: 'uk-west', - timestamp: '2023-07-06T00:00', - }, - { - 'cloud/instance-type': 'A1', - region: 'uk-west', - timestamp: '2023-07-06T05:00', - }, - { - 'cloud/instance-type': 'A1', - region: 'uk-west', - timestamp: '2023-07-06T10:00', - }, - ], - }, - }, - }, - }; - - const result = plugin.execute(inputs, config); - expect(result).toEqual(expectedOutput); - }); - - it('throws an error when config is not provided.', () => { - const inputs = [ - { - timestamp: '2023-07-06T00:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T05:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T10:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - ]; - - const config = undefined; - - expect.assertions(2); - try { - plugin.execute(inputs, config!); - } catch (error) { - expect(error).toBeInstanceOf(GlobalConfigError); - expect(error).toEqual(new GlobalConfigError(MISSING_GLOBAL_CONFIG)); - } - }); - - it('throws an error if `group` is an empty array.', () => { - const inputs = [ - { - timestamp: '2023-07-06T00:00', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T05:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T10:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - ]; - const config = { - group: ['region', 'cloud/instance-type'], - }; - - expect.assertions(2); - try { - plugin.execute(inputs, config); - } catch (error) { - expect(error).toBeInstanceOf(InvalidGroupingError); - expect(error).toEqual( - new InvalidGroupingError('Invalid group region.') - ); - } - }); - - it('throws an error if group type is missing from the input.', () => { - const inputs = [ - {timestamp: 1, region: 'uk-west', 'cloud/instance-type': 'A1'}, - ]; - const config = { - group: [], - }; - - expect.assertions(2); - try { - plugin.execute(inputs, config); - } catch (error) { - expect(error).toBeInstanceOf(InputValidationError); - expect(error).toEqual( - new InputValidationError( - '"group" parameter is array must contain at least 1 element(s). Error code: too_small.' - ) - ); - } - }); - - it('throws an error if input does not have required group type.', () => { - const inputs = [ - {timestamp: 1, region: 'uk-west', 'cloud/instance-type': 'A1'}, - ]; - const config = { - group: ['region', 'cloud/instance-type', 'unknown'], - }; - - expect.assertions(2); - try { - plugin.execute(inputs, config); - } catch (error) { - expect(error).toBeInstanceOf(InvalidGroupingError); - expect(error).toEqual( - new InvalidGroupingError(INVALID_GROUP_BY(config.group[2])) - ); - } - }); - }); - }); -}); diff --git a/src/__tests__/if-run/builtins/time-sync.test.ts b/src/__tests__/if-run/builtins/time-sync.test.ts index 314ed06d4..38621e12f 100644 --- a/src/__tests__/if-run/builtins/time-sync.test.ts +++ b/src/__tests__/if-run/builtins/time-sync.test.ts @@ -7,6 +7,7 @@ import {storeAggregationMetrics} from '../../../if-run/lib/aggregate'; import {TimeSync} from '../../../if-run/builtins/time-sync'; import {STRINGS} from '../../../if-run/config'; +import {AGGREGATION_METHODS} from '../../../if-run/types/aggregation'; Settings.defaultZone = 'utc'; const { @@ -56,16 +57,18 @@ jest.mock('luxon', () => { describe('builtins/time-sync:', () => { beforeAll(() => { const metricStorage: AggregationParams = { - metrics: { - carbon: {method: 'sum'}, - 'cpu/utilization': {method: 'sum'}, - 'time-reserved': {method: 'avg'}, - 'resources-total': {method: 'none'}, - }, + metrics: [ + 'carbon', + 'cpu/utilization', + 'time-reserved', + 'resources-total', + ], type: 'horizontal', }; - - storeAggregationMetrics(metricStorage); + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); + storeAggregationMetrics(...convertedMetrics); }); describe('time-sync: ', () => { @@ -472,6 +475,7 @@ describe('execute(): ', () => { interval: 1, 'allow-padding': true, }; + storeAggregationMetrics({carbon: 'sum'}); const timeModel = TimeSync(basicConfig, parametersMetadata); @@ -586,6 +590,8 @@ describe('execute(): ', () => { interval: 5, 'allow-padding': true, }; + storeAggregationMetrics({'time-reserved': 'avg'}); + storeAggregationMetrics({'resources-total': 'sum'}); const timeModel = TimeSync(basicConfig, parametersMetadata); @@ -659,6 +665,7 @@ describe('execute(): ', () => { interval: 5, 'allow-padding': true, }; + storeAggregationMetrics({'resources-total': 'none'}); const timeModel = TimeSync(basicConfig, parametersMetadata); diff --git a/src/__tests__/if-run/lib/aggregate.test.ts b/src/__tests__/if-run/lib/aggregate.test.ts index 36bbc0d1f..00d9c6d0e 100644 --- a/src/__tests__/if-run/lib/aggregate.test.ts +++ b/src/__tests__/if-run/lib/aggregate.test.ts @@ -6,17 +6,19 @@ import { aggregate, storeAggregationMetrics, } from '../../../if-run/lib/aggregate'; +import {AGGREGATION_METHODS} from '../../../if-run/types/aggregation'; describe('lib/aggregate: ', () => { beforeAll(() => { const metricStorage: AggregationParams = { - metrics: { - carbon: {method: 'sum'}, - }, + metrics: ['carbon'], type: 'horizontal', }; + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); - storeAggregationMetrics(metricStorage); + storeAggregationMetrics(...convertedMetrics); }); describe('aggregate(): ', () => { @@ -60,7 +62,7 @@ describe('lib/aggregate: ', () => { }; const aggregatedTree = aggregate(tree, { - metrics: {carbon: {method: 'sum'}}, + metrics: ['carbon'], type: 'horizontal', }); const expectedAggregated = { @@ -108,7 +110,7 @@ describe('lib/aggregate: ', () => { }; const aggregatedTree = aggregate(tree, { - metrics: {carbon: {method: 'sum'}}, + metrics: ['carbon'], type: 'vertical', }); const expectedOutputs = [ @@ -169,7 +171,7 @@ describe('lib/aggregate: ', () => { }; const aggregatedTree = aggregate(tree, { - metrics: {carbon: {method: 'sum'}}, + metrics: ['carbon'], type: 'both', }); diff --git a/src/__tests__/if-run/lib/compute.test.ts b/src/__tests__/if-run/lib/compute.test.ts index 4ed48b5a5..912793381 100644 --- a/src/__tests__/if-run/lib/compute.test.ts +++ b/src/__tests__/if-run/lib/compute.test.ts @@ -19,10 +19,57 @@ describe('lib/compute: ', () => { kind: 'execute', }, }); - const mockGroupByPlugin = () => ({ - execute: (inputs: any) => ({children: inputs}), + const mockObservePlugin = () => ({ + execute: () => [ + {timestamp: '2024-09-02', duration: 40, 'cpu/utilization': 30}, + {timestamp: '2024-09-03', duration: 60, 'cpu/utilization': 40}, + ], metadata: { - kind: 'groupby', + kind: 'execute', + }, + }); + const mockObservePluginTimeSync = () => ({ + execute: () => [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 60, + 'cpu/utilization': 30, + }, + { + timestamp: '2023-12-12T00:01:00.000Z', + duration: 60, + 'cpu/utilization': 40, + }, + ], + metadata: { + kind: 'execute', + }, + }); + const mockTimeSync = () => ({ + execute: () => [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 30, + 'cpu/utilization': 30, + }, + { + timestamp: '2023-12-12T00:00:30.000Z', + duration: 30, + 'cpu/utilization': 30, + }, + { + timestamp: '2023-12-12T00:01:00.000Z', + duration: 30, + 'cpu/utilization': 40, + }, + { + timestamp: '2023-12-12T00:01:30.000Z', + duration: 30, + 'cpu/utilization': 40, + }, + ], + metadata: { + kind: 'execute', }, }); /** @@ -41,22 +88,11 @@ describe('lib/compute: ', () => { }, }, }, - pluginStorage: pluginStorage().set('mock', mockExecutePlugin()), - }; - const params: ComputeParams = { - // @ts-ignore - context: { - name: 'mock-name', - initialize: { - plugins: { - mock: { - path: 'mockavizta', - method: 'Mockavizta', - }, - }, - }, - }, - pluginStorage: pluginStorage().set('mock', mockGroupByPlugin()), + pluginStorage: pluginStorage() + .set('mock', mockExecutePlugin()) + .set('mock-observe', mockObservePlugin()) + .set('mock-observe-time-sync', mockObservePluginTimeSync()) + .set('time-sync', mockTimeSync()), }; describe('compute(): ', () => { @@ -64,7 +100,7 @@ describe('lib/compute: ', () => { const tree = { children: { mockChild: { - pipeline: ['mock'], + pipeline: {compute: ['mock']}, inputs: [ {timestamp: 'mock-timestamp-1', duration: 10}, {timestamp: 'mock-timestamp-2', duration: 10}, @@ -85,7 +121,7 @@ describe('lib/compute: ', () => { const tree = { children: { mockChild: { - pipeline: ['mock'], + pipeline: {regroup: ['duration']}, inputs: [ {timestamp: 'mock-timestamp-1', duration: 10}, {timestamp: 'mock-timestamp-2', duration: 10}, @@ -93,19 +129,24 @@ describe('lib/compute: ', () => { }, }, }; - const response = await compute(tree, params); - const expectedResult = mockGroupByPlugin().execute( - tree.children.mockChild.inputs - ); + const response = await compute(tree, paramsExecute); + const expectedResponse = { + '10': { + inputs: [ + {duration: 10, timestamp: 'mock-timestamp-1'}, + {duration: 10, timestamp: 'mock-timestamp-2'}, + ], + }, + }; - expect(response.children.mockChild.children).toEqual(expectedResult); + expect(response.children.mockChild.children).toEqual(expectedResponse); }); it('computes simple tree with defaults and execute plugin.', async () => { const tree = { children: { mockChild: { - pipeline: ['mock'], + pipeline: {compute: ['mock']}, defaults: { 'cpu/name': 'Intel CPU', }, @@ -132,7 +173,7 @@ describe('lib/compute: ', () => { const tree = { children: { mockChild1: { - pipeline: ['mock'], + pipeline: {compute: ['mock']}, defaults: { 'cpu/name': 'Intel CPU', }, @@ -144,7 +185,7 @@ describe('lib/compute: ', () => { mockChild2: { children: { mockChild21: { - pipeline: ['mock'], + pipeline: {compute: ['mock']}, defaults: { 'cpu/name': 'Intel CPU', }, @@ -188,16 +229,17 @@ describe('lib/compute: ', () => { }, }; const response = await compute(tree, paramsExecute); - const expectedResult: any[] = []; - expect(response.children.mockChild.outputs).toEqual(expectedResult); + expect(response.children.mockChild.outputs).toBeUndefined(); }); it('computes simple tree with defaults and no inputs with execue plugin.', async () => { const tree = { children: { mockChild: { - pipeline: ['mock'], + pipeline: { + compute: ['mock'], + }, defaults: { carbon: 10, }, @@ -215,7 +257,9 @@ describe('lib/compute: ', () => { const tree = { children: { mockChild: { - pipeline: ['mock'], + pipeline: { + compute: ['mock'], + }, config: { 'cpu/name': 'Intel CPU', }, @@ -234,4 +278,22 @@ describe('lib/compute: ', () => { expect(response.children.mockChild.outputs).toEqual(expectedResult); }); }); + + it('computes simple tree with observe plugin.', async () => { + const tree = { + children: { + mockChild: { + pipeline: {observe: ['mock-observe']}, + }, + }, + }; + + const response = await compute(tree, paramsExecute); + const expectedResult = [ + {timestamp: '2024-09-02', duration: 40, 'cpu/utilization': 30}, + {timestamp: '2024-09-03', duration: 60, 'cpu/utilization': 40}, + ]; + + expect(response.children.mockChild.inputs).toEqual(expectedResult); + }); }); diff --git a/src/__tests__/if-run/lib/initialize.test.ts b/src/__tests__/if-run/lib/initialize.test.ts index f00142260..a1eda8924 100644 --- a/src/__tests__/if-run/lib/initialize.test.ts +++ b/src/__tests__/if-run/lib/initialize.test.ts @@ -18,7 +18,6 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {initialize} from '../../../if-run/lib/initialize'; import {STRINGS} from '../../../if-run/config'; -import {GlobalPlugins} from '../../../common/types/manifest'; const { MissingPluginPathError, @@ -30,8 +29,9 @@ const {MISSING_METHOD, MISSING_PATH, INVALID_MODULE_PATH} = STRINGS; describe('lib/initalize: ', () => { describe('initalize(): ', () => { it('creates instance with get and set methods.', async () => { - const plugins = {}; - const response = await initialize(plugins); + const context = {initialize: {plugins: {}}}; + // @ts-ignore + const response = await initialize(context); expect(response).toHaveProperty('get'); expect(response).toHaveProperty('set'); @@ -40,15 +40,20 @@ describe('lib/initalize: ', () => { }); it('checks if plugin is initalized, warning is logged and plugin has execute and metadata props.', async () => { - const plugins: GlobalPlugins = { - mockavizta: { - path: 'mockavizta', - method: 'Mockavizta', + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'mockavizta', + method: 'Mockavizta', + }, + }, }, }; - const storage = await initialize(plugins); + // @ts-ignore + const storage = await initialize(context); - const pluginName = Object.keys(plugins)[0]; + const pluginName = Object.keys(context.initialize.plugins)[0]; const module = storage.get(pluginName); expect(module).toHaveProperty('execute'); expect(module).toHaveProperty('metadata'); @@ -56,36 +61,45 @@ describe('lib/initalize: ', () => { }); it('checks if plugin is initalized with global config and has execute and metadata.', async () => { - const plugins: GlobalPlugins = { - mockavizta: { - path: 'mockavizta', - method: 'Mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'mockavizta', + method: 'Mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; - const storage = await initialize(plugins); + // @ts-ignore + const storage = await initialize(context); - const pluginName = Object.keys(plugins)[0]; + const pluginName = Object.keys(context.initialize.plugins)[0]; const module = storage.get(pluginName); expect(module).toHaveProperty('execute'); expect(module).toHaveProperty('metadata'); }); it('throws error if plugin does not have path property.', async () => { - const plugins: GlobalPlugins = { - // @ts-ignore - mockavizta: { - method: 'Mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + method: 'Mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; try { - await initialize(plugins); + // @ts-ignore + await initialize(context); } catch (error) { expect(error).toBeInstanceOf(MissingPluginPathError); @@ -96,18 +110,22 @@ describe('lib/initalize: ', () => { }); it('throws error if plugin does not have path property.', async () => { - const plugins: GlobalPlugins = { - // @ts-ignore - mockavizta: { - path: 'mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; try { - await initialize(plugins); + // @ts-ignore + await initialize(context); } catch (error) { expect(error).toBeInstanceOf(MissingPluginMethodError); @@ -118,59 +136,74 @@ describe('lib/initalize: ', () => { }); it('checks if builtin plugin is initalized.', async () => { - const plugins: GlobalPlugins = { - mockavizta: { - path: 'builtin', - method: 'Mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'builtin', + method: 'Mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; - const storage = await initialize(plugins); + // @ts-ignore + const storage = await initialize(context); - const pluginName = Object.keys(plugins)[0]; + const pluginName = Object.keys(context.initialize.plugins)[0]; const module = storage.get(pluginName); expect(module).toHaveProperty('execute'); expect(module).toHaveProperty('metadata'); }); it('checks if github plugin is initalized.', async () => { - const plugins: GlobalPlugins = { - mockavizta: { - path: 'https://github.com/mockavizta', - method: 'Mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'https://github.com/mockavizta', + method: 'Mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; - const storage = await initialize(plugins); + // @ts-ignore + const storage = await initialize(context); - const pluginName = Object.keys(plugins)[0]; + const pluginName = Object.keys(context.initialize.plugins)[0]; const module = storage.get(pluginName); expect(module).toHaveProperty('execute'); expect(module).toHaveProperty('metadata'); }); it('throws error if plugin path is invalid.', async () => { - const plugins: GlobalPlugins = { - mockavizta: { - path: 'failing-mock', - method: 'Mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'failing-mock', + method: 'Mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; try { - await initialize(plugins); + // @ts-ignore + await initialize(context); } catch (error: any) { expect(error).toBeInstanceOf(PluginInitializationError); expect(error.message).toEqual( INVALID_MODULE_PATH( - plugins.mockavizta.path, + context.initialize.plugins.mockavizta.path, new Error( "Cannot find module 'failing-mock' from 'src/if-run/lib/initialize.ts'" ) diff --git a/src/__tests__/if-run/lib/regroup.test.ts b/src/__tests__/if-run/lib/regroup.test.ts new file mode 100644 index 000000000..67ff72e53 --- /dev/null +++ b/src/__tests__/if-run/lib/regroup.test.ts @@ -0,0 +1,161 @@ +import {ERRORS} from '@grnsft/if-core/utils'; + +import {STRINGS} from '../../../if-run/config'; +import {Regroup} from '../../../if-run/lib/regroup'; + +const {InvalidGroupingError, InputValidationError} = ERRORS; +const {INVALID_GROUP_KEY} = STRINGS; + +describe('lib/regroup: ', () => { + describe('Regroup: ', () => { + it('groups inputs correctly.', () => { + const inputs = [ + { + timestamp: '2023-07-06T00:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T05:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T10:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + ]; + const groups = ['region', 'cloud/instance-type']; + + const expectedOutput = { + 'uk-west': { + children: { + A1: { + inputs: [ + { + 'cloud/instance-type': 'A1', + region: 'uk-west', + timestamp: '2023-07-06T00:00', + }, + { + 'cloud/instance-type': 'A1', + region: 'uk-west', + timestamp: '2023-07-06T05:00', + }, + { + 'cloud/instance-type': 'A1', + region: 'uk-west', + timestamp: '2023-07-06T10:00', + }, + ], + }, + }, + }, + }; + + const result = Regroup(inputs, groups); + expect(result).toEqual(expectedOutput); + }); + + it('throws an error when groups is not provided.', () => { + const inputs = [ + { + timestamp: '2023-07-06T00:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T05:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T10:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + ]; + + const groups = undefined; + + expect.assertions(2); + try { + Regroup(inputs, groups!); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toEqual( + new InputValidationError( + '"regroup" parameter is not an array or should contain at least one key. Error code: invalid_type.' + ) + ); + } + }); + + it('throws an error if `group` is an empty array.', () => { + const inputs = [ + { + timestamp: '2023-07-06T00:00', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T05:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T10:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + ]; + const groups = ['region', 'cloud/instance-type']; + + expect.assertions(2); + try { + Regroup(inputs, groups); + } catch (error) { + expect(error).toBeInstanceOf(InvalidGroupingError); + expect(error).toEqual( + new InvalidGroupingError('Invalid group region.') + ); + } + }); + + it('throws an error if group type is missing from the input.', () => { + const inputs = [ + {timestamp: 1, region: 'uk-west', 'cloud/instance-type': 'A1'}, + ]; + const groups: any[] = []; + + expect.assertions(2); + try { + Regroup(inputs, groups); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toEqual( + new InputValidationError( + '"regroup" parameter is array must contain at least 1 element(s). Error code: too_small.' + ) + ); + } + }); + + it('throws an error if input does not have required group type.', () => { + const inputs = [ + {timestamp: 1, region: 'uk-west', 'cloud/instance-type': 'A1'}, + ]; + const groups = ['region', 'cloud/instance-type', 'unknown']; + + expect.assertions(2); + try { + Regroup(inputs, groups); + } catch (error) { + expect(error).toBeInstanceOf(InvalidGroupingError); + expect(error).toEqual( + new InvalidGroupingError(INVALID_GROUP_KEY(groups[2])) + ); + } + }); + }); +}); diff --git a/src/__tests__/if-run/util/aggregation-helper.test.ts b/src/__tests__/if-run/util/aggregation-helper.test.ts index df8a74d9b..f83536f39 100644 --- a/src/__tests__/if-run/util/aggregation-helper.test.ts +++ b/src/__tests__/if-run/util/aggregation-helper.test.ts @@ -4,52 +4,33 @@ import {PluginParams} from '@grnsft/if-core/types'; import {AggregationParams} from '../../../common/types/manifest'; import {aggregateInputsIntoOne} from '../../../if-run/util/aggregation-helper'; -import {AggregationMetric} from '../../../if-run/types/aggregation'; +import { + AGGREGATION_METHODS, + AggregationMetric, +} from '../../../if-run/types/aggregation'; import {storeAggregationMetrics} from '../../../if-run/lib/aggregate'; import {STRINGS} from '../../../if-run/config'; -const {InvalidAggregationMethodError, MissingAggregationParamError} = ERRORS; -const {INVALID_AGGREGATION_METHOD, METRIC_MISSING} = STRINGS; +const {MissingAggregationParamError} = ERRORS; +const {METRIC_MISSING} = STRINGS; describe('util/aggregation-helper: ', () => { beforeAll(() => { const metricStorage: AggregationParams = { - metrics: { - carbon: {method: 'sum'}, - 'cpu/number-cores': {method: 'none'}, - 'cpu/utilization': {method: 'sum'}, - }, + metrics: ['carbon', 'cpu/number-cores', 'cpu/utilization'], type: 'horizontal', }; - - storeAggregationMetrics(metricStorage); + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); + storeAggregationMetrics(...convertedMetrics); }); describe('aggregateInputsIntoOne(): ', () => { - it('throws error if aggregation method is none.', () => { - const inputs: PluginParams[] = []; - const metrics: AggregationMetric = {'cpu/number-cores': {method: 'none'}}; - const isTemporal = false; - - expect.assertions(2); - - try { - aggregateInputsIntoOne(inputs, metrics, isTemporal); - } catch (error) { - expect(error).toBeInstanceOf(InvalidAggregationMethodError); - - if (error instanceof InvalidAggregationMethodError) { - expect(error.message).toEqual( - INVALID_AGGREGATION_METHOD('cpu/number-cores') - ); - } - } - }); - it('throws error if aggregation criteria is not found in input.', () => { const inputs: PluginParams[] = [{timestamp: '', duration: 10}]; - const metrics: AggregationMetric = {'cpu/utilization': {method: 'sum'}}; + const metrics: AggregationMetric[] = [{'cpu/utilization': 'sum'}]; const isTemporal = false; expect.assertions(2); @@ -70,7 +51,7 @@ describe('util/aggregation-helper: ', () => { {timestamp: '', duration: 10, carbon: 10}, {timestamp: '', duration: 10, carbon: 20}, ]; - const metrics: AggregationMetric = {carbon: {method: 'sum'}}; + const metrics: AggregationMetric[] = [{carbon: 'sum'}]; const isTemporal = true; const expectedValue = { @@ -87,7 +68,7 @@ describe('util/aggregation-helper: ', () => { {timestamp: '', duration: 10, carbon: 10}, {timestamp: '', duration: 10, carbon: 20}, ]; - const metrics: AggregationMetric = {carbon: {method: 'sum'}}; + const metrics: AggregationMetric[] = [{carbon: 'sum'}]; const isTemporal = false; const expectedValue = { @@ -99,18 +80,20 @@ describe('util/aggregation-helper: ', () => { it('calculates average of metrics.', () => { const metricStorage: AggregationParams = { - metrics: { - 'cpu/utilization': {method: 'avg'}, - }, + metrics: ['cpu/utilization'], type: 'horizontal', }; + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); + storeAggregationMetrics(...convertedMetrics); + storeAggregationMetrics({'cpu/utilization': 'avg'}); - storeAggregationMetrics(metricStorage); const inputs: PluginParams[] = [ {timestamp: '', duration: 10, 'cpu/utilization': 10}, {timestamp: '', duration: 10, 'cpu/utilization': 90}, ]; - const metrics: AggregationMetric = {'cpu/utilization': {method: 'avg'}}; + const metrics: AggregationMetric[] = [{'cpu/utilization': 'avg'}]; const isTemporal = false; const expectedValue = { diff --git a/src/__tests__/if-run/util/helpers.test.ts b/src/__tests__/if-run/util/helpers.test.ts index c694c0a9a..796f6b201 100644 --- a/src/__tests__/if-run/util/helpers.test.ts +++ b/src/__tests__/if-run/util/helpers.test.ts @@ -242,19 +242,13 @@ describe('if-run/util/helpers: ', () => { expect(storeAggregationMetrics).toHaveBeenCalledTimes(3); expect(storeAggregationMetrics).toHaveBeenNthCalledWith(1, { - metrics: { - carbon: {method: 'sum'}, - }, + carbon: 'sum', }); expect(storeAggregationMetrics).toHaveBeenNthCalledWith(2, { - metrics: { - cpu: {method: 'avg'}, - }, + cpu: 'avg', }); expect(storeAggregationMetrics).toHaveBeenNthCalledWith(3, { - metrics: { - carbon: {method: 'none'}, - }, + carbon: 'none', }); }); diff --git a/src/common/types/manifest.ts b/src/common/types/manifest.ts index c28fe7194..1efab523d 100644 --- a/src/common/types/manifest.ts +++ b/src/common/types/manifest.ts @@ -1,5 +1,7 @@ import {z} from 'zod'; +import {AggregationMethodTypes} from '../../if-run/types/aggregation'; + import {manifestSchema} from '../util/validations'; export type Manifest = z.infer; @@ -9,10 +11,9 @@ export type GlobalPlugins = Manifest['initialize']['plugins']; export type PluginOptions = GlobalPlugins[string]; export type AggregationParams = Manifest['aggregation']; -export type AggregationParamsWithoutType = Omit< - Exclude, - 'type' ->; +export type AggregationMetricsWithMethod = { + [key: string]: AggregationMethodTypes; +}; export type AggregationParamsSure = Extract; diff --git a/src/common/util/validations.ts b/src/common/util/validations.ts index 9f2d4c495..5cdbec5db 100644 --- a/src/common/util/validations.ts +++ b/src/common/util/validations.ts @@ -3,10 +3,7 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {STRINGS} from '../../if-run/config'; -import { - AGGREGATION_METHODS, - AGGREGATION_TYPES, -} from '../../if-run/types/aggregation'; +import {AGGREGATION_TYPES} from '../../if-run/types/aggregation'; const {ManifestValidationError, InputValidationError} = ERRORS; const {VALIDATING_MANIFEST} = STRINGS; @@ -24,6 +21,36 @@ export const atLeastOneDefined = ( export const allDefined = (obj: Record) => Object.values(obj).every(v => v !== undefined); +/** + * Schema for parameter metadata. + */ +const parameterMetadataSchema = z + .object({ + inputs: z + .record( + z.string(), + z.object({ + unit: z.string(), + description: z.string(), + 'aggregation-method': z.string(), + }) + ) + .optional() + .nullable(), + outputs: z + .record( + z.string(), + z.object({ + unit: z.string(), + description: z.string(), + 'aggregation-method': z.string(), + }) + ) + .optional() + .nullable(), + }) + .optional(); + /** * Validation schema for manifests. */ @@ -42,11 +69,7 @@ export const manifestSchema = z.object({ explain: z.record(z.string(), z.any()).optional(), aggregation: z .object({ - metrics: z.record( - z.object({ - method: z.enum(AGGREGATION_METHODS), - }) - ), + metrics: z.array(z.string()), type: z.enum(AGGREGATION_TYPES), }) .optional() @@ -54,37 +77,14 @@ export const manifestSchema = z.object({ initialize: z.object({ plugins: z.record( z.string(), - z.object({ - path: z.string(), - method: z.string(), - 'global-config': z.record(z.string(), z.any()).optional(), - 'parameter-metadata': z - .object({ - inputs: z - .record( - z.string(), - z.object({ - unit: z.string(), - description: z.string(), - 'aggregation-method': z.string(), - }) - ) - .optional() - .nullable(), - outputs: z - .record( - z.string(), - z.object({ - unit: z.string(), - description: z.string(), - 'aggregation-method': z.string(), - }) - ) - .optional() - .nullable(), - }) - .optional(), - }) + z + .object({ + path: z.string(), + method: z.string(), + 'global-config': z.record(z.string(), z.any()).optional(), + 'parameter-metadata': parameterMetadataSchema, + }) + .optional() ), }), execution: z diff --git a/src/if-env/config/config.ts b/src/if-env/config/config.ts index cc603d6de..f3f21b0c8 100644 --- a/src/if-env/config/config.ts +++ b/src/if-env/config/config.ts @@ -7,36 +7,34 @@ import {STRINGS} from '../../common/config'; const {DISCLAIMER_MESSAGE} = STRINGS; export const CONFIG = { - IF_ENV: { - ARGS: { - manifest: { - type: String, - optional: true, - alias: 'm', - description: '[path to the manifest file]', - }, - install: { - type: Boolean, - optional: true, - alias: 'i', - description: '[command to install package.json]', - }, - cwd: { - type: Boolean, - optional: true, - alias: 'c', - description: - '[command to generate the package.json in the command working directory]', - }, - } as ArgumentConfig, - HELP: { - helpArg: 'help', - headerContentSections: [ - {header: 'Impact Framework', content: 'IF-Env Helpful keywords:'}, - ], - footerContentSections: [ - {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, - ], - } as ParseOptions, - }, + ARGS: { + manifest: { + type: String, + optional: true, + alias: 'm', + description: '[path to the manifest file]', + }, + install: { + type: Boolean, + optional: true, + alias: 'i', + description: '[command to install package.json]', + }, + cwd: { + type: Boolean, + optional: true, + alias: 'c', + description: + '[command to generate the package.json in the command working directory]', + }, + } as ArgumentConfig, + HELP: { + helpArg: 'help', + headerContentSections: [ + {header: 'Impact Framework', content: 'IF-Env Helpful keywords:'}, + ], + footerContentSections: [ + {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, + ], + } as ParseOptions, }; diff --git a/src/if-env/config/env-template.yml b/src/if-env/config/env-template.yml index ba64c88dc..dc801cbc1 100644 --- a/src/if-env/config/env-template.yml +++ b/src/if-env/config/env-template.yml @@ -14,7 +14,8 @@ tree: children: # add a chile for each distinct component you want to measure child: pipeline: # the pipeline is an ordered list of plugins you want to execute - - memory-energy-from-memory-util # must match the name in initialize! + compute: + - memory-energy-from-memory-util # must match the name in initialize! config: # any plugin specific, node-level config inputs: - timestamp: 2023-12-12T00:00:00.000Z # ISO 8061 string diff --git a/src/if-env/util/args.ts b/src/if-env/util/args.ts index 940fa7b7a..525ae14f6 100644 --- a/src/if-env/util/args.ts +++ b/src/if-env/util/args.ts @@ -12,7 +12,7 @@ import {STRINGS as COMMON_STRINGS} from '../../common/config'; import {IFEnvArgs} from '../types/process-args'; const {ParseCliParamsError, CliSourceFileError} = ERRORS; -const {IF_ENV} = CONFIG; +const {ARGS, HELP} = CONFIG; const {MANIFEST_NOT_FOUND, SOURCE_IS_NOT_YAML} = COMMON_STRINGS; /** @@ -20,7 +20,7 @@ const {MANIFEST_NOT_FOUND, SOURCE_IS_NOT_YAML} = COMMON_STRINGS; */ const validateAndParseIfEnvArgs = () => { try { - return parse(IF_ENV.ARGS, IF_ENV.HELP); + return parse(ARGS, HELP); } catch (error) { if (error instanceof Error) { throw new ParseCliParamsError(error.message); diff --git a/src/if-env/util/npm.ts b/src/if-env/util/npm.ts index f029bcdf2..c5f272255 100644 --- a/src/if-env/util/npm.ts +++ b/src/if-env/util/npm.ts @@ -79,6 +79,7 @@ export const updatePackageJsonDependencies = async ( packageJsonPath, JSON.stringify(parsedPackageJson, null, 2) ); + await fs.appendFile(packageJsonPath, '\n'); }; /** @@ -122,7 +123,7 @@ export const updatePackageJsonProperties = async ( const parsedPackageJsonContent = JSON.parse(packageJsonContent); const properties = { - name: 'if-environment', + name: packageJson.name || 'if-environment', description: packageJson.description, author: packageJson.author, bugs: packageJson.bugs, diff --git a/src/if-run/builtins/README.md b/src/if-run/builtins/README.md index ccacf83e5..e05110887 100644 --- a/src/if-run/builtins/README.md +++ b/src/if-run/builtins/README.md @@ -4,7 +4,6 @@ There are three built-in features of IF: - time-sync - CSV exporter -- groupby On this page, you can find the documentation for each of these three builtins. @@ -242,11 +241,12 @@ tree: children: child: # an advanced grouping node pipeline: - - teads-curve - - sci-e - - sci-embodied - - sci-o - - time-sync + compute: + - teads-curve + - sci-e + - sci-embodied + - sci-o + - time-sync config: teads-curve: cpu/thermal-design-power: 65 @@ -307,20 +307,19 @@ The CSV above is generated from the following yaml. The `carbon` metric is extra ```yaml tree: pipeline: - - mock-observations - - group-by - - cloud-metadata - - time-sync - - watttime - - teads-curve - - operational-carbon + observe: + - mock-observations + regroup: + - cloud/region + - name + compute: + - cloud-metadata + - time-sync + - watttime + - teads-curve + - operational-carbon defaults: grid/carbon-intensity: 500 - config: - group-by: - group: - - cloud/region - - name children: westus3: children: @@ -634,192 +633,3 @@ tree: ### CSV and aggregation The CSV representation of the output data is helpful for intuiting how the aggregation procedure works. What we refer to as "horizontal" aggregation is really an aggregation of the _rows_ of the CSV. You can replicate the IF aggregation function by summing the cells in each row of the CSV. Similarly, what we refer to as "vertical" aggregation can be replicated by summing the _columns_ in the CSV representation (this is not _exactly_ accurate because you have to skip summing both parent nodes and their children, both of which are represented in the CSV, but it is true conceptually). - -## Groupby - -Groupby is an IF plugin that reorganizes a tree according to keys provided by the user. This allows users to regroup their observations according to various properties of their application. For example, the following manifest file contains a flat array of observations. This is how you might expect data to arrive from an importer plugin, maybe one that hits a metrics API for a cloud service. - -```yaml -name: if-demo -description: demo pipeline -graph: - children: - my-app: - pipeline: - - group-by - - teads-curve - config: - group-by: - - cloud/region - - instance-type - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 99 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 23 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 12 - - timestamp: 2023-07-06T00:00 # note this time restarts at the start timstamp - duration: 300 - instance-type: B1 - cloud/region: uk-west - cpu-util: 11 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: B1 - cloud/region: uk-west - cpu-util: 67 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: B1 - cloud/region: uk-west - cpu-util: 1 -``` - -However, each observation contains an `instance-type` field that varies between observations. There are two instance types being represented in this array of observations. This means there are duplicate entries for the same timestamp in this array. This is the problem that `group-by` solves. You provide `instance-type` as a key to the `group-by` plugin and it extracts the data belonging to the different instances and separates them into independent arrays. The above example would be restructured so that instance types `A1` and `B1` have their own data, as follows: - -```yaml -graph: - children: - my-app: - pipeline: - # - group-by - - teads-curve - config: - group-by: - groups: - - cloud/region - - instance-type - children: - A1: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 99 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 23 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 12 - B1: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 11 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 67 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 1 -``` - -### Using `group-by` - -To use `group-by`, you have to initialize it as a plugin and invoke it in a pipeline. - -The initialization looks as follows: - -```yaml -initialize: -plugins: -group-by: - path: 'builtin' - method: GroupBy -``` - -You then have to provide config defining which keys to group by in each component. This is done at the component level (i.e. not global config). -For example: - -```yaml -tree: - children: - my-app: - pipeline: - - group-by - config: - group-by: - group: - - cloud/region - - instance-type -``` - -In the example above, the plugin would regroup the input data for the specific component by `cloud/region` and by `instance-type`. - -Assuming the values `A1` and `B1` are found for `instance-type` and the values `uk-east` and `uk-west` are found for `cloud/region`, the result of `group-by` would look similar to the following: - -```yaml -tree: - children: - my-app: - pipeline: - - group-by - config: - group-by: - groups: - - cloud/region - - instance-type - children: - uk-west: - children: - A1: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 99 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 23 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 12 - uk-east: - children: - B1: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 11 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 67 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 1 -``` - -This reorganized data can then be used to feed the rest of a computation pipeline. diff --git a/src/if-run/builtins/coefficient/README.md b/src/if-run/builtins/coefficient/README.md index a80f4e4fb..903a15496 100644 --- a/src/if-run/builtins/coefficient/README.md +++ b/src/if-run/builtins/coefficient/README.md @@ -97,9 +97,8 @@ tree: children: child: pipeline: - - coefficient - config: - coefficient: + compute: + - coefficient inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/copy-param/README.md b/src/if-run/builtins/copy-param/README.md index 309a83fde..2f76c98da 100644 --- a/src/if-run/builtins/copy-param/README.md +++ b/src/if-run/builtins/copy-param/README.md @@ -107,7 +107,8 @@ tree: children: child-1: pipeline: - - copy-param + compute: + - copy-param inputs: - timestamp: '2023-12-12T00:00:00.000Z' original: 'hello' diff --git a/src/if-run/builtins/csv-lookup/README.md b/src/if-run/builtins/csv-lookup/README.md index 31bdcdb17..434e234a2 100644 --- a/src/if-run/builtins/csv-lookup/README.md +++ b/src/if-run/builtins/csv-lookup/README.md @@ -135,7 +135,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/divide/README.md b/src/if-run/builtins/divide/README.md index b17925dff..abb534c8f 100644 --- a/src/if-run/builtins/divide/README.md +++ b/src/if-run/builtins/divide/README.md @@ -89,9 +89,8 @@ tree: children: child: pipeline: - - divide - config: - divide: + compute: + - divide inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/exponent/README.md b/src/if-run/builtins/exponent/README.md index b4f4d7d17..fddb55ffa 100644 --- a/src/if-run/builtins/exponent/README.md +++ b/src/if-run/builtins/exponent/README.md @@ -90,9 +90,8 @@ tree: children: child: pipeline: - - exponent - config: - exponent: + compute: + - exponent inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/group-by.ts b/src/if-run/builtins/group-by.ts deleted file mode 100644 index c8694d4cc..000000000 --- a/src/if-run/builtins/group-by.ts +++ /dev/null @@ -1,95 +0,0 @@ -import {z} from 'zod'; -import {ERRORS} from '@grnsft/if-core/utils'; -import { - GroupByPlugin, - PluginParams, - GroupByConfig, -} from '@grnsft/if-core/types'; - -import {STRINGS} from '../config'; - -import {validate} from '../../common/util/validations'; - -const {InvalidGroupingError, GlobalConfigError} = ERRORS; - -const {INVALID_GROUP_BY, MISSING_GLOBAL_CONFIG} = STRINGS; - -/** - * Plugin for inputs grouping. - */ -export const GroupBy = (): GroupByPlugin => { - const metadata = { - kind: 'groupby', - }; - - /** - * Creates structure to insert inputs by groups. - */ - const appendGroup = ( - value: PluginParams, - object: any, - groups: string[] - ): any => { - if (groups.length > 0) { - const group = groups.shift() as string; - - object.children = object.children ?? {}; - object.children[group] = object.children[group] ?? {}; - - if (groups.length === 0) { - if ( - object.children[group].inputs && - object.children[group].inputs.length > 0 - ) { - object.children[group].inputs.push(value); - } else { - object.children[group].inputs = [value]; - } - } - - appendGroup(value, object.children[group], groups); - } - - return object; - }; - - /** - * Interates over inputs, grabs config-group types values for each one. - * Based on grouping types, initializes the structure grouped structure. - */ - const execute = (inputs: PluginParams[], config: GroupByConfig) => - inputs.reduce((acc, input) => { - const validatedConfig = validateConfig(config); - const groups = validatedConfig.group.map(groupType => { - if (!input[groupType]) { - throw new InvalidGroupingError(INVALID_GROUP_BY(groupType)); - } - - return input[groupType]; - }); - - acc = { - ...acc, - ...appendGroup(input, acc, groups), - }; - - return acc; - }, {} as any).children; - - /** - * Validates config parameter. - */ - const validateConfig = (config: GroupByConfig) => { - if (!config) { - throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); - } - - const schema = z.object({ - group: z.array(z.string()).min(1), - }); - - return validate>(schema, config); - }; - - return {metadata, execute}; -}; diff --git a/src/if-run/builtins/index.ts b/src/if-run/builtins/index.ts index d22491aca..36a414c95 100644 --- a/src/if-run/builtins/index.ts +++ b/src/if-run/builtins/index.ts @@ -1,5 +1,3 @@ -export {GroupBy} from './group-by'; -export {TimeSync} from './time-sync'; export {Interpolation} from './interpolation'; export {MockObservations} from './mock-observations'; export {Divide} from './divide'; @@ -14,3 +12,4 @@ export {CSVLookup} from './csv-lookup'; export {Shell} from './shell'; export {Regex} from './regex'; export {Copy} from './copy-param'; +export {TimeSync} from './time-sync'; diff --git a/src/if-run/builtins/interpolation/README.md b/src/if-run/builtins/interpolation/README.md index a6d897f32..243b70e70 100644 --- a/src/if-run/builtins/interpolation/README.md +++ b/src/if-run/builtins/interpolation/README.md @@ -131,7 +131,8 @@ tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 @@ -159,7 +160,8 @@ tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/mock-observations/README.md b/src/if-run/builtins/mock-observations/README.md index 91f418ccb..53bd89924 100644 --- a/src/if-run/builtins/mock-observations/README.md +++ b/src/if-run/builtins/mock-observations/README.md @@ -62,7 +62,7 @@ const result = await mockObservations.execute([]); ### manifest Example -IF users will typically call the plugin as part of a pipeline defined in a `manifest` file. In this case, instantiating the plugin is handled by `ie` and does not have to be done explicitly by the user. The following is an example `manifest` that calls `mock-observation`: +IF users will typically call the plugin as part of a pipeline defined in a `manifest` file. In this case, instantiating the plugin is handled by `if-run` and does not have to be done explicitly by the user. The following is an example `manifest` that calls `mock-observation`: ```yaml name: mock-observation-demo @@ -96,7 +96,8 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: ``` diff --git a/src/if-run/builtins/multiply/README.md b/src/if-run/builtins/multiply/README.md index 86a65aecb..dc6cde7c7 100644 --- a/src/if-run/builtins/multiply/README.md +++ b/src/if-run/builtins/multiply/README.md @@ -87,9 +87,8 @@ tree: children: child: pipeline: - - multiply - config: - multiply: + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/regex/README.md b/src/if-run/builtins/regex/README.md index d9f382274..2c6a4cd88 100644 --- a/src/if-run/builtins/regex/README.md +++ b/src/if-run/builtins/regex/README.md @@ -82,9 +82,8 @@ tree: children: child: pipeline: - - regex - config: - regex: + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/sci-embodied/README.md b/src/if-run/builtins/sci-embodied/README.md index ed9c1841e..a4c304c6d 100644 --- a/src/if-run/builtins/sci-embodied/README.md +++ b/src/if-run/builtins/sci-embodied/README.md @@ -102,7 +102,8 @@ tree: children: child: pipeline: - - sci-embodied # duration & config -> embodied + compute: + - sci-embodied # duration & config -> embodied defaults: device/emissions-embodied: 1533.120 # gCO2eq device/expected-lifespan: 3 # 3 years in seconds diff --git a/src/if-run/builtins/sci-embodied/index.ts b/src/if-run/builtins/sci-embodied/index.ts index 35e012b72..e8b69cd2b 100644 --- a/src/if-run/builtins/sci-embodied/index.ts +++ b/src/if-run/builtins/sci-embodied/index.ts @@ -35,6 +35,7 @@ export const SciEmbodied = ( 'resources-total': { description: 'total resources available', unit: 'count', + 'aggregation-method': 'none', }, 'vcpus-allocated': { description: 'number of vcpus allocated to particular resource', diff --git a/src/if-run/builtins/sci/README.md b/src/if-run/builtins/sci/README.md index d0b08c9c5..31b15afed 100644 --- a/src/if-run/builtins/sci/README.md +++ b/src/if-run/builtins/sci/README.md @@ -82,8 +82,8 @@ tree: children: child: pipeline: - - sci - config: + compute: + - sci inputs: - timestamp: 2023-07-06T00:00 carbon: 5 diff --git a/src/if-run/builtins/shell/README.md b/src/if-run/builtins/shell/README.md index 5e529f922..d114adb3b 100644 --- a/src/if-run/builtins/shell/README.md +++ b/src/if-run/builtins/shell/README.md @@ -91,7 +91,8 @@ tree: children: child: pipeline: - - sampler + compute: + - sampler inputs: - timestamp: 2023-07-06T00:00 duration: 1 # Secs @@ -117,7 +118,8 @@ tree: children: child: pipeline: - - sampler + compute: + - sampler inputs: - timestamp: 2023-07-06T00:00 duration: 1 # Secs diff --git a/src/if-run/builtins/subtract/README.md b/src/if-run/builtins/subtract/README.md index 89a186ab3..776526c7e 100644 --- a/src/if-run/builtins/subtract/README.md +++ b/src/if-run/builtins/subtract/README.md @@ -87,9 +87,8 @@ tree: children: child: pipeline: - - subtract - config: - subtract: + compute: + - subtract inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/sum/README.md b/src/if-run/builtins/sum/README.md index 014111c6c..998c54c11 100644 --- a/src/if-run/builtins/sum/README.md +++ b/src/if-run/builtins/sum/README.md @@ -97,9 +97,8 @@ tree: children: child: pipeline: - - sum - config: - sum: + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/src/if-run/builtins/time-sync.ts b/src/if-run/builtins/time-sync.ts index 9d589c491..e9f4ee1bc 100644 --- a/src/if-run/builtins/time-sync.ts +++ b/src/if-run/builtins/time-sync.ts @@ -36,6 +36,21 @@ const { INVALID_DATETIME, } = STRINGS; +/** + * Time synchronization plugin converted into framework integrated tool. + * It can't be requested in `initialize.plugins` section anymore. Instead describe configuration in context. + * @example + * ```yaml + * name: time-sync + * description: sample in time sync lib + * tags: sample, time, sync + * time-sync: + * start-time: '2023-12-12T00:00:00.000Z' + * end-time: '2023-12-12T00:01:00.000Z' + * interval: 5 + * allow-padding: true + * ``` + */ export const TimeSync = ( globalConfig: TimeNormalizerConfig, parametersMetadata: PluginParametersMetadata @@ -131,14 +146,16 @@ export const TimeSync = ( return resampleInputs(sortedInputs, timeParams) as PluginParams[]; }; + /** + * Dates are passed to `time-sync` both in ISO 8601 format + * and as a Date object (from the deserialization of a YAML file). + * If the YAML parser fails to identify as a date, it passes as a string. + */ const parseDate = (date: Date | string) => { if (!date) { return DateTime.invalid('Invalid date'); } - // dates are passed to time-sync.ts both in ISO 8601 format - // and as a Date object (from the deserialization of a YAML file) - // if the YAML parser fails to identify as a date, it passes as a string if (isDate(date)) { return DateTime.fromJSDate(date); } diff --git a/src/if-run/config/config.ts b/src/if-run/config/config.ts index 91f366543..9bd39d89f 100644 --- a/src/if-run/config/config.ts +++ b/src/if-run/config/config.ts @@ -7,49 +7,68 @@ import {IfRunArgs} from '../types/process-args'; const {DISCLAIMER_MESSAGE} = STRINGS; export const CONFIG = { - IF_RUN: { - ARGS: { - manifest: { - type: String, - optional: true, - alias: 'm', - description: '[path to the input file]', - }, - output: { - type: String, - optional: true, - alias: 'o', - description: '[path to the output file]', - }, - 'no-output': { - type: Boolean, - optional: true, - alias: 'n', - description: '[prevent output to the console]', - }, - help: { - type: Boolean, - optional: true, - alias: 'h', - description: '[prints out the above help instruction]', - }, - debug: { - type: Boolean, - optional: true, - alias: 'd', - description: '[prints out debug logs to the console]', - }, - } as ArgumentConfig, - HELP: { - helpArg: 'help', - headerContentSections: [ - {header: 'Impact Framework', content: 'Helpful keywords:'}, - ], - footerContentSections: [ - {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, - ], - } as ParseOptions, - }, + ARGS: { + manifest: { + type: String, + optional: true, + alias: 'm', + description: '[path to the input file]', + }, + output: { + type: String, + optional: true, + alias: 'o', + description: '[path to the output file]', + }, + 'override-params': { + type: String, + optional: true, + alias: 'p', + description: '[path to a parameter file that overrides our defaults]', + }, + 'no-output': { + type: Boolean, + optional: true, + alias: 'n', + description: '[prevent output to the console]', + }, + help: { + type: Boolean, + optional: true, + alias: 'h', + description: '[prints out the above help instruction]', + }, + debug: { + type: Boolean, + optional: true, + alias: 'd', + description: '[prints out debug logs to the console]', + }, + observe: { + type: Boolean, + optional: true, + description: '[executes only observe pipeline of the manifest]', + }, + regroup: { + type: Boolean, + optional: true, + description: '[executes only regroup pipeline of the manifest]', + }, + compute: { + type: Boolean, + optional: true, + description: '[executes only compute pipeline of the manifest]', + }, + } as ArgumentConfig, + HELP: { + helpArg: 'help', + headerContentSections: [ + {header: 'Impact Framework', content: 'Helpful keywords:'}, + ], + footerContentSections: [ + {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, + ], + } as ParseOptions, GITHUB_PATH: 'https://github.com', NATIVE_PLUGIN: 'if-plugins', AGGREGATION_ADDITIONAL_PARAMS: ['timestamp', 'duration'], diff --git a/src/if-run/config/strings.ts b/src/if-run/config/strings.ts index cdbc7c8c2..e5e6e449b 100644 --- a/src/if-run/config/strings.ts +++ b/src/if-run/config/strings.ts @@ -23,7 +23,8 @@ export const STRINGS = { `Aggregation is not possible for given ${metric} since method is 'none'.`, METRIC_MISSING: (metric: string, index: number) => `Aggregation metric ${metric} is not found in inputs[${index}].`, - INVALID_GROUP_BY: (type: string) => `Invalid group ${type}.`, + INVALID_GROUP_KEY: (key: string) => `Invalid group ${key}.`, + REGROUP_ERROR: 'not an array or should contain at least one key', INVALID_EXHAUST_PLUGIN: (pluginName: string) => `Invalid exhaust plugin: ${pluginName}.`, UNKNOWN_PARAM: (name: string) => diff --git a/src/if-run/index.ts b/src/if-run/index.ts index 5c1084cf9..6d9f61a78 100644 --- a/src/if-run/index.ts +++ b/src/if-run/index.ts @@ -12,8 +12,10 @@ import {compute} from './lib/compute'; import {exhaust} from './lib/exhaust'; import {explain} from './lib/explain'; +import {AGGREGATION_METHODS} from './types/aggregation'; + import {parseIfRunProcessArgs} from './util/args'; -import {andHandle, storeAggregationMethods} from './util/helpers'; +import {andHandle} from './util/helpers'; import {STRINGS} from './config'; @@ -22,7 +24,14 @@ const {DISCLAIMER_MESSAGE} = COMMON_STRINGS; const impactEngine = async () => { const options = parseIfRunProcessArgs(); - const {inputPath, outputOptions, debug} = options; + const { + inputPath, + outputOptions, + debug, + observe, + regroup, + compute: computeFlag, + } = options; debugLogger.overrideConsoleMethods(!!debug); @@ -34,17 +43,27 @@ const impactEngine = async () => { try { const {tree, ...context} = validateManifest(envManifest); - const pluginStorage = await initialize(context.initialize.plugins); if (context.aggregation) { - storeAggregationMetrics({metrics: context.aggregation?.metrics}); + const convertMetrics = context.aggregation?.metrics.map( + (metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + }) + ); + + storeAggregationMetrics(...convertMetrics); } - storeAggregationMethods(context.initialize.plugins, pluginStorage); + const pluginStorage = await initialize(context); + const computedTree = await compute(tree, { + context, + pluginStorage, + observe, + regroup, + compute: computeFlag, + }); - const computedTree = await compute(tree, {context, pluginStorage}); const aggregatedTree = aggregate(computedTree, context.aggregation); - envManifest.explainer && (context.explain = explain()); await exhaust(aggregatedTree, context, outputOptions); diff --git a/src/if-run/lib/aggregate.ts b/src/if-run/lib/aggregate.ts index 508117bb3..b80d0048a 100644 --- a/src/if-run/lib/aggregate.ts +++ b/src/if-run/lib/aggregate.ts @@ -5,7 +5,7 @@ import {logger} from '../../common/util/logger'; import { AggregationParams, AggregationParamsSure, - AggregationParamsWithoutType, + AggregationMetricsWithMethod, } from '../../common/types/manifest'; import {aggregateInputsIntoOne} from '../util/aggregation-helper'; @@ -39,7 +39,7 @@ const getIthElementsFromChildren = (children: any, i: number) => { * 1. Gets the i'th element from each childrens outputs (treating children as rows and we are after a column of data). * 2. Now we just aggregate over the `ithSliceOfOutputs` the same as we did for the normal outputs. */ -const temporalAggregation = (node: any, metrics: AggregationMetric) => { +const temporalAggregation = (node: any, metrics: AggregationMetric[]) => { const outputs: PluginParams[] = []; const values: any = Object.values(node.children); @@ -63,7 +63,9 @@ const temporalAggregation = (node: any, metrics: AggregationMetric) => { * 5. Now a grouping node has it's own outputs, it can horizotnally aggregate them. */ const aggregateNode = (node: any, aggregationParams: AggregationParamsSure) => { - const metrics = aggregationParams!.metrics; + const metrics: AggregationMetric[] = aggregationParams!.metrics.map( + metric => ({[metric]: 'none'}) + ); const type = aggregationParams!.type; if (node.children) { @@ -108,12 +110,12 @@ export const aggregate = (tree: any, aggregationParams: AggregationParams) => { * Gets or stores aggregation metrics. */ export const storeAggregationMetrics = ( - aggregationParams?: AggregationParamsWithoutType + aggregationMetrics?: AggregationMetricsWithMethod ) => { - if (aggregationParams?.metrics) { + if (aggregationMetrics) { metricManager.metrics = { ...metricManager.metrics, - ...aggregationParams?.metrics, + ...aggregationMetrics, }; } @@ -146,8 +148,11 @@ export const getAggregationMethod = (unitName: string) => { memoizedLog(console.debug, CHECKING_AGGREGATION_METHOD(unitName)); const aggregationMetricsStorage = storeAggregationMetrics(); - if (aggregationMetricsStorage && `${unitName}` in aggregationMetricsStorage) { - return aggregationMetricsStorage[unitName].method; + if ( + aggregationMetricsStorage && + Object.keys(aggregationMetricsStorage).includes(unitName) + ) { + return aggregationMetricsStorage[unitName]; } memoizedLog(logger.warn, UNKNOWN_PARAM(unitName)); diff --git a/src/if-run/lib/compute.ts b/src/if-run/lib/compute.ts index 02cc8e0e4..41a153448 100644 --- a/src/if-run/lib/compute.ts +++ b/src/if-run/lib/compute.ts @@ -1,20 +1,22 @@ -import {PluginParams, GroupByConfig} from '@grnsft/if-core/types'; +import {PluginParams} from '@grnsft/if-core/types'; + +import {Regroup} from './regroup'; +import {addExplainData} from './explain'; import {mergeObjects} from '../util/helpers'; import {debugLogger} from '../../common/util/debug-logger'; import {STRINGS} from '../config/strings'; -import {isExecute, isGroupBy} from '../types/interface'; -import {ComputeParams, Node, Params} from '../types/compute'; -import {addExplainData} from './explain'; +import {ComputeParams, Node, PhasedPipeline} from '../types/compute'; +import {isExecute} from '../types/interface'; -const {MERGING_DEFAULTS_WITH_INPUT_DATA, COMPUTING_PIPELINE_FOR_NODE} = STRINGS; +const {MERGING_DEFAULTS_WITH_INPUT_DATA} = STRINGS; /** * Traverses all child nodes based on children grouping. */ -const traverse = async (children: any, params: Params) => { +const traverse = async (children: any, params: ComputeParams) => { for (const child in children) { await computeNode(children[child], params); } @@ -46,15 +48,21 @@ const mergeDefaults = ( * 2. If it's a grouping node, then first of all computes all it's children. * This is doing a depth first traversal. * 3. Otherwise merges the defaults into the inputs. - * 4. Goes through the pipeline plugins, by checking if it's `execute` plugin. If so sets outputs. - * If is a `groupby` plugin, it will return child components rather than outputs. - * 5. Since after `groupby`, there are new child components, then computes them. - * Note: `pipeline` now equals the remaining plu.gins to apply to each child + * 4. Iterates over pipeline phases (observe, regroup, compute). + * 5. Observe plugins are used to insert input values + * (isolated execution can be achived by passing `--observe` flag to CLI command). + * 6. Regroup plugin is used to group existing inputs by criteria + * (isolated execution can be achived by passing `--regroup` flag to CLI command). + * Since it creates new children for node, existing inputs and outputs are dropped and recursive traversal is called + * for newbord child component. + * 7. Compute plugins are used to do desired computations and appending the result to outputs + * (isolated execution can be achived by passing `--compute` flag to CLI command). */ -const computeNode = async (node: Node, params: Params): Promise => { - const pipeline = (node.pipeline || params.pipeline) as string[]; +const computeNode = async (node: Node, params: ComputeParams): Promise => { + const pipeline = node.pipeline || (params.pipeline as PhasedPipeline); const config = node.config || params.config; const defaults = node.defaults || params.defaults; + const noFlags = !params.observe && !params.regroup && !params.compute; if (node.children) { return traverse(node.children, { @@ -67,50 +75,67 @@ const computeNode = async (node: Node, params: Params): Promise => { let inputStorage = structuredClone(node.inputs) as PluginParams[]; inputStorage = mergeDefaults(inputStorage, defaults); - const pipelineCopy = structuredClone(pipeline); - - while (pipelineCopy.length !== 0) { - const pluginName = pipelineCopy.shift() as string; - const plugin = params.pluginStorage.get(pluginName); - const nodeConfig = config && config[pluginName]; - - console.debug(COMPUTING_PIPELINE_FOR_NODE(pluginName)); - debugLogger.setExecutingPluginName(pluginName); - - if (isExecute(plugin)) { - inputStorage = await plugin.execute(inputStorage, nodeConfig); - - if (params.context.explainer) { - addExplainData({ - pluginName, - metadata: plugin.metadata, - pluginData: params.context.initialize.plugins[pluginName], - }); + const pipelineCopy = structuredClone(pipeline) || {}; + + /** + * If iteration is on observe pipeline, then executes observe plugins and sets the inputs value. + */ + if ((noFlags || params.observe) && pipelineCopy.observe) { + while (pipelineCopy.observe.length !== 0) { + const pluginName = pipelineCopy.observe.shift() as string; + const plugin = params.pluginStorage.get(pluginName); + const nodeConfig = config && config[pluginName]; + + if (isExecute(plugin)) { + inputStorage = await plugin.execute(inputStorage, nodeConfig); + node.inputs = inputStorage; + + if (params.context.explainer) { + addExplainData({ + pluginName, + metadata: plugin.metadata, + pluginData: params.context.initialize!.plugins[pluginName], + }); + } + + node.outputs = inputStorage; } - - debugLogger.setExecutingPluginName(); - - node.outputs = inputStorage; } + } - if (isGroupBy(plugin)) { - node.children = await plugin.execute( - inputStorage, - nodeConfig as GroupByConfig - ); - delete node.inputs; - delete node.outputs; - - await traverse(node.children, { - ...params, - pipeline: pipelineCopy, - defaults, - config, - }); + /** + * If regroup is requested, execute regroup strategy, delete child's inputs, outputs and empty regroup array. + */ + if ((noFlags || params.regroup) && pipelineCopy.regroup) { + node.children = Regroup(inputStorage, pipelineCopy.regroup); + delete node.inputs; + delete node.outputs; - debugLogger.setExecutingPluginName(); + return traverse(node.children, { + ...params, + pipeline: { + ...pipelineCopy, + regroup: undefined, + }, + defaults, + config, + }); + } - break; + /** + * If iteration is on compute plugin, then executes compute plugins and sets the outputs value. + */ + if ((noFlags || params.compute) && pipelineCopy.compute) { + while (pipelineCopy.compute.length !== 0) { + const pluginName = pipelineCopy.compute.shift() as string; + const plugin = params.pluginStorage.get(pluginName); + const nodeConfig = config && config[pluginName]; + + if (isExecute(plugin)) { + inputStorage = await plugin.execute(inputStorage, nodeConfig); + node.outputs = inputStorage; + debugLogger.setExecutingPluginName(); + } } } }; diff --git a/src/if-run/lib/explain.ts b/src/if-run/lib/explain.ts index add1a02b6..dce49b75a 100644 --- a/src/if-run/lib/explain.ts +++ b/src/if-run/lib/explain.ts @@ -30,8 +30,8 @@ export const addExplainData = (params: ExplainParams) => { const {pluginName, pluginData, metadata} = params; const plugin = { [pluginName]: { - method: pluginData.method, - path: pluginData.path, + method: pluginData!.method, + path: pluginData!.path, inputs: metadata?.inputs || 'undefined', outputs: metadata?.outputs || 'undefined', }, diff --git a/src/if-run/lib/initialize.ts b/src/if-run/lib/initialize.ts index 5adcc2b18..181e88d15 100644 --- a/src/if-run/lib/initialize.ts +++ b/src/if-run/lib/initialize.ts @@ -9,8 +9,9 @@ import {pluginStorage} from '../util/plugin-storage'; import {CONFIG, STRINGS} from '../config'; import {PluginInterface} from '../types/interface'; -import {GlobalPlugins, PluginOptions} from '../../common/types/manifest'; +import {Context, PluginOptions} from '../../common/types/manifest'; import {PluginStorageInterface} from '../types/plugin-storage'; +import {storeAggregationMetrics} from './aggregate'; const { PluginInitializationError, @@ -84,7 +85,7 @@ const initPlugin = async ( path, 'global-config': globalConfig, 'parameter-metadata': parameterMetadata, - } = initPluginParams; + } = initPluginParams!; console.debug(INITIALIZING_PLUGIN(method)); @@ -105,14 +106,22 @@ const initPlugin = async ( * Registers all plugins from `manifest`.`initialize` property. */ export const initialize = async ( - plugins: GlobalPlugins + context: Context ): Promise => { console.debug(INITIALIZING_PLUGINS); - + const {plugins} = context.initialize; const storage = pluginStorage(); for await (const pluginName of Object.keys(plugins)) { const plugin = await initPlugin(plugins[pluginName]); + const parameters = {...plugin.metadata.inputs, ...plugin.metadata.outputs}; + + Object.keys(parameters).forEach(key => { + storeAggregationMetrics({ + [key]: parameters[key]['aggregation-method'], + }); + }); + storage.set(pluginName, plugin); } diff --git a/src/if-run/lib/regroup.ts b/src/if-run/lib/regroup.ts new file mode 100644 index 000000000..affd51307 --- /dev/null +++ b/src/if-run/lib/regroup.ts @@ -0,0 +1,80 @@ +import {z} from 'zod'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {PluginParams} from '@grnsft/if-core/types'; + +import {validate} from '../../common/util/validations'; + +import {STRINGS} from '../config'; + +const {InvalidGroupingError} = ERRORS; + +const {INVALID_GROUP_KEY, REGROUP_ERROR} = STRINGS; + +/** + * Grouping strategy. + */ +export const Regroup = (inputs: PluginParams[], groups: string[]) => { + /** + * Creates structure to insert inputs by groups. + */ + const appendGroup = (value: PluginParams, object: any, groups: string[]) => { + if (groups.length > 0) { + const group = groups.shift() as string; + + object.children = object.children ?? {}; + object.children[group] = object.children[group] ?? {}; + + if (groups.length === 0) { + if ( + object.children[group].inputs && + object.children[group].inputs.length > 0 + ) { + object.children[group].inputs.push(value); + } else { + object.children[group].inputs = [value]; + } + } + + appendGroup(value, object.children[group], groups); + } + + return object; + }; + + /** + * Validates groups array. + */ + const validateGroups = (regroup: string[]) => { + const inputData = {regroup}; + const validationSchema = z.record( + z.string(), + z.array(z.string(), {message: REGROUP_ERROR}).min(1) + ); + + validate(validationSchema, inputData); + + return groups; + }; + + /** + * Interates over inputs, grabs group values for each one. + * Based on grouping, initializes the structure. + */ + return inputs.reduce((acc, input) => { + const validtedGroups = validateGroups(groups); + const groupsWithData = validtedGroups.map(groupType => { + if (!input[groupType]) { + throw new InvalidGroupingError(INVALID_GROUP_KEY(groupType)); + } + + return input[groupType]; + }); + + acc = { + ...acc, + ...appendGroup(input, acc, groupsWithData), + }; + + return acc; + }, {} as any).children; +}; diff --git a/src/if-run/types/aggregation.ts b/src/if-run/types/aggregation.ts index a7e895da4..c3b143a1f 100644 --- a/src/if-run/types/aggregation.ts +++ b/src/if-run/types/aggregation.ts @@ -3,7 +3,5 @@ export type AggregationResult = Record; export const AGGREGATION_TYPES = ['horizontal', 'vertical', 'both'] as const; export const AGGREGATION_METHODS = ['sum', 'avg', 'none'] as const; -export type AggregationMetric = Record< - string, - {method: 'sum' | 'avg' | 'none'} ->; +export type AggregationMethodTypes = 'sum' | 'avg' | 'none'; +export type AggregationMetric = Record; diff --git a/src/if-run/types/compute.ts b/src/if-run/types/compute.ts index bfe294241..6777bf553 100644 --- a/src/if-run/types/compute.ts +++ b/src/if-run/types/compute.ts @@ -7,24 +7,28 @@ export type NodeConfig = { [key: string]: Record; }; -export type Params = { +export type PhasedPipeline = { + observe?: string[]; + regroup?: string[]; + compute?: string[]; +}; + +export type ComputeParams = { pluginStorage: PluginStorageInterface; context: Context; - pipeline?: string[]; + pipeline?: PhasedPipeline; config?: NodeConfig; defaults?: PluginParams; + observe?: Boolean; + regroup?: Boolean; + compute?: Boolean; }; export type Node = { children?: any; - pipeline?: string[]; + pipeline?: PhasedPipeline; config?: NodeConfig; defaults?: PluginParams; inputs?: PluginParams[]; outputs?: PluginParams[]; }; - -export type ComputeParams = { - context: Context; - pluginStorage: PluginStorageInterface; -}; diff --git a/src/if-run/types/explain.ts b/src/if-run/types/explain.ts index b1b684882..c7f1a6d38 100644 --- a/src/if-run/types/explain.ts +++ b/src/if-run/types/explain.ts @@ -1,7 +1,9 @@ import {ParameterMetadata} from '@grnsft/if-core/types'; +import {PluginOptions} from '../../common/types/manifest'; + export type ExplainParams = { pluginName: string; - pluginData: {method: string; path: string}; + pluginData: PluginOptions; metadata: {inputs?: ParameterMetadata; outputs?: ParameterMetadata}; }; diff --git a/src/if-run/types/interface.ts b/src/if-run/types/interface.ts index 6834a3ebb..3884305a2 100644 --- a/src/if-run/types/interface.ts +++ b/src/if-run/types/interface.ts @@ -1,9 +1,6 @@ -import {ExecutePlugin, GroupByPlugin} from '@grnsft/if-core/types'; +import {ExecutePlugin} from '@grnsft/if-core/types'; -export type PluginInterface = ExecutePlugin | GroupByPlugin; +export type PluginInterface = ExecutePlugin; -export const isExecute = (plugin: PluginInterface): plugin is ExecutePlugin => +export const isExecute = (plugin: ExecutePlugin): plugin is ExecutePlugin => (plugin as ExecutePlugin).metadata.kind === 'execute'; - -export const isGroupBy = (plugin: PluginInterface): plugin is GroupByPlugin => - (plugin as GroupByPlugin).metadata.kind === 'groupby'; diff --git a/src/if-run/types/process-args.ts b/src/if-run/types/process-args.ts index 184877bd0..298cadda6 100644 --- a/src/if-run/types/process-args.ts +++ b/src/if-run/types/process-args.ts @@ -3,6 +3,9 @@ export interface IfRunArgs { output?: string; 'no-output'?: boolean; debug?: boolean; + observe?: boolean; + regroup?: boolean; + compute?: boolean; } export interface ProcessArgsOutputs { @@ -13,6 +16,9 @@ export interface ProcessArgsOutputs { }; paramPath?: string; debug?: boolean; + observe?: boolean; + regroup?: boolean; + compute?: boolean; } export interface Options { diff --git a/src/if-run/util/aggregation-helper.ts b/src/if-run/util/aggregation-helper.ts index 87d3e15fc..10b1a0bdd 100644 --- a/src/if-run/util/aggregation-helper.ts +++ b/src/if-run/util/aggregation-helper.ts @@ -7,40 +7,21 @@ import {AggregationMetric, AggregationResult} from '../types/aggregation'; import {getAggregationMethod} from '../lib/aggregate'; -const {InvalidAggregationMethodError, MissingAggregationParamError} = ERRORS; -const {INVALID_AGGREGATION_METHOD, METRIC_MISSING} = STRINGS; +const {MissingAggregationParamError} = ERRORS; +const {METRIC_MISSING} = STRINGS; const {AGGREGATION_ADDITIONAL_PARAMS} = CONFIG; -/** - * Validates metrics array before applying aggregator. - * If aggregation method is `none`, then throws error. - */ -const checkIfMetricsAreValid = (metrics: AggregationMetric) => { - Object.keys(metrics).forEach(metric => { - const method = metrics[metric].method; - - if (method === 'none') { - throw new InvalidAggregationMethodError( - INVALID_AGGREGATION_METHOD(metric) - ); - } - }); -}; - /** * Aggregates child node level metrics. Validates if metric aggregation type is `none`, then rejects with error. * Appends aggregation additional params to metrics. Otherwise iterates over inputs by aggregating per given `metrics`. */ export const aggregateInputsIntoOne = ( inputs: PluginParams[], - metrics: AggregationMetric, + metrics: AggregationMetric[], isTemporal?: boolean ) => { - checkIfMetricsAreValid(metrics); - const extendedMetrics = [ - ...Object.keys(metrics), - ...AGGREGATION_ADDITIONAL_PARAMS, - ]; + const metricsKeys: string[] = metrics.map(metric => Object.keys(metric)[0]); + const extendedMetrics = [...metricsKeys, ...AGGREGATION_ADDITIONAL_PARAMS]; return inputs.reduce((acc, input, index) => { for (const metric of extendedMetrics) { diff --git a/src/if-run/util/args.ts b/src/if-run/util/args.ts index 140c5d140..538d37a36 100644 --- a/src/if-run/util/args.ts +++ b/src/if-run/util/args.ts @@ -13,16 +13,16 @@ import {IfRunArgs, ProcessArgsOutputs} from '../types/process-args'; const {ParseCliParamsError, CliSourceFileError} = ERRORS; -const {IF_RUN} = CONFIG; +const {ARGS, HELP} = CONFIG; const {NO_OUTPUT} = STRINGS; const {SOURCE_IS_NOT_YAML, MANIFEST_IS_MISSING} = COMMON_STRINGS; /** - * Validates `ie` process arguments. + * Validates `if-run` process arguments. */ const validateAndParseProcessArgs = () => { try { - return parse(IF_RUN.ARGS, IF_RUN.HELP); + return parse(ARGS, HELP); } catch (error) { if (error instanceof Error) { throw new ParseCliParamsError(error.message); @@ -33,9 +33,8 @@ const validateAndParseProcessArgs = () => { }; /** - * 1. Parses process arguments like `manifest`, `output`, `help` and `debug`. - * 2. Checks if `help` param is provided, then logs help message and exits. - * 3. If output params are missing, warns user about it. + * 1. Parses process arguments for `if-run`. + * 2. If output params are missing, warns user about it. * 3. Otherwise checks if `manifest` param is there, then processes with checking if it's a yaml file. * If it is, then returns object containing full path. * 4. If params are missing or invalid, then rejects with `ParseCliParamsError`. @@ -46,6 +45,9 @@ export const parseIfRunProcessArgs = (): ProcessArgsOutputs => { output, 'no-output': noOutput, debug, + observe, + regroup, + compute, } = validateAndParseProcessArgs(); if (!output && noOutput) { @@ -61,6 +63,9 @@ export const parseIfRunProcessArgs = (): ProcessArgsOutputs => { ...(noOutput && {noOutput}), }, debug, + observe, + regroup, + compute, }; } diff --git a/src/if-run/util/helpers.ts b/src/if-run/util/helpers.ts index da50f22f4..2939022e2 100644 --- a/src/if-run/util/helpers.ts +++ b/src/if-run/util/helpers.ts @@ -62,10 +62,9 @@ export const storeAggregationMethods = ( const {'aggregation-method': aggregationMethod} = parameterMetadata; if (aggregationMethod) { - const metrics = { - [parameterName]: {method: aggregationMethod}, - }; - storeAggregationMetrics({metrics}); + const metrics = {[parameterName]: aggregationMethod}; + + storeAggregationMetrics(metrics); } } );