Merge branch 'main' into mapping-feature

Green-Software-Foundation · Aug 8, 2024 · fb44137 · fb44137
2 parents b8bc3c4 + eef1183
commit fb44137
Show file tree

Hide file tree

Showing 52 changed files with 861 additions and 1,016 deletions.
diff --git a/.github/workflows/release-commit.yml b/.github/workflows/release-commit.yml
@@ -8,7 +8,7 @@ env:
   PRE_RELEASE: ${{ github.event.release.prerelease}}
 
 jobs:
-  release-commit-pr:
+  unit-tests:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -28,9 +28,45 @@ jobs:
       - name: Run unit tests
         run: npm test
 
+  integration-tests:
+    runs-on: ubuntu-latest
+    needs: unit-tests
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          registry-url: https://registry.npmjs.org/
+
+      - name: Install dependencies
+        run: npm ci
+
       - name: Run integration tests
         run: npm run if-check -- -d manifests/outputs
 
+  publish:
+    runs-on: ubuntu-latest
+    needs: integration-tests
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          registry-url: https://registry.npmjs.org/
+
+      - name: Install dependencies
+        run: npm ci
+
       - name: Initialize git user email
         run: git config --global user.email "${{ vars.RELEASE_USER_EMAIL }}"
 

diff --git a/Refactor-migration-guide.md b/Refactor-migration-guide.md
@@ -125,24 +125,23 @@ There have also been some changes to the structure of manifest files. Some of th
 
 - **Node level config**
 
-  We have also introduced the concept of node-level config. This is designed for pluin configuration that might vary between components in the tree. For example, for each child in the tree you might wish to use the `groupby` plugin to group the outputs according to a different set of keys.
+  We have also introduced the concept of node-level config. This is designed for pluin configuration that might vary between components in the tree. For example, for each child in the tree you might wish to use the `regroup` feature to group the outputs according to a different set of keys.
 
   ```yaml
   tree:
     children:
       child-1:
         pipeline:
-          - teads-curve
-          - sci-e
-          - sci-embodied
-          - sci-o
-          - time-sync
-          - sci
-        config:
-          group-by:
-            group:
-              - region
-              - cloud/instance-type
+          compute:
+            - teads-curve
+            - sci-e
+            - sci-embodied
+            - sci-o
+            - time-sync
+            - sci
+          regroup:
+            - region
+            - cloud/instance-type
   ```
 
 - **Defaults**
@@ -156,12 +155,13 @@ There have also been some changes to the structure of manifest files. Some of th
     children:
       child-1:
         pipeline:
-          - teads-curve
-          - sci-e
-          - sci-embodied
-          - sci-o
-          - time-sync
-          - sci
+          compute:
+            - teads-curve
+            - sci-e
+            - sci-embodied
+            - sci-o
+            - time-sync
+            - sci
        defaults:
           cpu/thermal-design-power: 100
           grid/carbon-intensity: 800
@@ -199,9 +199,7 @@ The aggregate plugin aggregates data in two ways: first it condenses individual
 
 This is a builtin feature of IF, meaning it does not have to be initialized as a plugin. Instead, you just have to include a short config block in the top of the manifest file. There are two pieces of information required:
 
-- `metrics`: which metrics do you want to aggregate? Every metric you provide here must exist in the output array.
-
-  - `method`: the aggregation method for the specied metric
+- `metrics`: which metrics do you want to aggregate? Every metric you provide here must exist in the output array and be described in the `parameter-metadata` of the plugin.
 
 - `type`: the options are `horizontal`, `vertical` or both. Horizontal aggregation is the type that condenses each time series into a single summary value. Vertical aggregation is aggregated across components.
 
@@ -210,45 +208,10 @@ Here's what the config block should look like:
 ```yaml
 aggregation:
   metrics:
-    'carbon':
-      method: 'sum'
+    - carbon
   type: 'both'
 ```
 
-### Groupby
-
-Groupby allows you to regroup your outputs according to keys you define. For example, maybe you want to group your outputs by region (show me all the outputs for applications run in `uk-south` etc). Groupby _is_ a plugin that needs to be initialized in the manifest.
-
-You can initialize the plugin as follows:
-
-```yaml
-initialize:
-  plugins:
-    'group-by':
-      path: builtin
-      method: GroupBy
-```
-
-Then you configure groupby for each component in the node level config. In the following example we will regroup the outputs by the `region`:
-
-```yaml
-tree:
-  children:
-    child-1:
-      pipeline:
-        - teads-curve
-        - sci-e
-        - sci-embodied
-        - sci-o
-        - time-sync
-        - group-by
-        - sci
-      config:
-        group-by:
-          group:
-            - region
-```
-
 ### Exhaust
 
 We have introduced `exhaust` as an IF feature. This is a wrapper around export plugins and it allows community contributors to create plugins for exporting to different formats.

diff --git a/.../regroup/failure-invalid-config-group.yml → ...tures/regroup/failure-invalid-regroup.yml b/.../regroup/failure-invalid-config-group.yml → ...tures/regroup/failure-invalid-regroup.yml
@@ -1,13 +1,12 @@
 name: regroup
-description: failure when `config->group-by->group` is not an array
+description: failure when `regroup` is not an array
 initialize:
   plugins: {}
 tree:
   children:
     my-app:
       pipeline:
-        regroup:
-          cloud/region
+        regroup: cloud/region
       inputs:
         - timestamp: 2023-07-06T00:00
           duration: 300

diff --git a/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml b/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml
@@ -2,9 +2,6 @@ name: regroup
 description:
 initialize:
   plugins:
-    group-by:
-      path: "builtin"
-      method: GroupBy
 tree:
   children:
     my-app:

diff --git a/manifests/examples/features/regroup/success.yml b/manifests/examples/features/regroup/success.yml
@@ -2,9 +2,6 @@ name: regroup
 description: successful path
 initialize:
   plugins:
-    group-by:
-      path: "builtin"
-      method: GroupBy
 tree:
   children:
     my-app:

diff --git a/manifests/examples/pipelines/nesting.yml b/manifests/examples/pipelines/nesting.yml
@@ -6,8 +6,7 @@ tags:
   category: on-premise
 aggregation:
   metrics:
-    "carbon":
-      method: sum
+    - carbon
   type: "both"
 initialize:
   plugins:
@@ -139,6 +138,21 @@ initialize:
           - carbon-operational
           - carbon-embodied
         output-parameter: carbon
+      parameter-metadata:
+        inputs:
+          carbon-operational:
+            description: Operational carbon footprint
+            unit: gCO2eq
+            aggregation-method: sum
+          carbon-embodied:
+            description: Embodied carbon footprint
+            unit: gCO2eq
+            aggregation-method: sum
+        outputs:
+          carbon:
+            description: Total carbon footprint
+            unit: gCO2eq
+            aggregation-method: sum
     time-sync:
       method: TimeSync
       path: "builtin"
@@ -169,10 +183,10 @@ initialize:
             unit: seconds
             description: time reserved for a component
             aggregation-method: avg
-          network/energy: 
-            description: 'Energy consumed by the Network of the component'
-            unit: 'kWh'
-            aggregation-method: 'sum'
+          network/energy:
+            description: "Energy consumed by the Network of the component"
+            unit: "kWh"
+            aggregation-method: "sum"
 tree:
   children:
     child-0:

diff --git a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml
@@ -3,8 +3,7 @@ description: a full pipeline with the aggregate feature enabled
 tags: null
 aggregation:
   metrics:
-    carbon:
-      method: sum
+    - carbon
   type: both
 initialize:
   plugins:
@@ -159,6 +158,21 @@ initialize:
           - carbon-operational
           - carbon-embodied
         output-parameter: carbon
+      parameter-metadata:
+        inputs:
+          carbon-operational:
+            description: Operational carbon footprint
+            unit: gCO2eq
+            aggregation-method: sum
+          carbon-embodied:
+            description: Embodied carbon footprint
+            unit: gCO2eq
+            aggregation-method: sum
+        outputs:
+          carbon:
+            description: Total carbon footprint
+            unit: gCO2eq
+            aggregation-method: sum
     time-sync:
       path: builtin
       method: TimeSync
@@ -237,22 +251,21 @@ tree:
   children:
     child-1:
       pipeline:
-        - interpolate
-        - cpu-factor-to-wattage
-        - wattage-times-duration
-        - wattage-to-energy-kwh
-        - calculate-vcpu-ratio
-        - correct-cpu-energy-for-vcpu-ratio
-        - sci-embodied
-        - operational-carbon
-        - sum-carbon
-        - time-sync
-        - sci
-      config:
-        group-by:
-          group:
-            - cloud/region
-            - cloud/instance-type
+        regroup:
+          - cloud/region
+          - cloud/instance-type
+        compute:
+          - interpolate
+          - cpu-factor-to-wattage
+          - wattage-times-duration
+          - wattage-to-energy-kwh
+          - calculate-vcpu-ratio
+          - correct-cpu-energy-for-vcpu-ratio
+          - sci-embodied
+          - operational-carbon
+          - sum-carbon
+          - time-sync
+          - sci  
       defaults:
         cpu/thermal-design-power: 100
         grid/carbon-intensity: 800
@@ -590,22 +603,21 @@ tree:
         carbon: 0.04532668505834602
     child-2:
       pipeline:
-        - interpolate
-        - cpu-factor-to-wattage
-        - wattage-times-duration
-        - wattage-to-energy-kwh
-        - calculate-vcpu-ratio
-        - correct-cpu-energy-for-vcpu-ratio
-        - sci-embodied
-        - operational-carbon
-        - sum-carbon
-        - time-sync
-        - sci
-      config:
-        group-by:
-          group:
-            - cloud/region
-            - cloud/instance-type
+        regroup:
+          - cloud/region
+          - cloud/instance-type
+        compute:
+          - interpolate
+          - cpu-factor-to-wattage
+          - wattage-times-duration
+          - wattage-to-energy-kwh
+          - calculate-vcpu-ratio
+          - correct-cpu-energy-for-vcpu-ratio
+          - sci-embodied
+          - operational-carbon
+          - sum-carbon
+          - time-sync
+          - sci
       defaults:
         cpu/thermal-design-power: 100
         grid/carbon-intensity: 800