diff --git a/README.md b/README.md index c7e9b45..57e57ad 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +> [!IMPORTANT] +> If you are participating in Carbon Hack 24, please **don't submit your plugin to the if-plugins repository**. This is the standard library of plugins for Impact Framework which are being maintained by the IF team. Please commit your plugin to your **own repository** and submit a link to your repo in your final submission for the hackaton. More info about building and publishing plugins [here](https://if.greensoftware.foundation/developers/how-to-build-plugins#step-5-publishing-your-plugin). + + # Impact Framework - Plugins `if-plugins` are set of plugins maintained as a part of Impact Framework. diff --git a/package-lock.json b/package-lock.json index f065ff4..19359d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@grnsft/if-plugins", - "version": "v0.3.0", + "version": "v0.3.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@grnsft/if-plugins", - "version": "v0.2.0", + "version": "v0.3.1", "license": "MIT", "dependencies": { "@azure/arm-compute": "^21.2.0", @@ -15,7 +15,6 @@ "axios": "^1.6.0", "copyfiles": "^2.4.1", "csv-parse": "^5.5.5", - "dayjs": "^1.11.10", "dotenv": "^16.3.1", "js-yaml": "^4.1.0", "typescript": "^5.1.6", @@ -29,12 +28,14 @@ "@jest/globals": "^29.6.1", "@types/jest": "^29.5.7", "@types/js-yaml": "^4.0.8", + "@types/luxon": "^3.4.2", "@types/node": "^20.4.5", "fixpack": "4.0.0", "gts": "^5.0.0", "husky": "^8.0.0", "jest": "^29.6.1", "jest-mock-axios": "^4.7.2", + "luxon": "^3.4.4", "rimraf": "^5.0.5", "ts-jest": "^29.1.1" }, @@ -2670,6 +2671,12 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, + "node_modules/@types/luxon": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.4.2.tgz", + "integrity": "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==", + "dev": true + }, "node_modules/@types/minimist": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", @@ -3863,11 +3870,6 @@ "node": ">=8" } }, - "node_modules/dayjs": { - "version": "1.11.10", - "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.10.tgz", - "integrity": "sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==" - }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -7859,6 +7861,15 @@ "yallist": "^3.0.2" } }, + "node_modules/luxon": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", + "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==", + "dev": true, + "engines": { + "node": ">=12" + } + }, "node_modules/make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", @@ -11966,6 +11977,12 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, + "@types/luxon": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.4.2.tgz", + "integrity": "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==", + "dev": true + }, "@types/minimist": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", @@ -12816,11 +12833,6 @@ "integrity": "sha512-2iy1EkLdlBzQGvbweYRFxmFath8+K7+AKB0TlhHWkNuH+TmovaMH/Wp7V7R4u7f4SnX3OgLsU9t1NI9ioDnUpg==", "dev": true }, - "dayjs": { - "version": "1.11.10", - "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.10.tgz", - "integrity": "sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==" - }, "debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -15773,6 +15785,12 @@ "yallist": "^3.0.2" } }, + "luxon": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", + "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==", + "dev": true + }, "make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", diff --git a/package.json b/package.json index e34b647..ad15d38 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@grnsft/if-plugins", "description": "Impact Framework Standard Plugins", - "version": "v0.3.0", + "version": "v0.3.1", "author": { "name": "Green Software Foundation", "email": "info@gsf.com" @@ -16,7 +16,6 @@ "axios": "^1.6.0", "copyfiles": "^2.4.1", "csv-parse": "^5.5.5", - "dayjs": "^1.11.10", "dotenv": "^16.3.1", "js-yaml": "^4.1.0", "typescript": "^5.1.6", @@ -30,12 +29,14 @@ "@jest/globals": "^29.6.1", "@types/jest": "^29.5.7", "@types/js-yaml": "^4.0.8", + "@types/luxon": "^3.4.2", "@types/node": "^20.4.5", "fixpack": "4.0.0", "gts": "^5.0.0", "husky": "^8.0.0", "jest": "^29.6.1", "jest-mock-axios": "^4.7.2", + "luxon": "^3.4.4", "rimraf": "^5.0.5", "ts-jest": "^29.1.1" }, diff --git a/src/__tests__/unit/lib/cloud-metadata/index.test.ts b/src/__tests__/unit/lib/cloud-metadata/index.test.ts index 7de45aa..2a044fd 100644 --- a/src/__tests__/unit/lib/cloud-metadata/index.test.ts +++ b/src/__tests__/unit/lib/cloud-metadata/index.test.ts @@ -73,6 +73,134 @@ describe('lib/cloud-metadata:', () => { ]); }); + it('returns a result when azure instance type do not have size number.', async () => { + const inputs = [ + { + timestamp: '', + duration: 5, + 'cloud/instance-type': 'Standard_B1ms', + 'cloud/vendor': 'azure', + }, + ]; + + const result = await cloudMetadata.execute(inputs); + + expect.assertions(1); + + expect(result).toStrictEqual([ + { + timestamp: '', + duration: 5, + 'cloud/instance-type': 'Standard_B1ms', + 'cloud/vendor': 'azure', + 'cpu/thermal-design-power': 270, + 'physical-processor': + 'Intel® Xeon® Platinum 8370C,Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz', + 'vcpus-allocated': 1, + 'vcpus-total': 64, + 'memory-available': 2, + }, + ]); + }); + + it('returns a result with configured outputs.', async () => { + const inputs = [ + { + timestamp: '', + duration: 5, + 'cloud/instance-type': 'Standard_A1_v2', + 'cloud/vendor': 'azure', + 'cloud/region': 'francesouth', + }, + ]; + const config = { + fields: [ + 'cloud/vendor', + 'cloud/region-wt-id', + 'cloud/instance-type', + 'physical-processor', + 'cpu/thermal-design-power', + ], + }; + const result = await cloudMetadata.execute(inputs, config); + + expect.assertions(1); + + expect(result).toStrictEqual([ + { + timestamp: '', + duration: 5, + 'cloud/instance-type': 'Standard_A1_v2', + 'cloud/region': 'francesouth', + 'cloud/region-wt-id': 'FR', + 'cloud/vendor': 'azure', + 'cpu/thermal-design-power': 205, + 'physical-processor': + 'Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz', + }, + ]); + }); + + it('returns a result when provided a `cloud/region` in the input.', async () => { + const inputs = [ + { + timestamp: '', + duration: 5, + 'cloud/instance-type': 'Standard_A1_v2', + 'cloud/vendor': 'azure', + 'cloud/region': 'francesouth', + }, + ]; + + const result = await cloudMetadata.execute(inputs); + + expect.assertions(1); + + expect(result).toStrictEqual([ + { + timestamp: '', + duration: 5, + 'cloud/instance-type': 'Standard_A1_v2', + 'cloud/region': 'francesouth', + 'cloud/region-cfe': 'France', + 'cloud/region-em-zone-id': 'FR', + 'cloud/region-geolocation': '48.8567,2.3522', + 'cloud/region-location': 'Paris', + 'cloud/region-wt-id': 'FR', + 'cloud/vendor': 'azure', + 'cpu/thermal-design-power': 205, + 'memory-available': 2, + 'physical-processor': + 'Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz', + 'vcpus-allocated': 1, + 'vcpus-total': 52, + }, + ]); + }); + + it('throws an error when provided a wrong `cloud/region` for vendor in the input.', async () => { + const errorMessage = + "CloudMetadata: 'uk-west' region is not supported in 'azure' cloud vendor."; + const inputs = [ + { + timestamp: '', + duration: 5, + 'cloud/instance-type': 'Standard_A1_v2', + 'cloud/vendor': 'azure', + 'cloud/region': 'uk-west', + }, + ]; + + expect.assertions(2); + + try { + await cloudMetadata.execute(inputs); + } catch (error) { + expect(error).toStrictEqual(new UnsupportedValueError(errorMessage)); + expect(error).toBeInstanceOf(UnsupportedValueError); + } + }); + it('throws on `cloud/instance-type` when `cloud/vendor` is aws.', async () => { const errorMessage = "CloudMetadata(cloud/instance-type): 't2.micro2' instance type is not supported in 'aws' cloud vendor."; diff --git a/src/__tests__/unit/lib/csv-export/index.test.ts b/src/__tests__/unit/lib/csv-export/index.test.ts index 5b452f0..d744738 100644 --- a/src/__tests__/unit/lib/csv-export/index.test.ts +++ b/src/__tests__/unit/lib/csv-export/index.test.ts @@ -5,7 +5,7 @@ import {CsvExport} from '../../../../lib/csv-export'; import {ERRORS} from '../../../../util/errors'; -const {MakeDirectoryError, WriteFileError} = ERRORS; +const {MakeDirectoryError, WriteFileError, InputValidationError} = ERRORS; jest.mock('fs/promises', () => ({ mkdir: jest.fn<() => Promise>().mockResolvedValue(), @@ -126,6 +126,36 @@ describe('lib/csv-export: ', () => { expect(result).toStrictEqual(input); }); + it('throws an error when node config is not provided.', async () => { + const csvExport = CsvExport(); + + const input = [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 10, + energy: 10, + carbon: 2, + }, + { + timestamp: '2023-12-12T00:00:10.000Z', + duration: 30, + energy: 20, + carbon: 5, + }, + ]; + + expect.assertions(2); + + try { + await csvExport.execute(input); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toEqual( + new InputValidationError('CsvExport: Configuration data is missing.') + ); + } + }); + it('throws an error when a file writing fails.', async () => { (fs.writeFile as jest.Mock).mockImplementation(() => { throw new Error('Permission denied'); diff --git a/src/__tests__/unit/lib/divide/index.test.ts b/src/__tests__/unit/lib/divide/index.test.ts index e2842bc..71aff39 100644 --- a/src/__tests__/unit/lib/divide/index.test.ts +++ b/src/__tests__/unit/lib/divide/index.test.ts @@ -2,7 +2,7 @@ import {Divide} from '../../../../lib'; import {ERRORS} from '../../../../util/errors'; -const {InputValidationError} = ERRORS; +const {InputValidationError, ConfigValidationError} = ERRORS; describe('lib/divide: ', () => { describe('Divide: ', () => { @@ -102,6 +102,25 @@ describe('lib/divide: ', () => { }); }); + it('throws an error on missing global config.', async () => { + const expectedMessage = 'Divide: Configuration data is missing.'; + const config = undefined; + const divide = Divide(config!); + + expect.assertions(1); + + try { + await divide.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + }, + ]); + } catch (error) { + expect(error).toStrictEqual(new ConfigValidationError(expectedMessage)); + } + }); + it('throws an error when `denominator` is 0.', async () => { const expectedMessage = '"denominator" parameter is number must be greater than 0. Error code: too_small.'; @@ -127,5 +146,30 @@ describe('lib/divide: ', () => { expect(error).toStrictEqual(new InputValidationError(expectedMessage)); } }); + + it('throws an error when `denominator` is string.', async () => { + const expectedMessage = 'Divide: `10` is missing from the input.'; + + const globalConfig = { + numerator: 'vcpus-allocated', + denominator: '10', + output: 'vcpus-allocated-per-second', + }; + const divide = Divide(globalConfig); + + expect.assertions(1); + + try { + await divide.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'vcpus-allocated': 24, + }, + ]); + } catch (error) { + expect(error).toStrictEqual(new InputValidationError(expectedMessage)); + } + }); }); }); diff --git a/src/__tests__/unit/lib/e-mem/index.test.ts b/src/__tests__/unit/lib/e-mem/index.test.ts index b817c69..a2b17d7 100644 --- a/src/__tests__/unit/lib/e-mem/index.test.ts +++ b/src/__tests__/unit/lib/e-mem/index.test.ts @@ -76,7 +76,7 @@ describe('lib/e-mem: ', () => { } }); - it('does not throw an error for a missing `energy-per-gb` but instead uses the default value of 0.38.', async () => { + it('does not throw an error for a missing `energy-per-gb` but instead uses the default value of 0.000392.', async () => { const data = [ { timestamp: '2023-11-02T10:35:31.820Z', @@ -97,5 +97,39 @@ describe('lib/e-mem: ', () => { expect(response[0]['memory/energy']).toEqual(expectedMemory); }); }); + + describe('execute() with no global config: ', () => { + it('gets energy-memory from fallback.', async () => { + const globalConfig = {}; + const eMem = EMem(globalConfig); + + const inputs = [ + { + 'memory/utilization': 80, + 'memory/capacity': 16, + duration: 3600, + timestamp: '2022-01-01T01:00:00Z', + }, + { + 'memory/utilization': 60, + 'memory/capacity': 8, + duration: 3600, + timestamp: '2022-01-01T01:00:00Z', + }, + ]; + expect.assertions(3); + + const result = await eMem.execute(inputs); + + expect(result).toHaveLength(inputs.length); + result.forEach((output, index) => { + expect(output['memory/energy']).toBeCloseTo( + inputs[index]['memory/capacity'] * + (inputs[index]['memory/utilization'] / 100) * + 0.000392 + ); + }); + }); + }); }); }); diff --git a/src/__tests__/unit/lib/e-net/index.test.ts b/src/__tests__/unit/lib/e-net/index.test.ts index ffcf04e..f764dfa 100644 --- a/src/__tests__/unit/lib/e-net/index.test.ts +++ b/src/__tests__/unit/lib/e-net/index.test.ts @@ -67,3 +67,36 @@ describe('lib/e-net', () => { }); }); }); + +describe('execute() without global config: ', () => { + it('calculates energy for each input.', async () => { + const globalConfig = {}; + const eNet = ENet(globalConfig); + const inputs = [ + { + 'network/data-in': 10, + 'network/data-out': 5, + duration: 3600, + timestamp: '2022-01-01T01:00:00Z', + }, + { + 'network/data-in': 20, + 'network/data-out': 15, + duration: 3600, + timestamp: '2022-01-01T01:00:00Z', + }, + ]; + + expect.assertions(3); + + const result = await eNet.execute(inputs); + + expect(result).toHaveLength(inputs.length); + result.forEach((output, index) => { + expect(output['network/energy']).toBeCloseTo( + (inputs[index]['network/data-in'] + inputs[index]['network/data-out']) * + 0.001 + ); + }); + }); +}); diff --git a/src/__tests__/unit/lib/mock-observations/CommonGenerator.test.ts b/src/__tests__/unit/lib/mock-observations/CommonGenerator.test.ts index 0622b1c..539f5bb 100644 --- a/src/__tests__/unit/lib/mock-observations/CommonGenerator.test.ts +++ b/src/__tests__/unit/lib/mock-observations/CommonGenerator.test.ts @@ -8,9 +8,13 @@ const {InputValidationError} = ERRORS; describe('lib/mock-observations/CommonGenerator: ', () => { describe('initialize: ', () => { - it('initialize with an empty config.', async () => { + it('throws an error when config is not empty object.', async () => { + const commonGenerator = CommonGenerator({}); + + expect.assertions(1); + try { - CommonGenerator({}); + commonGenerator.next([]); } catch (error) { expect(error).toEqual( new InputValidationError( @@ -29,6 +33,8 @@ describe('lib/mock-observations/CommonGenerator: ', () => { }; const commonGenerator = CommonGenerator(config); + expect.assertions(1); + expect(commonGenerator.next([])).toStrictEqual({ key1: 'value1', key2: 'value2', diff --git a/src/__tests__/unit/lib/mock-observations/RandIntGenerator.test.ts b/src/__tests__/unit/lib/mock-observations/RandIntGenerator.test.ts index 1840b67..b69ae59 100644 --- a/src/__tests__/unit/lib/mock-observations/RandIntGenerator.test.ts +++ b/src/__tests__/unit/lib/mock-observations/RandIntGenerator.test.ts @@ -8,7 +8,8 @@ const {InputValidationError} = ERRORS; describe('lib/mock-observations/RandIntGenerator: ', () => { describe('initialize', () => { - it('initialize with an empty name', async () => { + it('throws an error when the generator name is empty string.', async () => { + expect.assertions(1); try { RandIntGenerator('', {}); } catch (error) { @@ -20,7 +21,8 @@ describe('lib/mock-observations/RandIntGenerator: ', () => { } }); - it('initialize with an empty config', async () => { + it('throws an error when config is empty object.', async () => { + expect.assertions(1); try { RandIntGenerator('generator-name', {}); } catch (error) { @@ -31,6 +33,22 @@ describe('lib/mock-observations/RandIntGenerator: ', () => { ); } }); + + it('throws an error `min` is missing from the config.', async () => { + const config = {max: 90}; + + expect.assertions(1); + + try { + RandIntGenerator('random', config); + } catch (error) { + expect(error).toEqual( + new InputValidationError( + 'RandIntGenerator: Config is missing min or max.' + ) + ); + } + }); }); describe('next(): ', () => { @@ -42,6 +60,8 @@ describe('lib/mock-observations/RandIntGenerator: ', () => { const randIntGenerator = RandIntGenerator('random', config); const result = randIntGenerator.next([]) as {random: number}; + expect.assertions(4); + expect(result).toBeInstanceOf(Object); expect(result).toHaveProperty('random'); expect(result.random).toBeGreaterThanOrEqual(10); diff --git a/src/__tests__/unit/lib/mock-observations/index.test.ts b/src/__tests__/unit/lib/mock-observations/index.test.ts index 953e23e..663dbbe 100644 --- a/src/__tests__/unit/lib/mock-observations/index.test.ts +++ b/src/__tests__/unit/lib/mock-observations/index.test.ts @@ -41,6 +41,9 @@ describe('lib/mock-observations: ', () => { region: 'uk-west', 'common-key': 'common-val', }, + randint: { + 'cpu/utilization': {min: 10, max: 11}, + }, }, }; const mockObservations = MockObservations(config); @@ -50,36 +53,70 @@ describe('lib/mock-observations: ', () => { expect(result).toStrictEqual([ { - 'common-key': 'common-val', + timestamp: '2023-07-06T00:00:00.000Z', duration: 30, + 'common-key': 'common-val', 'instance-type': 'A1', region: 'uk-west', - timestamp: '2023-07-06T00:00:00.000Z', + 'cpu/utilization': 10, }, { - 'common-key': 'common-val', + timestamp: '2023-07-06T00:00:30.000Z', duration: 30, + 'common-key': 'common-val', 'instance-type': 'A1', region: 'uk-west', - timestamp: '2023-07-06T00:00:30.000Z', + 'cpu/utilization': 10, }, { - 'common-key': 'common-val', + timestamp: '2023-07-06T00:00:00.000Z', duration: 30, + 'common-key': 'common-val', 'instance-type': 'B1', region: 'uk-west', - timestamp: '2023-07-06T00:00:00.000Z', + 'cpu/utilization': 10, }, { - 'common-key': 'common-val', + timestamp: '2023-07-06T00:00:30.000Z', duration: 30, + 'common-key': 'common-val', 'instance-type': 'B1', region: 'uk-west', - timestamp: '2023-07-06T00:00:30.000Z', + 'cpu/utilization': 10, }, ]); }); + it('throws an error when the `min` is greater then `max` of `randint` config.', async () => { + const errorMessage = + 'RandIntGenerator: Min value should not be greater than or equal to max value of cpu/utilization.'; + const config = { + 'timestamp-from': '2023-07-06T00:00', + 'timestamp-to': '2023-07-06T00:01', + duration: 30, + components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], + generators: { + common: { + region: 'uk-west', + 'common-key': 'common-val', + }, + randint: { + 'cpu/utilization': {min: 20, max: 11}, + }, + }, + }; + + expect.assertions(2); + + const mockObservations = MockObservations(config); + try { + await mockObservations.execute([]); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toEqual(new InputValidationError(errorMessage)); + } + }); + it('throws when `generators` are not provided.', async () => { const config = { 'timestamp-from': '2023-07-06T00:00', @@ -97,13 +134,15 @@ describe('lib/mock-observations: ', () => { expect(error).toBeInstanceOf(InputValidationError); expect(error).toEqual( new InputValidationError( - 'MockObservations: generators missing from global config.' + '"generators" parameter is required. Error code: invalid_type.' ) ); } }); it('throws when `components` are not provided.', async () => { + const errorMessage = + '"components" parameter is required. Error code: invalid_type.'; const config = { 'timestamp-from': '2023-07-06T00:00', 'timestamp-to': '2023-07-06T00:01', @@ -127,11 +166,7 @@ describe('lib/mock-observations: ', () => { await mockObservations.execute([]); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); - expect(error).toEqual( - new InputValidationError( - 'MockObservations: components missing from global config.' - ) - ); + expect(error).toEqual(new InputValidationError(errorMessage)); } }); @@ -159,7 +194,7 @@ describe('lib/mock-observations: ', () => { expect(error).toBeInstanceOf(InputValidationError); expect(error).toEqual( new InputValidationError( - 'MockObservations: duration missing from global config.' + '"duration" parameter is required. Error code: invalid_type.' ) ); } @@ -189,7 +224,7 @@ describe('lib/mock-observations: ', () => { expect(error).toBeInstanceOf(InputValidationError); expect(error).toEqual( new InputValidationError( - 'MockObservations: timestamp-to missing from global config.' + '"timestamp-to" parameter is required. Error code: invalid_type.' ) ); } @@ -219,7 +254,67 @@ describe('lib/mock-observations: ', () => { expect(error).toBeInstanceOf(InputValidationError); expect(error).toEqual( new InputValidationError( - 'MockObservations: timestamp-from missing from global config.' + '"timestamp-from" parameter is required. Error code: invalid_type.' + ) + ); + } + }); + + it('throws an error when `randInt` is not valid.', async () => { + const config = { + 'timestamp-from': '2023-07-06T00:00', + 'timestamp-to': '2023-07-06T00:01', + duration: 30, + components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], + generators: { + common: { + region: 'uk-west', + 'common-key': 'common-val', + }, + randint: null, + }, + }; + const mockObservations = MockObservations(config); + + expect.assertions(2); + + try { + await mockObservations.execute([]); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toEqual( + new InputValidationError( + '"generators.randint" parameter is expected object, received null. Error code: invalid_type.' + ) + ); + } + }); + + it('throws an error when `common` is not valid.', async () => { + const config = { + 'timestamp-from': '2023-07-06T00:00', + 'timestamp-to': '2023-07-06T00:01', + duration: 30, + components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], + generators: { + common: null, + randint: { + 'cpu/utilization': {min: 10, max: 95}, + 'memory/utilization': {min: 10, max: 85}, + }, + }, + }; + const mockObservations = MockObservations(config); + + expect.assertions(2); + + try { + await mockObservations.execute([]); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toEqual( + new InputValidationError( + '"generators.common" parameter is expected object, received null. Error code: invalid_type.' ) ); } diff --git a/src/__tests__/unit/lib/regex/index.test.ts b/src/__tests__/unit/lib/regex/index.test.ts index ade7e9f..15864d3 100644 --- a/src/__tests__/unit/lib/regex/index.test.ts +++ b/src/__tests__/unit/lib/regex/index.test.ts @@ -2,7 +2,7 @@ import {Regex} from '../../../../lib'; import {ERRORS} from '../../../../util/errors'; -const {InputValidationError} = ERRORS; +const {InputValidationError, ConfigValidationError} = ERRORS; describe('lib/regex: ', () => { describe('Regex: ', () => { @@ -46,6 +46,38 @@ describe('lib/regex: ', () => { expect(result).toStrictEqual(expectedResult); }); + it('returns a result when regex is not started and ended with ``.', async () => { + const physicalProcessor = + 'Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz'; + expect.assertions(1); + + const globalConfig = { + parameter: 'physical-processor', + match: '[^,]+/', + output: 'cpu/name', + }; + const regex = Regex(globalConfig); + + const expectedResult = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'physical-processor': physicalProcessor, + 'cpu/name': 'Intel® Xeon® Platinum 8272CL', + }, + ]; + + const result = await regex.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'physical-processor': physicalProcessor, + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + it('throws an error when `parameter` does not match to `match`.', async () => { const physicalProcessor = 'Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz'; @@ -75,6 +107,28 @@ describe('lib/regex: ', () => { } }); + it('throws an error on missing global config.', async () => { + const expectedMessage = 'Regex: Configuration data is missing.'; + + const config = undefined; + const regex = Regex(config!); + + expect.assertions(1); + + try { + await regex.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new ConfigValidationError(expectedMessage) + ); + } + }); + it('throws an error on missing params in input.', async () => { const expectedMessage = 'Regex: `physical-processor` is missing from the input.'; diff --git a/src/__tests__/unit/lib/sci-m/index.test.ts b/src/__tests__/unit/lib/sci-m/index.test.ts index 7e10582..cad559c 100644 --- a/src/__tests__/unit/lib/sci-m/index.test.ts +++ b/src/__tests__/unit/lib/sci-m/index.test.ts @@ -191,6 +191,37 @@ describe('lib/sci-m:', () => { ]); }); + it('throws an error when `device/emissions-embodied` is string.', async () => { + const errorMessage = + '"device/emissions-embodied" parameter is not a valid number in input. please provide it as `gco2e`. Error code: invalid_union.'; + const inputs = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 60 * 60 * 24 * 30, + 'device/emissions-embodied': '10,00', + 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, + 'resources-reserved': 1, + 'resources-total': 1, + }, + { + timestamp: '2021-01-01T00:00:00Z', + duration: 60 * 60 * 24 * 30 * 2, + 'device/emissions-embodied': 200, + 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, + 'resources-reserved': 1, + 'resources-total': 1, + }, + ]; + + expect.assertions(2); + try { + await sciM.execute(inputs); + } catch (error) { + expect(error).toStrictEqual(new InputValidationError(errorMessage)); + expect(error).toBeInstanceOf(InputValidationError); + } + }); + it('throws an exception on missing `device/emissions-embodied`.', async () => { const errorMessage = '"device/emissions-embodied" parameter is required. Error code: invalid_union.'; @@ -239,7 +270,7 @@ describe('lib/sci-m:', () => { it('throws an exception on invalid values.', async () => { const errorMessage = - '"device/emissions-embodied" parameter is expected number, received string. Error code: invalid_union.'; + '"device/emissions-embodied" parameter is not a valid number in input. please provide it as `gco2e`. Error code: invalid_union.'; const inputs = [ { timestamp: '2021-01-01T00:00:00Z', diff --git a/src/__tests__/unit/lib/sci/index.test.ts b/src/__tests__/unit/lib/sci/index.test.ts index 188f3ea..c3212f7 100644 --- a/src/__tests__/unit/lib/sci/index.test.ts +++ b/src/__tests__/unit/lib/sci/index.test.ts @@ -90,6 +90,90 @@ describe('lib/sci:', () => { ]); }); + it('returns a result when `functional-unit-time` is not provided.', async () => { + const sci = Sci({ + 'functional-unit': 'requests', + }); + const inputs = [ + { + timestamp: '2021-01-01T00:00:00Z', + 'carbon-operational': 0.2, + 'carbon-embodied': 0.05, + duration: 100, + }, + { + timestamp: '2021-01-01T00:00:00Z', + 'carbon-operational': 0.002, + 'carbon-embodied': 0.0005, + duration: 2, + }, + ]; + const result = await sci.execute(inputs); + + expect.assertions(1); + + expect(result).toStrictEqual([ + { + timestamp: '2021-01-01T00:00:00Z', + 'carbon-operational': 0.2, + 'carbon-embodied': 0.05, + carbon: 0.0025, + duration: 100, + sci: 0.0025, + }, + { + timestamp: '2021-01-01T00:00:00Z', + 'carbon-operational': 0.002, + 'carbon-embodied': 0.0005, + carbon: 0.00125, + duration: 2, + sci: 0.00125, + }, + ]); + }); + + it('returns a result when `functional-unit` is not provided.', async () => { + const sci = Sci({ + 'functional-unit-time': '1 day', + }); + const inputs = [ + { + timestamp: '2021-01-01T00:00:00Z', + 'carbon-operational': 0.2, + 'carbon-embodied': 0.05, + duration: 100, + }, + { + timestamp: '2021-01-01T00:00:00Z', + 'carbon-operational': 0.002, + 'carbon-embodied': 0.0005, + duration: 2, + }, + ]; + const result = await sci.execute(inputs); + + expect.assertions(1); + + expect(result).toStrictEqual([ + { + timestamp: '2021-01-01T00:00:00Z', + 'carbon-operational': 0.2, + 'carbon-embodied': 0.05, + carbon: 0.0025, + duration: 100, + sci: 216, + }, + { + timestamp: '2021-01-01T00:00:00Z', + 'carbon-operational': 0.002, + 'carbon-embodied': 0.0005, + carbon: 0.00125, + duration: 2, + sci: 108, + }, + ]); + }); + it('throws exception on invalid functional unit data.', async () => { const sci = Sci({ 'functional-unit': 'requests', diff --git a/src/__tests__/unit/lib/shell/index.test.ts b/src/__tests__/unit/lib/shell/index.test.ts index e0d018b..1a71609 100644 --- a/src/__tests__/unit/lib/shell/index.test.ts +++ b/src/__tests__/unit/lib/shell/index.test.ts @@ -3,6 +3,10 @@ import {loadAll} from 'js-yaml'; import {Shell} from '../../../../lib'; +import {ERRORS} from '../../../../util/errors'; + +const {InputValidationError} = ERRORS; + jest.mock('child_process'); jest.mock('js-yaml'); @@ -57,6 +61,30 @@ describe('lib/shell', () => { await expect(shell.execute(invalidInputs)).rejects.toThrow(); }); + + it('throw an error when shell could not run command.', async () => { + const shell = Shell({command: 'python3 /path/to/script.py'}); + (spawnSync as jest.Mock).mockImplementation(() => { + throw new InputValidationError('Could not run the command'); + }); + + const inputs = [ + { + duration: 3600, + timestamp: '2022-01-01T00:00:00Z', + }, + ]; + expect.assertions(2); + + try { + await shell.execute(inputs); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toStrictEqual( + new InputValidationError('Could not run the command') + ); + } + }); }); }); }); diff --git a/src/__tests__/unit/lib/tdp-finder/index.test.ts b/src/__tests__/unit/lib/tdp-finder/index.test.ts index 0c4412c..a16201e 100644 --- a/src/__tests__/unit/lib/tdp-finder/index.test.ts +++ b/src/__tests__/unit/lib/tdp-finder/index.test.ts @@ -1,8 +1,9 @@ +import * as fs from 'fs'; import {TdpFinder} from '../../../../lib'; import {ERRORS} from '../../../../util/errors'; -const {InputValidationError, UnsupportedValueError} = ERRORS; +const {InputValidationError, UnsupportedValueError, ReadFileError} = ERRORS; describe('lib/tdp-finder:', () => { describe('TdpFinder', () => { @@ -101,6 +102,28 @@ describe('lib/tdp-finder:', () => { expect(error).toBeInstanceOf(UnsupportedValueError); } }); + + it('throws an error when the file cannot be read/', async () => { + jest.spyOn(fs.promises, 'readFile').mockRejectedValueOnce('data.csv'); + const inputs = [ + { + timestamp: '2023-11-02T10:35:31.820Z', + duration: 3600, + 'physical-processor': 'Intel Xeon Platinum 8175M,AMD A8-9600f', + }, + ]; + + expect.assertions(2); + + try { + await tdpFinder.execute(inputs); + } catch (error) { + expect(error).toStrictEqual( + new ReadFileError('Error reading file data.csv: data.csv') + ); + expect(error).toBeInstanceOf(ReadFileError); + } + }); }); }); }); diff --git a/src/__tests__/unit/util/validations.test.ts b/src/__tests__/unit/util/validations.test.ts new file mode 100644 index 0000000..1954c23 --- /dev/null +++ b/src/__tests__/unit/util/validations.test.ts @@ -0,0 +1,76 @@ +import {z} from 'zod'; + +import {validate} from '../../../util/validations'; +import {ERRORS} from '../../../util/errors'; + +const {InputValidationError} = ERRORS; + +describe('Validations: ', () => { + it('returns validated data if input is valid according to schema.', () => { + const schema = z.object({ + timestamp: z.string(), + duration: z.number(), + energy: z.number(), + }); + + const input = { + timestamp: '2023-12-12T00:00:00.000Z', + energy: 10, + duration: 60, + }; + + expect(validate(schema, input)).toEqual(input); + }); + + it('throws an error and prettify error message.', () => { + const schema = z.object({ + energy: z.number(), + }); + + const invalidInput = { + energy: 'invalidEnergy', + }; + + expect.assertions(2); + + try { + validate(schema, invalidInput); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toStrictEqual( + new InputValidationError( + '"energy" parameter is expected number, received string. Error code: invalid_type.' + ) + ); + } + }); + + it('throws an error when not valid union is provided.', () => { + const schema1 = z.object({ + energy: z.number(), + 7: z.string(), + }); + + const schema2 = z.object({ + timestamp: z.string(), + }); + const invalidInput = { + carbon: 'invalidEnergy', + 4: '2', + }; + + const schema = schema1.or(schema2); + + expect.assertions(2); + try { + validate(schema, invalidInput); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toStrictEqual( + new InputValidationError( + '"7" parameter is required. Error code: invalid_union.' + ) + ); + } + }); +}); diff --git a/src/lib/cloud-metadata/README.md b/src/lib/cloud-metadata/README.md index d6db2e5..eb5f885 100644 --- a/src/lib/cloud-metadata/README.md +++ b/src/lib/cloud-metadata/README.md @@ -10,6 +10,8 @@ Not Needed ### Inputs +- `timestamp`: ISO 8061 string, +- `duration`: number of seconds the observation spans, - `cloud/vendor`: the cloud platform provider, e.g. `aws` - `cloud/instance-type`: the name of the specific instance being used, e.g. `m5n.large` @@ -17,17 +19,23 @@ Not Needed An array containing: -- `cloud/instance-type`: echo input `instance-type` -- `cloud/vendor`: echo input `vendor` -- `physical-processor`: physical processor used in the given instance + +- `timestamp`: "2023-07-06T00:01" +- `duration`: 10 +- `cloud/vendor`: the name of the cloud vendor as a string, options are "azure", "gcp" or "aws" +- `cloud/instance-type`: name of the instance type as a string, e.g. "m5n.large" - `vcpus-allocated`: number of vCPUs allocated to this instance - `vcpus-total`: total number of vCPUs available to this instance +- `memory-available`: total memory available on this instance, in GB, +- `physical-processor`: name of the physical processor used by this instance as a string, e.g. "Intel® Xeon® Platinum 8259CL" (note some instances return multiple possible processors separated by commas) +- `cpu/thermal-design-power`: the thermal design power of the given processor (selects the first in the list of multiple are returned) + ## IF Implementation IF implements this plugin using data from Cloud Carbon Footprint. This allows determination of cpu for type of instance in a cloud and can be invoked as part of a plugin pipeline defined in a `manifest`. -Cloud Metadata currently implements only for 'AWS' and 'Azure'. +Note that "gcp" data are not yet available in our implementation. ## Usage @@ -95,6 +103,8 @@ name: cloud-metadata description: example manifest invoking Cloud Metadata plugin tags: initialize: + outputs: + - yaml plugins: cloud-metadata: method: CloudMetadata @@ -128,7 +138,7 @@ You can run this example `manifest` by saving it as `./examples/manifests/test/c ```sh npm i -g @grnsft/if npm i -g @grnsft/if-plugins -ie --manifest ./examples/manifests/test/cim.yml --output ./examples/outputs/cim.yml +ie --manifest ./examples/manifests/test/cloud-metadata.yml --output ./examples/outputs/cloud-metadata ``` The results will be saved to a new `yaml` file in `./examples/outputs`. diff --git a/src/lib/cloud-metadata/gsf-data.csv b/src/lib/cloud-metadata/gsf-data.csv index 58e02d7..c6adf0b 100644 --- a/src/lib/cloud-metadata/gsf-data.csv +++ b/src/lib/cloud-metadata/gsf-data.csv @@ -37,7 +37,7 @@ year,cloud-provider,cloud-region,cfe-region,em-zone-id,wt-region-id,location,geo 2022,Google Cloud,us-west4,NVE,US-NW-NEVP,NEVP,Las Vegas,"35.6011,-105.2206 ",0.27,,,0,396,,,,396 2022,Amazon Web Services,us-east-2,PJM,US-MIDA-PJM,PJM_SOUTHWEST_OH,US East (Ohio),"39.8689,-84.3292 ",,1,,0,,,,, 2022,Amazon Web Services,us-east-1,PJM,US-MIDA-PJM,PJM_DC,US East (N. Virginia),"39.1057,-77.5544 ",,1,,0,,,,, -2022,Amazon Web Services,us-west-1,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"34.0497,-118.1326 ",,1,,0,,,,, +2022,Amazon Web Services,us-west-1,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"36.7783,-119.417931",,1,,0,,,,, 2022,Amazon Web Services,us-west-2,BPA,US-NW-BPAT,BPA,US West (Oregon),"45.5371,-122.65 ",,1,,0,,,,, 2022,Amazon Web Services,af-south-1,South Africa,ZA,ZA,Africa (Cape Town),"-33.9253,18.4239 ",,,,,,,,, 2022,Amazon Web Services,ap-east-1,Hong Kong,HK,HK,Asia Pacific (Hong Kong),"22.3,114.2",,,,,,,,, @@ -70,8 +70,8 @@ year,cloud-provider,cloud-region,cfe-region,em-zone-id,wt-region-id,location,geo 2024,Microsoft Azure,eastus,PJM,US-MIDA-PJM,PJM_SOUTHWEST_OH,US East (Ohio),"39.8689,-84.3292 ",,,,,,,,, 2024,Microsoft Azure,eastus2,PJM,US-MIDA-PJM,PJM_DC,US East (N. Virginia),"39.1057,-77.5544 ",,,,,,,,, 2024,Microsoft Azure,southcentralus,ERCOT,US-TEX-ERCO,ERCOT_NORTHCENTRAL,Dallas,"44.9221,-123.313 ",,,,,,,,, -2024,Microsoft Azure,westus2,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"34.0497,-118.1326 ",,,,,,,,, -2024,Microsoft Azure,westus3,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"34.0497,-118.1326 ",,,,,,,,, +2024,Microsoft Azure,westus2,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"36.7783,-119.417931",,,,,,,,, +2024,Microsoft Azure,westus3,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"36.7783,-119.417931",,,,,,,,, 2024,Microsoft Azure,australiaeast,Victoria,AUS-VIC,NEM_VIC,Asia Pacific (Melbourne),"-37.8142,144.963",,,,,,,,, 2024,Microsoft Azure,southeastasia,Singapore,SG,SGP,Singapore,"1.3,103.8",,0.13,,,,,,, 2024,Microsoft Azure,northeurope,Sweden,SE,SE,Europe (Stockholm),"59.3294,18.0686 ",,,,,,,,, @@ -122,18 +122,18 @@ year,cloud-provider,cloud-region,cfe-region,em-zone-id,wt-region-id,location,geo 2024,Microsoft Azure,switzerland,Switzerland,CH,CH,Europe (Zurich),"47.3744,8.5411",,,,,,,,, 2024,Microsoft Azure,uae,UAE,AE,AE,Middle East (UAE),"25.2631,55.2972 ",,,,,,,,, 2024,Microsoft Azure,uk,Great Britain,GB,UK,London,"51.726,-0.3",0.85,,,,,,,, -2024,Microsoft Azure,unitedstates,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"34.0497,-118.1326 ",,,,,,,,, -2024,Microsoft Azure,unitedstateseuap,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"34.0497,-118.1326 ",,,,,,,,, +2024,Microsoft Azure,unitedstates,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"36.7783,-119.417931",,,,,,,,, +2024,Microsoft Azure,unitedstateseuap,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"36.7783,-119.417931",,,,,,,,, 2024,Microsoft Azure,eastasiastage,Singapore,SG,SGP,Singapore,"1.3,103.8",,0.13,,,,,,, 2024,Microsoft Azure,southeastasiastage,Singapore,SG,SGP,Singapore,"1.3,103.8",,0.13,,,,,,, 2024,Microsoft Azure,brazilus,Brazil,BR-CS,BRA,Brazil,"-3.45,-68.95",,,,,,,,, 2024,Microsoft Azure,eastusstg,PJM,US-MIDA-PJM,PJM_SOUTHWEST_OH,US East (Ohio),,,,,,,,,, 2024,Microsoft Azure,northcentralus,ERCOT,US-TEX-ERCO,ERCOT_NORTHCENTRAL,Dallas,"44.9221,-123.313 ",,,,,,,,, -2024,Microsoft Azure,westus,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"34.0497,-118.1326 ",,,,,,,,, +2024,Microsoft Azure,westus,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"36.7783,-119.417931",,,,,,,,, 2024,Microsoft Azure,japanwest,Tokyo,JP-TK,JP_TK,Tokyo,"35.6897,139.692",0.16,,,,,,,, 2024,Microsoft Azure,jioindiawest,India,,IND,Asia Pacific (Hyderabad),"17.385,78.4867 ",,,,,,,,, 2024,Microsoft Azure,eastus2euap,PJM,US-MIDA-PJM,PJM_SOUTHWEST_OH,US East (Ohio),"39.8689,-84.3292 ",,,,,,,,, -2024,Microsoft Azure,westcentralus,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"34.0497,-118.1326 ",,,,,,,,, +2024,Microsoft Azure,westcentralus,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"36.7783,-119.417931",,,,,,,,, 2024,Microsoft Azure,southafricawest,South Africa,ZA,ZA,Africa (Cape Town),"-33.9253,18.4239 ",,,,,,,,, 2024,Microsoft Azure,australiacentral,Victoria,AUS-VIC,NEM_VIC,Asia Pacific (Melbourne),"-37.8142,144.963",,,,,,,,, 2024,Microsoft Azure,australiacentral2,Victoria,AUS-VIC,NEM_VIC,Asia Pacific (Melbourne),"-37.8142,144.963",,,,,,,,, @@ -186,7 +186,7 @@ year,cloud-provider,cloud-region,cfe-region,em-zone-id,wt-region-id,location,geo 2021,Google Cloud,us-west4,NVE,US-NW-NEVP,NEVP,Las Vegas,"35.6011,-105.2206 ",0.21,,,0,365,,,,365 2021,Amazon Web Services,us-east-2,PJM,US-MIDA-PJM,PJM_SOUTHWEST_OH,US East (Ohio),"39.8689,-84.3292 ",,0.95,,,,https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html,,, 2021,Amazon Web Services,us-east-1,PJM,US-MIDA-PJM,PJM_DC,US East (N. Virginia),"39.1057,-77.5544 ",,0.95,,,,,,, -2021,Amazon Web Services,us-west-1,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"34.0497,-118.1326 ",,0.95,,,,,,, +2021,Amazon Web Services,us-west-1,CAISO,US-CAL-CISO,CAISO_NORTH,US West (N. California),"36.7783,-119.417931",,0.95,,,,,,, 2021,Amazon Web Services,us-west-2,BPA,US-NW-BPAT,BPA,US West (Oregon),"45.5371,-122.65 ",,0.95,,,,,,, 2021,Amazon Web Services,af-south-1,,ZA,ZA,Africa (Cape Town),"-33.9253,18.4239 ",,,,,,,,, 2021,Amazon Web Services,ap-east-1,Hong Kong,HK,HK,Asia Pacific (Hong Kong),"22.3,114.2",,,,,,,,, diff --git a/src/lib/cloud-metadata/index.ts b/src/lib/cloud-metadata/index.ts index e518ad9..a6b606f 100644 --- a/src/lib/cloud-metadata/index.ts +++ b/src/lib/cloud-metadata/index.ts @@ -15,7 +15,7 @@ import {AWS_HEADERS, AZURE_HEADERS, GSF_HEADERS} from './config'; const AWS_INSTANCES = path.resolve(__dirname, './aws-instances.csv'); const AZURE_INSTANCES = path.resolve(__dirname, './azure-instances.csv'); -const GSF_DATA = path.resolve(__dirname, './GSF-data.csv'); +const GSF_DATA = path.resolve(__dirname, './gsf-data.csv'); const {UnsupportedValueError} = ERRORS; @@ -163,10 +163,7 @@ export const CloudMetadata = (): PluginInterface => { if (instanceType.includes('-')) { const [instanceFamily, instanceSize] = instanceType.split('-'); const sizeNumberIndex = instanceSize.search(/\D/); - const instanceSizeNumber = - sizeNumberIndex !== -1 - ? instanceSize.slice(sizeNumberIndex) - : instanceSize; + const instanceSizeNumber = instanceSize.slice(sizeNumberIndex); instanceType = `${instanceFamily}${instanceSizeNumber}`; } diff --git a/src/lib/coefficient/README.md b/src/lib/coefficient/README.md index 3d29a02..d74a81e 100644 --- a/src/lib/coefficient/README.md +++ b/src/lib/coefficient/README.md @@ -62,6 +62,8 @@ name: coefficient-demo description: tags: initialize: + outputs: + - yaml plugins: coefficient: method: Coefficient diff --git a/src/lib/csv-export/README.md b/src/lib/csv-export/README.md index 1c7bdaf..335c2f1 100644 --- a/src/lib/csv-export/README.md +++ b/src/lib/csv-export/README.md @@ -52,6 +52,8 @@ name: csv-export-demo description: example exporting output to a csv file tags: initialize: + outputs: + - yaml plugins: csv-exporter: method: CsvExport diff --git a/src/lib/divide/README.md b/src/lib/divide/README.md index 2dab1b5..0611859 100644 --- a/src/lib/divide/README.md +++ b/src/lib/divide/README.md @@ -64,6 +64,8 @@ name: divide-demo description: tags: initialize: + outputs: + - yaml plugins: divide: method: Divide diff --git a/src/lib/e-mem/README.md b/src/lib/e-mem/README.md index 6729711..9d20f99 100644 --- a/src/lib/e-mem/README.md +++ b/src/lib/e-mem/README.md @@ -8,7 +8,7 @@ ### Plugin global config - `energy-per-gb`: a coefficient for energy in kWh per GB. If not provided, - defaults to 0.000392. (optional) + defaults to 0.000392 (optional), take from [this case study](https://www.cloudcarbonfootprint.org/docs/methodology/#memory) ### Inputs @@ -59,6 +59,8 @@ name: e-mem-demo description: tags: initialize: + outputs: + - yaml plugins: e-mem: method: EMem diff --git a/src/lib/e-mem/index.ts b/src/lib/e-mem/index.ts index a021f6d..bebe7fb 100644 --- a/src/lib/e-mem/index.ts +++ b/src/lib/e-mem/index.ts @@ -47,11 +47,12 @@ export const EMem = (globalConfig: ConfigParams): PluginInterface => { const validateConfig = () => { const schema = z.object({ - 'energy-per-gb': z.number().gt(0), + 'energy-per-gb': z.number().gte(0.000392), }); - //Manually add default value from CCF: https://www.cloudcarbonfootprint.org/docs/methodology/#memory - const energyPerGB = globalConfig['energy-per-gb'] ?? 0.000392; + // Manually add default value from CCF: https://www.cloudcarbonfootprint.org/docs/methodology/#memory + const energyPerGB = + (globalConfig && globalConfig['energy-per-gb']) ?? 0.000392; return validate>(schema, { ...globalConfig, diff --git a/src/lib/e-net/README.md b/src/lib/e-net/README.md index ba563f9..1581dbf 100644 --- a/src/lib/e-net/README.md +++ b/src/lib/e-net/README.md @@ -6,7 +6,7 @@ ### Plugin global config -- `energy-per-gb`: coefficient for converting data transferred to energy, in kWh/GB. The default, if no data or invalid data is provided, is 0.001 kWh/GB, taken from [this case study](https://github.com/Green-Software-Foundation/sci-guide/blob/dev/use-case-submissions/msft-eShoppen.md). +- `energy-per-gb`: coefficient for converting data transferred to energy, in kWh/GB. The default, if no data or invalid data is provided, is 0.001 kWh/GB, taken from [case study 1](https://www.cloudcarbonfootprint.org/docs/methodology/#chosen-coefficient) and [case study 2](https://www.cloudcarbonfootprint.org/docs/methodology/#appendix-iv-recent-networking-studies). ### Inputs @@ -57,6 +57,8 @@ name: e-net-demo description: tags: initialize: + outputs: + - yaml plugins: e-net: method: ENet diff --git a/src/lib/e-net/index.ts b/src/lib/e-net/index.ts index 292aede..dc7fb0f 100644 --- a/src/lib/e-net/index.ts +++ b/src/lib/e-net/index.ts @@ -35,15 +35,17 @@ export const ENet = (globalConfig: ConfigParams): PluginInterface => { */ const validateConfig = () => { const schema = z.object({ - 'energy-per-gb': z.number(), + 'energy-per-gb': z.number().gte(0.001), }); - // Manually add default value - if (!globalConfig['energy-per-gb'] || globalConfig['energy-per-gb'] === 0) { - globalConfig['energy-per-gb'] = 0.001; - } + // Manually add default value from CCF: https://www.cloudcarbonfootprint.org/docs/methodology/#chosen-coefficient + const energyPerGB = + (globalConfig && globalConfig['energy-per-gb']) ?? 0.001; - return validate>(schema, globalConfig); + return validate>(schema, { + ...globalConfig, + 'energy-per-gb': energyPerGB, + }); }; /** diff --git a/src/lib/mock-observations/helpers/rand-int-generator.ts b/src/lib/mock-observations/helpers/rand-int-generator.ts index 0615b91..cc3c225 100644 --- a/src/lib/mock-observations/helpers/rand-int-generator.ts +++ b/src/lib/mock-observations/helpers/rand-int-generator.ts @@ -13,11 +13,9 @@ export const RandIntGenerator = ( ): Generator => { const errorBuilder = buildErrorMessage(RandIntGenerator.name); - const next = (_historical: Object[] | undefined): Object => - (validatedName && { - [validatedName]: generateRandInt(getFieldToPopulate()), - }) || - {}; + const next = (_historical: Object[] | undefined) => ({ + [validatedName]: generateRandInt(getFieldToPopulate()), + }); const validateName = (name: string | null): string => { if (!name || name.trim() === '') { @@ -38,16 +36,22 @@ export const RandIntGenerator = ( }) ); } - if ( - !Object.prototype.hasOwnProperty.call(config, 'min') || - !Object.prototype.hasOwnProperty.call(config, 'max') - ) { + + if (!config.min || !config.max) { throw new InputValidationError( errorBuilder({ message: 'Config is missing min or max', }) ); } + + if (config.min >= config.max) { + throw new InputValidationError( + errorBuilder({ + message: `Min value should not be greater than or equal to max value of ${validatedName}`, + }) + ); + } return {min: config.min, max: config.max}; }; diff --git a/src/lib/mock-observations/index.ts b/src/lib/mock-observations/index.ts index a0f2b93..9b2ce48 100644 --- a/src/lib/mock-observations/index.ts +++ b/src/lib/mock-observations/index.ts @@ -1,27 +1,19 @@ -import * as dayjs from 'dayjs'; -import * as utc from 'dayjs/plugin/utc'; -import * as timezone from 'dayjs/plugin/timezone'; - -import {buildErrorMessage} from '../../util/helpers'; -import {ERRORS} from '../../util/errors'; +import {DateTime, Duration} from 'luxon'; +import {z} from 'zod'; import {PluginInterface} from '../../interfaces'; import {ConfigParams, KeyValuePair, PluginParams} from '../../types/common'; +import {validate} from '../../util/validations'; + import {CommonGenerator} from './helpers/common-generator'; import {RandIntGenerator} from './helpers/rand-int-generator'; import {Generator} from './interfaces/index'; import {ObservationParams} from './types'; -dayjs.extend(utc); -dayjs.extend(timezone); - -const {InputValidationError} = ERRORS; - export const MockObservations = ( globalConfig: ConfigParams ): PluginInterface => { - const errorBuilder = buildErrorMessage('MockObservations'); const metadata = { kind: 'execute', }; @@ -48,7 +40,7 @@ export const MockObservations = ( generatorToHistory ); - acc.push(Object.assign(observation, defaults)); + acc.push(Object.assign({}, defaults, observation)); }); return acc; @@ -57,62 +49,67 @@ export const MockObservations = ( ); }; + /** + * Validates global config parameters. + */ + const validateGlobalConfig = () => { + const schema = z.object({ + 'timestamp-from': z.string(), + 'timestamp-to': z.string(), + duration: z.number(), + components: z.array(z.record(z.string())), + generators: z.object({ + common: z.record(z.string().or(z.number())), + randint: z.record(z.object({min: z.number(), max: z.number()})), + }), + }); + + return validate>(schema, globalConfig); + }; + /** * Configures the MockObservations Plugin for IF */ const generateParamsFromConfig = async () => { - const timestampFrom = dayjs.tz( - getValidatedParam('timestamp-from', globalConfig), - 'UTC' - ); - const timestampTo = dayjs.tz( - getValidatedParam('timestamp-to', globalConfig), - 'UTC' - ); - const duration = getValidatedParam('duration', globalConfig); + const { + 'timestamp-from': timestampFrom, + 'timestamp-to': timestampTo, + duration, + generators, + components, + } = validateGlobalConfig(); + const convertedTimestampFrom = DateTime.fromISO(timestampFrom, { + zone: 'UTC', + }); + const convertedTimestampTo = DateTime.fromISO(timestampTo, {zone: 'UTC'}); return { duration, - timeBuckets: createTimeBuckets(timestampFrom, timestampTo, duration), - components: getValidatedParam('components', globalConfig) as KeyValuePair, - generators: createGenerators( - getValidatedParam('generators', globalConfig) + timeBuckets: createTimeBuckets( + convertedTimestampFrom, + convertedTimestampTo, + duration ), + generators: createGenerators(generators), + components, }; }; - /* - * validate a parameter is included in a given parameters map. - * return the validated param value, otherwise throw an InputValidationError. - */ - const getValidatedParam = ( - paramName: string, - params: {[key: string]: any} - ): T => { - if (!(paramName in params)) { - throw new InputValidationError( - errorBuilder({message: `${paramName} missing from global config`}) - ); - } - - return params[paramName]; - }; - /* * create time buckets based on start time, end time and duration of each bucket. */ const createTimeBuckets = ( - timestampFrom: dayjs.Dayjs, - timestampTo: dayjs.Dayjs, + timestampFrom: DateTime, + timestampTo: DateTime, duration: number, - timeBuckets: dayjs.Dayjs[] = [] - ): dayjs.Dayjs[] => { + timeBuckets: DateTime[] = [] + ): DateTime[] => { if ( - timestampFrom.isBefore(timestampTo) || - timestampFrom.add(duration, 'second').isBefore(timestampTo) + timestampFrom < timestampTo || + timestampFrom.plus(Duration.fromObject({seconds: duration})) < timestampTo ) { return createTimeBuckets( - timestampFrom.add(duration, 'second'), + timestampFrom.plus(Duration.fromObject({seconds: duration})), timestampTo, duration, [...timeBuckets, timestampFrom] @@ -135,14 +132,11 @@ export const MockObservations = ( ); }; - return Object.entries(generatorsConfig).flatMap(([key, value]) => { - if (key === 'common') { - return createCommonGenerator(value); - } else if (key === 'randint') { - return createRandIntGenerators(value).flat(); - } - return []; - }); + return Object.entries(generatorsConfig).flatMap(([key, value]) => + key === 'randint' + ? createRandIntGenerators(value).flat() + : createCommonGenerator(value) + ); }; /* @@ -153,7 +147,7 @@ export const MockObservations = ( generatorToHistory: Map ): PluginParams => { const {duration, component, timeBucket, generators} = observationParams; - const timestamp = timeBucket.toISOString(); + const timestamp = timeBucket.toISO(); const generateObservation = (generator: Generator) => { const history = generatorToHistory.get(generator) || []; diff --git a/src/lib/mock-observations/types.ts b/src/lib/mock-observations/types.ts index dfb2948..8259120 100644 --- a/src/lib/mock-observations/types.ts +++ b/src/lib/mock-observations/types.ts @@ -1,10 +1,10 @@ -import * as dayjs from 'dayjs'; +import {DateTime} from 'luxon'; import {Generator} from './interfaces/index'; export type ObservationParams = { duration: number; - timeBucket: dayjs.Dayjs; + timeBucket: DateTime; component: Record; generators: Generator[]; }; diff --git a/src/lib/multiply/README.md b/src/lib/multiply/README.md index f20ac1f..fbbc14e 100644 --- a/src/lib/multiply/README.md +++ b/src/lib/multiply/README.md @@ -61,6 +61,8 @@ name: multiply-demo description: tags: initialize: + outputs: + - yaml plugins: multiply: method: Multiply diff --git a/src/lib/regex/README.md b/src/lib/regex/README.md index e907be8..55a8ba4 100644 --- a/src/lib/regex/README.md +++ b/src/lib/regex/README.md @@ -59,6 +59,8 @@ name: regex-demo description: tags: initialize: + outputs: + - yaml plugins: regex: method: Regex diff --git a/src/lib/regex/index.ts b/src/lib/regex/index.ts index 767a5ed..d7cc445 100644 --- a/src/lib/regex/index.ts +++ b/src/lib/regex/index.ts @@ -77,15 +77,12 @@ export const Regex = (globalConfig: ConfigParams): PluginInterface => { parameter: string, match: string ) => { - if ( - !match.startsWith('/') || - (match.endsWith('/g') && match.lastIndexOf('/') === 0) - ) { + if (!match.startsWith('/')) { match = '/' + match; + } - if (!match.endsWith('/g') || !match.endsWith('/')) { - match += '/'; - } + if (!match.endsWith('/g') && !match.endsWith('/')) { + match += '/'; } const regex = eval(match); diff --git a/src/lib/sci-e/README.md b/src/lib/sci-e/README.md index 7a92925..5e1e86b 100644 --- a/src/lib/sci-e/README.md +++ b/src/lib/sci-e/README.md @@ -67,6 +67,8 @@ name: sci-e-demo description: tags: initialize: + outputs: + - yaml plugins: sci-e: method: SciE diff --git a/src/lib/sci-m/README.md b/src/lib/sci-m/README.md index b73782c..f945d91 100644 --- a/src/lib/sci-m/README.md +++ b/src/lib/sci-m/README.md @@ -79,6 +79,8 @@ name: sci-m description: simple demo invoking sci-m tags: initialize: + outputs: + - yaml plugins: sci-m: method: SciM diff --git a/src/lib/sci-m/index.ts b/src/lib/sci-m/index.ts index ee0cbea..84be886 100644 --- a/src/lib/sci-m/index.ts +++ b/src/lib/sci-m/index.ts @@ -4,13 +4,8 @@ import {PluginInterface} from '../../interfaces'; import {PluginParams} from '../../types/common'; import {validate, allDefined} from '../../util/validations'; -import {buildErrorMessage} from '../../util/helpers'; -import {ERRORS} from '../../util/errors'; - -const {InputValidationError} = ERRORS; export const SciM = (): PluginInterface => { - const errorBuilder = buildErrorMessage(SciM.name); const metadata = { kind: 'execute', }; @@ -42,23 +37,12 @@ export const SciM = (): PluginInterface => { * M = totalEmissions * (duration/ExpectedLifespan) * (resourcesReserved/totalResources) */ const calculateEmbodiedCarbon = (input: PluginParams) => { - const totalEmissions = parseNumberInput( - input['device/emissions-embodied'], - 'gCO2e' - ); - const duration = parseNumberInput(input['duration'], 'seconds'); - const expectedLifespan = parseNumberInput( - input['device/expected-lifespan'], - 'seconds' - ); - const resourcesReserved = parseNumberInput( - input['vcpus-allocated'] || input['resources-reserved'], - 'count' - ); - const totalResources = parseNumberInput( - input['vcpus-total'] || input['resources-total'], - 'count' - ); + const totalEmissions = input['device/emissions-embodied']; + const duration = input['duration']; + const expectedLifespan = input['device/expected-lifespan']; + const resourcesReserved = + input['vcpus-allocated'] || input['resources-reserved']; + const totalResources = input['vcpus-total'] || input['resources-total']; return ( totalEmissions * @@ -68,41 +52,69 @@ export const SciM = (): PluginInterface => { }; /** - * Parses the input value, ensuring it is a valid number, and returns the parsed number. - * Throws an InputValidationError if the value is not a valid number. + * Checks for required fields in input. */ - const parseNumberInput = (value: any, unit: string): number => { - const parsedValue = typeof value === 'string' ? parseFloat(value) : value; + const validateInput = (input: PluginParams) => { + const errorMessage = (unit: string) => + `not a valid number in input. Please provide it as \`${unit}\``; - if (typeof parsedValue !== 'number' || isNaN(parsedValue)) { - throw new InputValidationError( - errorBuilder({ - message: `'${value}' is not a valid number in input. Please provide it as ${unit}.`, + const commonSchemaPart = (errorMessage: (unit: string) => string) => ({ + 'device/emissions-embodied': z + .number({ + invalid_type_error: errorMessage('gCO2e'), + }) + .gte(0) + .min(0), + 'device/expected-lifespan': z + .number({ + invalid_type_error: errorMessage('gCO2e'), }) - ); - } + .gte(0) + .min(0), + duration: z + .number({ + invalid_type_error: errorMessage('seconds'), + }) + .gte(1), + }); - return parsedValue; - }; + const vcpusSchemaPart = { + 'vcpus-allocated': z + .number({ + invalid_type_error: errorMessage('count'), + }) + .gte(0) + .min(0), + 'vcpus-total': z + .number({ + invalid_type_error: errorMessage('count'), + }) + .gte(0) + .min(0), + }; + + const resourcesSchemaPart = { + 'resources-reserved': z + .number({ + invalid_type_error: errorMessage('count'), + }) + .gte(0) + .min(0), + 'resources-total': z + .number({ + invalid_type_error: errorMessage('count'), + }) + .gte(0) + .min(0), + }; - /** - * Checks for required fields in input. - */ - const validateInput = (input: PluginParams) => { const schemaWithVcpus = z.object({ - 'device/emissions-embodied': z.number().gte(0).min(0), - 'device/expected-lifespan': z.number().gte(0).min(0), - 'vcpus-allocated': z.number().gte(0).min(0), - 'vcpus-total': z.number().gte(0).min(0), - duration: z.number().gte(1), + ...commonSchemaPart(errorMessage), + ...vcpusSchemaPart, }); - const schemaWithResources = z.object({ - 'device/emissions-embodied': z.number().gte(0).min(0), - 'device/expected-lifespan': z.number().gte(0).min(0), - 'resources-reserved': z.number().gte(0).min(0), - 'resources-total': z.number().gte(0).min(0), - duration: z.number().gte(1), + ...commonSchemaPart(errorMessage), + ...resourcesSchemaPart, }); const schema = schemaWithVcpus.or(schemaWithResources).refine(allDefined, { diff --git a/src/lib/sci-o/README.md b/src/lib/sci-o/README.md index 2342052..ef1c97a 100644 --- a/src/lib/sci-o/README.md +++ b/src/lib/sci-o/README.md @@ -68,6 +68,8 @@ name: sci-o description: tags: initialize: + outputs: + - yaml plugins: sci-o: method: SciO diff --git a/src/lib/sci/README.md b/src/lib/sci/README.md index fb9f41b..12f795b 100644 --- a/src/lib/sci/README.md +++ b/src/lib/sci/README.md @@ -135,6 +135,8 @@ name: sci-demo description: example invoking sci plugin tags: initialize: + outputs: + - yaml plugins: sci: method: Sci diff --git a/src/lib/shell/README.md b/src/lib/shell/README.md index 9c4865b..8627824 100644 --- a/src/lib/shell/README.md +++ b/src/lib/shell/README.md @@ -83,6 +83,8 @@ name: shell-demo description: tags: initialize: + outputs: + - yaml plugins: sampler: method: Shell diff --git a/src/lib/sum/README.md b/src/lib/sum/README.md index b947a10..88b9582 100644 --- a/src/lib/sum/README.md +++ b/src/lib/sum/README.md @@ -61,6 +61,8 @@ name: sum demo description: tags: initialize: + outputs: + - yaml plugins: sum: method: Sum diff --git a/src/lib/tdp-finder/README.md b/src/lib/tdp-finder/README.md index 4e7a92b..f186035 100644 --- a/src/lib/tdp-finder/README.md +++ b/src/lib/tdp-finder/README.md @@ -58,6 +58,8 @@ name: tdp-demo description: tags: initialize: + outputs: + - yaml plugins: finder: method: TdpFinder