From 61201ce5a967d9402c948be8d706d7450006e39a Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 8 Jul 2024 14:27:30 +0400 Subject: [PATCH 01/37] feat(src): add util functions for if-csv --- src/if-csv/util/args.ts | 60 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 src/if-csv/util/args.ts diff --git a/src/if-csv/util/args.ts b/src/if-csv/util/args.ts new file mode 100644 index 000000000..4b9674882 --- /dev/null +++ b/src/if-csv/util/args.ts @@ -0,0 +1,60 @@ +import {parse} from 'ts-command-line-args'; +import {ERRORS} from '@grnsft/if-core/utils'; + +import {checkIfFileIsYaml} from '../../common/util/yaml'; +import {prependFullFilePath} from '../../common/util/helpers'; + +import {CONFIG, STRINGS} from '../config'; +import {STRINGS as COMMON_STRINGS} from '../../common/config'; + +import {IFCsvArgs} from '../types/process-args'; +import {isFileExists} from '../../common/util/fs'; + +const {ParseCliParamsError, CliSourceFileError} = ERRORS; + +const {ARGS, HELP} = CONFIG; +const {PARAMS_NOT_PRESENT} = STRINGS; +const {SOURCE_IS_NOT_YAML, MANIFEST_NOT_FOUND} = COMMON_STRINGS; + +/** + * Parses `if-csv` process arguments. + */ +const validateAndParseIfCsvArgs = () => { + try { + return parse(ARGS, HELP); + } catch (error) { + if (error instanceof Error) { + throw new ParseCliParamsError(error.message); + } + + throw error; + } +}; + +/** + * Checks for `manifest` and `params` flags to be present. + */ +export const parseIfCsvArgs = async () => { + const {manifest, output, params} = validateAndParseIfCsvArgs(); + + if (!params) { + throw new ParseCliParamsError(PARAMS_NOT_PRESENT); + } + + if (manifest) { + const response = prependFullFilePath(manifest); + const isManifestFileExists = await isFileExists(response); + + if (!isManifestFileExists) { + throw new ParseCliParamsError(MANIFEST_NOT_FOUND); + } + + if (checkIfFileIsYaml(manifest)) { + return {manifest, output, params}; + } + + throw new CliSourceFileError(SOURCE_IS_NOT_YAML); + } + + return {output, params}; +}; From 04a57638c837d6a24d6907508280078b46c69bcc Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 8 Jul 2024 14:30:16 +0400 Subject: [PATCH 02/37] feat(src): add executeCsv helper function for if-csv --- src/if-csv/util/helpers.ts | 66 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 src/if-csv/util/helpers.ts diff --git a/src/if-csv/util/helpers.ts b/src/if-csv/util/helpers.ts new file mode 100644 index 000000000..b70ca2194 --- /dev/null +++ b/src/if-csv/util/helpers.ts @@ -0,0 +1,66 @@ +import {stringify} from 'csv-stringify/sync'; +import * as fs from 'fs/promises'; +import {PluginParams} from '@grnsft/if-core/types'; + +import {CsvOptions} from '../types/csv'; + +/** + * Executes a CSV generation based on the provided tree structure, context, output path, and params. + */ +export const executeCsv = async (options: CsvOptions) => { + const {tree, context, outputPath, params} = options; + const columns = ['Path']; + const matrix = [columns]; + const aggregationIsEnabled = !!context.aggregation; + + const traverseTree = (node: any, params: string, path = 'tree') => { + if (node.aggregated) { + if (path === 'tree') { + columns.push('Aggregated'); + } + matrix.push([`${path}.${params}`, node.aggregated[params]]); + } + + if (node.outputs) { + node.outputs.forEach((output: PluginParams) => { + const {timestamp} = output; + + if (!columns.includes(timestamp)) { + columns.push(timestamp); + } + + const lastRow = matrix[matrix.length - 1]; + + if (aggregationIsEnabled) { + lastRow.push(output[params]); + } else { + if (matrix.length === 1 || lastRow.length === columns.length) { + matrix.push([`${path}.${params}`, output[params]]); + } else { + lastRow.push(output[params]); + } + } + }); + } + + if (node.children) { + Object.keys(node.children).forEach(childKey => { + traverseTree( + node.children![childKey], + params, + `${path}.children.${childKey}` + ); + }); + } + }; + + traverseTree(tree, params); + + const csvString = stringify(matrix, {columns}); + + if (!outputPath) { + return csvString; + } + + return await fs.writeFile(`${outputPath}.csv`, csvString); +}; From 142d957d62ea194bd00b477b81595a939c2c8146 Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 8 Jul 2024 14:33:38 +0400 Subject: [PATCH 03/37] feat(src): move `parseManifestFromStdin` function from if-diff helpers to common helpers file --- src/common/util/helpers.ts | 44 +++++++++++++++++++++++++++++++++++++ src/if-diff/index.ts | 3 ++- src/if-diff/util/helpers.ts | 44 ------------------------------------- 3 files changed, 46 insertions(+), 45 deletions(-) diff --git a/src/common/util/helpers.ts b/src/common/util/helpers.ts index 8399e5b3e..01b87c79d 100644 --- a/src/common/util/helpers.ts +++ b/src/common/util/helpers.ts @@ -1,3 +1,4 @@ +import {createInterface} from 'node:readline/promises'; import {exec} from 'child_process'; import * as path from 'path'; import {promisify} from 'util'; @@ -19,3 +20,46 @@ export const prependFullFilePath = (filePath: string) => { return path.normalize(`${processRunningPath}/${filePath}`); }; + +/** + * Checks if there is data piped, then collects it. + * Otherwise returns empty string. + */ +const collectPipedData = async () => { + if (process.stdin.isTTY) { + return ''; + } + + const readline = createInterface({ + input: process.stdin, + }); + + const data: string[] = []; + + for await (const line of readline) { + data.push(line); + } + + return data.join('\n'); +}; + +/** + * Checks if there is piped data, tries to parse yaml from it. + * Returns empty string if haven't found anything. + */ +export const parseManifestFromStdin = async () => { + const pipedSourceManifest = await collectPipedData(); + + if (!pipedSourceManifest) { + return ''; + } + + const regex = /# start((?:.*\n)+?)# end/; + const match = regex.exec(pipedSourceManifest); + + if (!match) { + return ''; + } + + return match![1]; +}; diff --git a/src/if-diff/index.ts b/src/if-diff/index.ts index 52a6a9b4e..aa4ae1871 100644 --- a/src/if-diff/index.ts +++ b/src/if-diff/index.ts @@ -4,7 +4,8 @@ import {loadIfDiffFiles} from './lib/load'; import {compare} from './lib/compare'; import {parseIfDiffArgs} from './util/args'; -import {formatNotMatchingLog, parseManifestFromStdin} from './util/helpers'; +import {formatNotMatchingLog} from './util/helpers'; +import {parseManifestFromStdin} from '../common/util/helpers'; import {validateManifest} from '../common/util/validations'; import {logger} from '../common/util/logger'; diff --git a/src/if-diff/util/helpers.ts b/src/if-diff/util/helpers.ts index 858d5333c..9552e8f5b 100644 --- a/src/if-diff/util/helpers.ts +++ b/src/if-diff/util/helpers.ts @@ -1,4 +1,3 @@ -import {createInterface} from 'node:readline/promises'; import {Difference} from '../types/compare'; /** @@ -83,46 +82,3 @@ export const formatNotMatchingLog = (message: Difference) => { } }); }; - -/** - * Checks if there is data piped, then collects it. - * Otherwise returns empty string. - */ -const collectPipedData = async () => { - if (process.stdin.isTTY) { - return ''; - } - - const readline = createInterface({ - input: process.stdin, - }); - - const data: string[] = []; - - for await (const line of readline) { - data.push(line); - } - - return data.join('\n'); -}; - -/** - * Checks if there is piped data, tries to parse yaml from it. - * Returns empty string if haven't found anything. - */ -export const parseManifestFromStdin = async () => { - const pipedSourceManifest = await collectPipedData(); - - if (!pipedSourceManifest) { - return ''; - } - - const regex = /# start((?:.*\n)+?)# end/; - const match = regex.exec(pipedSourceManifest); - - if (!match) { - return ''; - } - - return match![1]; -}; From db51036b4828ee1bce98f0c4f59ec462ca241a71 Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 8 Jul 2024 14:35:48 +0400 Subject: [PATCH 04/37] test(src): update tests according to changes --- src/__tests__/common/lib/load.test.ts | 2 +- src/__tests__/common/util/helpers.test.ts | 44 ++++++++++++++++++++++ src/__tests__/if-diff/lib/load.test.ts | 4 +- src/__tests__/if-diff/util/helpers.test.ts | 40 -------------------- 4 files changed, 47 insertions(+), 43 deletions(-) create mode 100644 src/__tests__/common/util/helpers.test.ts diff --git a/src/__tests__/common/lib/load.test.ts b/src/__tests__/common/lib/load.test.ts index 43f6ee2af..a76d3bd27 100644 --- a/src/__tests__/common/lib/load.test.ts +++ b/src/__tests__/common/lib/load.test.ts @@ -14,7 +14,7 @@ jest.mock( }), {virtual: true} ); -jest.mock('../../../if-diff/util/helpers', () => ({ +jest.mock('../../../common/util/helpers', () => ({ parseManifestFromStdin: () => { if (process.env.readline === 'valid-source') { return ` diff --git a/src/__tests__/common/util/helpers.test.ts b/src/__tests__/common/util/helpers.test.ts new file mode 100644 index 000000000..92b4f9dab --- /dev/null +++ b/src/__tests__/common/util/helpers.test.ts @@ -0,0 +1,44 @@ +jest.mock('node:readline/promises', () => + require('../../../__mocks__/readline') +); + +import {parseManifestFromStdin} from '../../../common/util/helpers'; + +describe('common/util/helpers: ', () => { + describe('parseManifestFromStdin(): ', () => { + it('returns empty string if there is no data in stdin.', async () => { + const response = await parseManifestFromStdin(); + const expectedResult = ''; + + expect(response).toEqual(expectedResult); + }); + + it('returns empty string if nothing is piped.', async () => { + const originalIsTTY = process.stdin.isTTY; + process.stdin.isTTY = true; + const response = await parseManifestFromStdin(); + const expectedResult = ''; + + expect(response).toEqual(expectedResult); + process.stdin.isTTY = originalIsTTY; + }); + + it('throws error if there is no manifest in stdin.', async () => { + process.env.readline = 'no_manifest'; + expect.assertions(1); + + const response = await parseManifestFromStdin(); + + expect(response).toEqual(''); + }); + + it('returns empty string if there is no data in stdin.', async () => { + process.env.readline = 'manifest'; + const response = await parseManifestFromStdin(); + const expectedMessage = + '\nname: mock-name\ndescription: mock-description\n'; + + expect(response).toEqual(expectedMessage); + }); + }); +}); diff --git a/src/__tests__/if-diff/lib/load.test.ts b/src/__tests__/if-diff/lib/load.test.ts index f722757e5..8a703df76 100644 --- a/src/__tests__/if-diff/lib/load.test.ts +++ b/src/__tests__/if-diff/lib/load.test.ts @@ -14,7 +14,7 @@ jest.mock( }), {virtual: true} ); -jest.mock('../../../if-diff/util/helpers', () => ({ +jest.mock('../../../common/util/helpers', () => ({ parseManifestFromStdin: () => { if (process.env.readline === 'valid-source') { return ` @@ -44,7 +44,7 @@ import {PluginParams} from '@grnsft/if-core/types'; import {loadIfDiffFiles} from '../../../if-diff/lib/load'; -import {parseManifestFromStdin} from '../../../if-diff/util/helpers'; +import {parseManifestFromStdin} from '../../../common/util/helpers'; import {STRINGS} from '../../../if-diff/config'; diff --git a/src/__tests__/if-diff/util/helpers.test.ts b/src/__tests__/if-diff/util/helpers.test.ts index a7d25599e..eb9a0e553 100644 --- a/src/__tests__/if-diff/util/helpers.test.ts +++ b/src/__tests__/if-diff/util/helpers.test.ts @@ -7,7 +7,6 @@ import { checkIfEqual, formatNotMatchingLog, oneIsPrimitive, - parseManifestFromStdin, } from '../../../if-diff/util/helpers'; describe('if-diff/util/helpers: ', () => { @@ -193,43 +192,4 @@ describe('if-diff/util/helpers: ', () => { console.log = actualLogger; }); }); - - describe('parseManifestFromStdin(): ', () => { - it('returns empty string if there is no data in stdin.', async () => { - const response = await parseManifestFromStdin(); - const expectedResult = ''; - - expect(response).toEqual(expectedResult); - }); - - it('returns empty string if nothing is piped.', async () => { - const originalIsTTY = process.stdin.isTTY; - process.stdin.isTTY = true; - const response = await parseManifestFromStdin(); - const expectedResult = ''; - - expect(response).toEqual(expectedResult); - process.stdin.isTTY = originalIsTTY; - }); - - it('throws error if there is no manifest in stdin.', async () => { - process.env.readline = 'no_manifest'; - expect.assertions(1); - - const response = await parseManifestFromStdin(); - - expect(response).toEqual(''); - }); - - it('returns empty string if there is no data in stdin.', async () => { - process.env.readline = 'manifest'; - const response = await parseManifestFromStdin(); - const expectedMessage = ` -name: mock-name -description: mock-description -`; - - expect(response).toEqual(expectedMessage); - }); - }); }); From 981e925933a425e9cb3b317c39541319ce8b9d6e Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 8 Jul 2024 14:37:07 +0400 Subject: [PATCH 05/37] feat(src): add string and types files for if-csv --- src/if-csv/config/index.ts | 2 ++ src/if-csv/config/strings.ts | 3 +++ src/if-csv/types/csv.ts | 8 ++++++++ 3 files changed, 13 insertions(+) create mode 100644 src/if-csv/config/index.ts create mode 100644 src/if-csv/config/strings.ts create mode 100644 src/if-csv/types/csv.ts diff --git a/src/if-csv/config/index.ts b/src/if-csv/config/index.ts new file mode 100644 index 000000000..4972b390b --- /dev/null +++ b/src/if-csv/config/index.ts @@ -0,0 +1,2 @@ +export {CONFIG} from './config'; +export {STRINGS} from './strings'; diff --git a/src/if-csv/config/strings.ts b/src/if-csv/config/strings.ts new file mode 100644 index 000000000..be90a4547 --- /dev/null +++ b/src/if-csv/config/strings.ts @@ -0,0 +1,3 @@ +export const STRINGS = { + PARAMS_NOT_PRESENT: 'Parameter not provided.', +}; diff --git a/src/if-csv/types/csv.ts b/src/if-csv/types/csv.ts new file mode 100644 index 000000000..3e2f969c6 --- /dev/null +++ b/src/if-csv/types/csv.ts @@ -0,0 +1,8 @@ +import {Context} from '../../common/types/manifest'; + +export type CsvOptions = { + tree: any; + context: Context; + outputPath?: string; + params: string; +}; From b96e466e8eef1797ada0324332203a2b5bbcf4da Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 8 Jul 2024 14:39:23 +0400 Subject: [PATCH 06/37] feat(src): add config and types for args for if-csv --- src/if-csv/config/config.ts | 39 ++++++++++++++++++++++++++++++++ src/if-csv/types/process-args.ts | 5 ++++ 2 files changed, 44 insertions(+) create mode 100644 src/if-csv/config/config.ts create mode 100644 src/if-csv/types/process-args.ts diff --git a/src/if-csv/config/config.ts b/src/if-csv/config/config.ts new file mode 100644 index 000000000..8be9dba75 --- /dev/null +++ b/src/if-csv/config/config.ts @@ -0,0 +1,39 @@ +import {ArgumentConfig, ParseOptions} from 'ts-command-line-args'; + +import {STRINGS} from '../../common/config'; + +import {IFCsvArgs} from '../types/process-args'; + +const {DISCLAIMER_MESSAGE} = STRINGS; + +export const CONFIG = { + ARGS: { + manifest: { + type: String, + optional: true, + alias: 'm', + description: '[path to the manifest file]', + }, + output: { + type: String, + optional: true, + alias: 'o', + description: '[path to the csv output file]', + }, + params: { + type: String, + optional: false, + alias: 'p', + description: '[parameter to export]', + }, + } as ArgumentConfig, + HELP: { + helpArg: 'help', + headerContentSections: [ + {header: 'Impact Framework', content: 'IF-Csv Helpful keywords:'}, + ], + footerContentSections: [ + {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, + ], + } as ParseOptions, +}; diff --git a/src/if-csv/types/process-args.ts b/src/if-csv/types/process-args.ts new file mode 100644 index 000000000..f0bcb44ad --- /dev/null +++ b/src/if-csv/types/process-args.ts @@ -0,0 +1,5 @@ +export interface IFCsvArgs { + manifest?: string; + output?: string; + params: string; +} From 8d199be6b314efef23825ca7256b0386bfbf6f03 Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 8 Jul 2024 14:40:44 +0400 Subject: [PATCH 07/37] test(src): add test for helpers --- src/__tests__/if-csv/util/helpers.test.ts | 196 ++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 src/__tests__/if-csv/util/helpers.test.ts diff --git a/src/__tests__/if-csv/util/helpers.test.ts b/src/__tests__/if-csv/util/helpers.test.ts new file mode 100644 index 000000000..61e6f1e9c --- /dev/null +++ b/src/__tests__/if-csv/util/helpers.test.ts @@ -0,0 +1,196 @@ +import * as fs from 'fs/promises'; + +import {stringify} from 'csv-stringify/sync'; +import {jest} from '@jest/globals'; + +import {executeCsv} from '../../../if-csv/util/helpers'; + +import {CsvOptions} from '../../../if-csv/types/csv'; + +import { + tree, + context, + outputs, + aggregated, + aggregation, +} from '../../../__mocks__/builtins/export-csv'; + +jest.mock('fs/promises', () => ({ + writeFile: jest.fn<() => Promise>().mockResolvedValue(), +})); + +describe('executeCsv(): ', () => { + it('generates CSV file with correct data.', async () => { + const outputPath = 'output'; + const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; + const matrix = [ + columns, + ['tree.carbon', 8000.000004051243, 8000.000004051243], + ['tree.children.child-1.carbon', 4000.0000020256216, 4000.0000020256216], + ['tree.children.child-2.carbon', 4000.0000020256216, 4000.0000020256216], + ]; + const reformedContext = Object.assign({}, context, {outputs}); + const reformedTree = Object.assign({}, tree, { + children: { + ...tree.children, + 'child-1': { + ...tree.children['child-1'], + aggregated, + }, + 'child-2': { + ...tree.children['child-2'], + aggregated, + }, + }, + }); + + const options: CsvOptions = { + tree: reformedTree, + context: reformedContext, + outputPath, + params: 'carbon', + }; + + await executeCsv(options); + + expect(fs.writeFile).toHaveBeenCalledWith( + 'output.csv', + stringify(matrix, {columns}) + ); + }); + + it('generates CSV file when the `outputs` type is missing.', async () => { + const outputPath = 'output'; + const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; + const matrix = [ + columns, + ['tree.carbon', 8000.000004051243, 8000.000004051243], + ['tree.children.child-1.carbon', 4000.0000020256216, 4000.0000020256216], + ['tree.children.child-2.carbon', 4000.0000020256216, 4000.0000020256216], + ]; + + const reformedTree = Object.assign({}, tree, { + children: { + ...tree.children, + 'child-1': { + ...tree.children['child-1'], + aggregated, + }, + 'child-2': { + ...tree.children['child-2'], + aggregated, + }, + }, + }); + + const options: CsvOptions = { + tree: reformedTree, + context, + outputPath, + params: 'carbon', + }; + + await executeCsv(options); + + expect.assertions(1); + + expect(fs.writeFile).toHaveBeenCalledWith( + 'output.csv', + stringify(matrix, {columns}) + ); + }); + + it('generates CSV file when `aggregation` persists.', async () => { + const outputPath = 'output'; + const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; + const matrix = [ + columns, + ['tree.carbon', 8000.000004051243, 8000.000004051243], + ['tree.children.child-1.carbon', 4000.0000020256216, 4000.0000020256216], + ['tree.children.child-2.carbon', 4000.0000020256216, 4000.0000020256216], + ]; + + const reformedContext = Object.assign( + {}, + context, + {outputs}, + {aggregation} + ); + const reformedTree = Object.assign({}, tree, { + children: { + ...tree.children, + 'child-1': { + ...tree.children['child-1'], + aggregated, + }, + 'child-2': { + ...tree.children['child-2'], + aggregated, + }, + }, + }); + + const options: CsvOptions = { + tree: reformedTree, + context: reformedContext, + outputPath, + params: 'carbon', + }; + + await executeCsv(options); + + expect.assertions(1); + expect(fs.writeFile).toHaveBeenCalledWith( + 'output.csv', + stringify(matrix, {columns}) + ); + }); + + it('returns string when `outputPath` is not provided', async () => { + const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; + const matrix = [ + columns, + ['tree.carbon', 8000.000004051243, 8000.000004051243], + ['tree.children.child-1.carbon', 4000.0000020256216, 4000.0000020256216], + ]; + + const options: CsvOptions = { + tree, + context, + outputPath: undefined, + params: 'carbon', + }; + + const result = await executeCsv(options); + + expect.assertions(1); + expect(result).toEqual(stringify(matrix, {columns})); + }); + + it('generates CSV file when `aggregation` is missing.', async () => { + const outputPath = 'output'; + const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; + const matrix = [ + columns, + ['tree.carbon', 8000.000004051243, 8000.000004051243], + ['tree.children.child-1.carbon', 4000.0000020256216, 4000.0000020256216], + ['tree.children.child-2.carbon', 4000.0000020256216, 4000.0000020256216], + ]; + + const reformedContext = Object.assign({}, context, {outputs}); + const options: CsvOptions = { + tree, + context: reformedContext, + outputPath, + params: 'carbon', + }; + + await executeCsv(options); + + expect.assertions(1); + expect(fs.writeFile).toHaveBeenCalledWith( + 'output.csv', + stringify(matrix, {columns}) + ); + }); +}); From a24f1b4a17dc7fdeecd2a1e3567d7dcc42d9b8b1 Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 8 Jul 2024 14:42:59 +0400 Subject: [PATCH 08/37] feat(src): add logic for if-csv --- src/if-csv/index.ts | 62 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 src/if-csv/index.ts diff --git a/src/if-csv/index.ts b/src/if-csv/index.ts new file mode 100644 index 000000000..a53e54994 --- /dev/null +++ b/src/if-csv/index.ts @@ -0,0 +1,62 @@ +#!/usr/bin/env node +/* eslint-disable no-process-exit */ +import * as YAML from 'js-yaml'; + +import {parseManifestFromStdin} from '../common/util/helpers'; +import {validateManifest} from '../common/util/validations'; +import {debugLogger} from '../common/util/debug-logger'; +import {logger} from '../common/util/logger'; +import {load} from '../common/lib/load'; + +import {injectEnvironment} from '../if-run/lib/environment'; +import {initialize} from '../if-run/lib/initialize'; +import {aggregate} from '../if-run/lib/aggregate'; +import {compute} from '../if-run/lib/compute'; + +import {parseIfCsvArgs} from './util/args'; +import {executeCsv} from './util/helpers'; +import {CsvOptions} from './types/csv'; + +const IfCsv = async () => { + // Call this function with false parameter to prevent log debug messages. + debugLogger.overrideConsoleMethods(false); + + const pipedManifest = await parseManifestFromStdin(); + const {manifest, output, params} = await parseIfCsvArgs(); + const resolvedManifest = manifest || pipedManifest; + let envManifest; + + if (resolvedManifest) { + if (pipedManifest) { + envManifest = await YAML.load(pipedManifest!); + } else { + const {rawManifest} = await load(resolvedManifest); + envManifest = await injectEnvironment(rawManifest); + } + + const {tree, ...context} = validateManifest(envManifest); + const pluginStorage = await initialize(context.initialize.plugins); + const computedTree = await compute(tree, {context, pluginStorage}); + const aggregatedTree = aggregate(computedTree, context.aggregation); + const options: CsvOptions = { + tree: aggregatedTree, + context, + outputPath: output, + params, + }; + const result = await executeCsv(options); + + if (!output && result) { + console.log(result); + } + } + + process.exit(0); +}; + +IfCsv().catch(error => { + if (error instanceof Error) { + logger.error(error); + process.exit(2); + } +}); From 2cac2e374f6b9db83aea60b8f94c55ab98145f5c Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 8 Jul 2024 14:44:08 +0400 Subject: [PATCH 09/37] feat(package): add if-csv script --- package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 1b888aa3d..7c80610c5 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ "if-diff": "./build/if-diff/index.js", "if-run": "./build/if-run/index.js", "if-env": "./build/if-env/index.js", - "if-check": "./build/if-check/index.js" + "if-check": "./build/if-check/index.js", + "if-csv": "./build/if-csv/index.js" }, "bugs": { "url": "https://github.com/Green-Software-Foundation/if/issues/new?assignees=&labels=feedback&projects=&template=feedback.md&title=Feedback+-+" @@ -76,6 +77,7 @@ "fix": "gts fix", "fix:package": "fixpack", "if-check": "cross-env CURRENT_DIR=$(node -p \"process.env.INIT_CWD\") npx ts-node src/if-check/index.ts", + "if-csv": "cross-env CURRENT_DIR=$(node -p \"process.env.INIT_CWD\") npx ts-node src/if-csv/index.ts", "if-diff": "npx ts-node src/if-diff/index.ts", "if-env": "cross-env CURRENT_DIR=$(node -p \"process.env.INIT_CWD\") npx ts-node src/if-env/index.ts", "if-run": "npx ts-node src/if-run/index.ts", From 01ace664f03515b651016808fbec83778ccdf486 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 10 Jul 2024 13:31:38 +0400 Subject: [PATCH 10/37] fix(config): add failure string for if-csv --- src/if-csv/config/strings.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/if-csv/config/strings.ts b/src/if-csv/config/strings.ts index be90a4547..717461c1b 100644 --- a/src/if-csv/config/strings.ts +++ b/src/if-csv/config/strings.ts @@ -1,3 +1,4 @@ export const STRINGS = { PARAMS_NOT_PRESENT: 'Parameter not provided.', + FAILURE_MESSAGE_OUTPUTS: 'Manifest outputs are not available!', }; From ebf1770ff44469fcea46b13cd45250a71bb689e1 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 10 Jul 2024 13:33:24 +0400 Subject: [PATCH 11/37] fix(util): add getManifestData function --- src/if-csv/util/helpers.ts | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/if-csv/util/helpers.ts b/src/if-csv/util/helpers.ts index b70ca2194..d9f7ec09d 100644 --- a/src/if-csv/util/helpers.ts +++ b/src/if-csv/util/helpers.ts @@ -1,8 +1,29 @@ -import {stringify} from 'csv-stringify/sync'; import * as fs from 'fs/promises'; +import {stringify} from 'csv-stringify/sync'; import {PluginParams} from '@grnsft/if-core/types'; +import {ERRORS} from '@grnsft/if-core/utils'; + +import {load} from '../../common/lib/load'; import {CsvOptions} from '../types/csv'; +import {STRINGS} from '../config'; + +const {FAILURE_MESSAGE_OUTPUTS} = STRINGS; +const {ManifestValidationError} = ERRORS; + +/** + * Gets the folder path of the manifest file, dependencies from manifest file and install argument from the given arguments. + */ +export const getManifestData = async (manifest: string) => { + const loadedManifest = await load(manifest); + const children = loadedManifest.rawManifest.tree.children; + + if ((children.child || children['child-0']).outputs) { + return loadedManifest.rawManifest; + } + + throw new ManifestValidationError(FAILURE_MESSAGE_OUTPUTS); +}; /** * Executes a CSV generation based on the provided tree structure, context, output path, and params. From 083340d70f531551d4fe6f7d505578486c60157b Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 10 Jul 2024 13:35:19 +0400 Subject: [PATCH 12/37] fix(src): update logix to accpet executed manifest --- src/if-csv/index.ts | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/src/if-csv/index.ts b/src/if-csv/index.ts index a53e54994..94ede0bb4 100644 --- a/src/if-csv/index.ts +++ b/src/if-csv/index.ts @@ -3,18 +3,12 @@ import * as YAML from 'js-yaml'; import {parseManifestFromStdin} from '../common/util/helpers'; -import {validateManifest} from '../common/util/validations'; import {debugLogger} from '../common/util/debug-logger'; +import {Manifest} from '../common/types/manifest'; import {logger} from '../common/util/logger'; -import {load} from '../common/lib/load'; - -import {injectEnvironment} from '../if-run/lib/environment'; -import {initialize} from '../if-run/lib/initialize'; -import {aggregate} from '../if-run/lib/aggregate'; -import {compute} from '../if-run/lib/compute'; +import {executeCsv, getManifestData} from './util/helpers'; import {parseIfCsvArgs} from './util/args'; -import {executeCsv} from './util/helpers'; import {CsvOptions} from './types/csv'; const IfCsv = async () => { @@ -24,23 +18,18 @@ const IfCsv = async () => { const pipedManifest = await parseManifestFromStdin(); const {manifest, output, params} = await parseIfCsvArgs(); const resolvedManifest = manifest || pipedManifest; - let envManifest; + let manifestData: Manifest; if (resolvedManifest) { if (pipedManifest) { - envManifest = await YAML.load(pipedManifest!); + manifestData = (await YAML.load(pipedManifest!)) as Manifest; } else { - const {rawManifest} = await load(resolvedManifest); - envManifest = await injectEnvironment(rawManifest); + manifestData = await getManifestData(manifest!); } - const {tree, ...context} = validateManifest(envManifest); - const pluginStorage = await initialize(context.initialize.plugins); - const computedTree = await compute(tree, {context, pluginStorage}); - const aggregatedTree = aggregate(computedTree, context.aggregation); const options: CsvOptions = { - tree: aggregatedTree, - context, + tree: manifestData.tree, + context: manifestData, outputPath: output, params, }; From bcc872fa23fbc7aff8ed8ff9c43e4e22b927220c Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 10 Jul 2024 13:36:57 +0400 Subject: [PATCH 13/37] test(util): add tests for getManifestData function --- src/__tests__/if-csv/util/helpers.test.ts | 400 +++++++++++++--------- 1 file changed, 238 insertions(+), 162 deletions(-) diff --git a/src/__tests__/if-csv/util/helpers.test.ts b/src/__tests__/if-csv/util/helpers.test.ts index 61e6f1e9c..4712f5773 100644 --- a/src/__tests__/if-csv/util/helpers.test.ts +++ b/src/__tests__/if-csv/util/helpers.test.ts @@ -1,12 +1,13 @@ +import {jest} from '@jest/globals'; import * as fs from 'fs/promises'; - import {stringify} from 'csv-stringify/sync'; -import {jest} from '@jest/globals'; - -import {executeCsv} from '../../../if-csv/util/helpers'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {executeCsv, getManifestData} from '../../../if-csv/util/helpers'; import {CsvOptions} from '../../../if-csv/types/csv'; +const {ManifestValidationError} = ERRORS; + import { tree, context, @@ -15,182 +16,257 @@ import { aggregation, } from '../../../__mocks__/builtins/export-csv'; -jest.mock('fs/promises', () => ({ - writeFile: jest.fn<() => Promise>().mockResolvedValue(), -})); - -describe('executeCsv(): ', () => { - it('generates CSV file with correct data.', async () => { - const outputPath = 'output'; - const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; - const matrix = [ - columns, - ['tree.carbon', 8000.000004051243, 8000.000004051243], - ['tree.children.child-1.carbon', 4000.0000020256216, 4000.0000020256216], - ['tree.children.child-2.carbon', 4000.0000020256216, 4000.0000020256216], - ]; - const reformedContext = Object.assign({}, context, {outputs}); - const reformedTree = Object.assign({}, tree, { - children: { - ...tree.children, - 'child-1': { - ...tree.children['child-1'], - aggregated, - }, - 'child-2': { - ...tree.children['child-2'], - aggregated, +jest.mock('fs/promises', () => { + const originalModule = + jest.requireActual('fs/promises'); + return { + __esModule: true, + ...originalModule, + writeFile: jest.fn(() => Promise.resolve()), + }; +}); + +describe('if-csv/util/helpers: ', () => { + describe('executeCsv(): ', () => { + it('generates CSV file with correct data.', async () => { + const outputPath = 'output'; + const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; + const matrix = [ + columns, + ['tree.carbon', 8000.000004051243, 8000.000004051243], + [ + 'tree.children.child-1.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + [ + 'tree.children.child-2.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + ]; + const reformedContext = Object.assign({}, context, {outputs}); + const reformedTree = Object.assign({}, tree, { + children: { + ...tree.children, + 'child-1': { + ...tree.children['child-1'], + aggregated, + }, + 'child-2': { + ...tree.children['child-2'], + aggregated, + }, }, - }, - }); + }); - const options: CsvOptions = { - tree: reformedTree, - context: reformedContext, - outputPath, - params: 'carbon', - }; + const options: CsvOptions = { + tree: reformedTree, + context: reformedContext, + outputPath, + params: 'carbon', + }; - await executeCsv(options); + await executeCsv(options); - expect(fs.writeFile).toHaveBeenCalledWith( - 'output.csv', - stringify(matrix, {columns}) - ); - }); + expect(fs.writeFile).toHaveBeenCalledWith( + 'output.csv', + stringify(matrix, {columns}) + ); + }); - it('generates CSV file when the `outputs` type is missing.', async () => { - const outputPath = 'output'; - const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; - const matrix = [ - columns, - ['tree.carbon', 8000.000004051243, 8000.000004051243], - ['tree.children.child-1.carbon', 4000.0000020256216, 4000.0000020256216], - ['tree.children.child-2.carbon', 4000.0000020256216, 4000.0000020256216], - ]; - - const reformedTree = Object.assign({}, tree, { - children: { - ...tree.children, - 'child-1': { - ...tree.children['child-1'], - aggregated, - }, - 'child-2': { - ...tree.children['child-2'], - aggregated, + it('generates CSV file when the `outputs` type is missing.', async () => { + const outputPath = 'output'; + const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; + const matrix = [ + columns, + ['tree.carbon', 8000.000004051243, 8000.000004051243], + [ + 'tree.children.child-1.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + [ + 'tree.children.child-2.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + ]; + + const reformedTree = Object.assign({}, tree, { + children: { + ...tree.children, + 'child-1': { + ...tree.children['child-1'], + aggregated, + }, + 'child-2': { + ...tree.children['child-2'], + aggregated, + }, }, - }, - }); + }); - const options: CsvOptions = { - tree: reformedTree, - context, - outputPath, - params: 'carbon', - }; + const options: CsvOptions = { + tree: reformedTree, + context, + outputPath, + params: 'carbon', + }; - await executeCsv(options); + await executeCsv(options); - expect.assertions(1); + expect.assertions(1); - expect(fs.writeFile).toHaveBeenCalledWith( - 'output.csv', - stringify(matrix, {columns}) - ); - }); + expect(fs.writeFile).toHaveBeenCalledWith( + 'output.csv', + stringify(matrix, {columns}) + ); + }); - it('generates CSV file when `aggregation` persists.', async () => { - const outputPath = 'output'; - const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; - const matrix = [ - columns, - ['tree.carbon', 8000.000004051243, 8000.000004051243], - ['tree.children.child-1.carbon', 4000.0000020256216, 4000.0000020256216], - ['tree.children.child-2.carbon', 4000.0000020256216, 4000.0000020256216], - ]; - - const reformedContext = Object.assign( - {}, - context, - {outputs}, - {aggregation} - ); - const reformedTree = Object.assign({}, tree, { - children: { - ...tree.children, - 'child-1': { - ...tree.children['child-1'], - aggregated, - }, - 'child-2': { - ...tree.children['child-2'], - aggregated, + it('generates CSV file when `aggregation` persists.', async () => { + const outputPath = 'output'; + const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; + const matrix = [ + columns, + ['tree.carbon', 8000.000004051243, 8000.000004051243], + [ + 'tree.children.child-1.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + [ + 'tree.children.child-2.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + ]; + + const reformedContext = Object.assign( + {}, + context, + {outputs}, + {aggregation} + ); + const reformedTree = Object.assign({}, tree, { + children: { + ...tree.children, + 'child-1': { + ...tree.children['child-1'], + aggregated, + }, + 'child-2': { + ...tree.children['child-2'], + aggregated, + }, }, - }, + }); + + const options: CsvOptions = { + tree: reformedTree, + context: reformedContext, + outputPath, + params: 'carbon', + }; + + await executeCsv(options); + + expect.assertions(1); + expect(fs.writeFile).toHaveBeenCalledWith( + 'output.csv', + stringify(matrix, {columns}) + ); }); - const options: CsvOptions = { - tree: reformedTree, - context: reformedContext, - outputPath, - params: 'carbon', - }; + it('returns string when `outputPath` is not provided', async () => { + const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; + const matrix = [ + columns, + ['tree.carbon', 8000.000004051243, 8000.000004051243], + [ + 'tree.children.child-1.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + ]; - await executeCsv(options); + const options: CsvOptions = { + tree, + context, + outputPath: undefined, + params: 'carbon', + }; - expect.assertions(1); - expect(fs.writeFile).toHaveBeenCalledWith( - 'output.csv', - stringify(matrix, {columns}) - ); - }); + const result = await executeCsv(options); + + expect.assertions(1); + expect(result).toEqual(stringify(matrix, {columns})); + }); + + it('generates CSV file when `aggregation` is missing.', async () => { + const outputPath = 'output'; + const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; + const matrix = [ + columns, + ['tree.carbon', 8000.000004051243, 8000.000004051243], + [ + 'tree.children.child-1.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + [ + 'tree.children.child-2.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + ]; - it('returns string when `outputPath` is not provided', async () => { - const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; - const matrix = [ - columns, - ['tree.carbon', 8000.000004051243, 8000.000004051243], - ['tree.children.child-1.carbon', 4000.0000020256216, 4000.0000020256216], - ]; - - const options: CsvOptions = { - tree, - context, - outputPath: undefined, - params: 'carbon', - }; - - const result = await executeCsv(options); - - expect.assertions(1); - expect(result).toEqual(stringify(matrix, {columns})); + const reformedContext = Object.assign({}, context, {outputs}); + const options: CsvOptions = { + tree, + context: reformedContext, + outputPath, + params: 'carbon', + }; + + await executeCsv(options); + + expect.assertions(1); + expect(fs.writeFile).toHaveBeenCalledWith( + 'output.csv', + stringify(matrix, {columns}) + ); + }); }); - it('generates CSV file when `aggregation` is missing.', async () => { - const outputPath = 'output'; - const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; - const matrix = [ - columns, - ['tree.carbon', 8000.000004051243, 8000.000004051243], - ['tree.children.child-1.carbon', 4000.0000020256216, 4000.0000020256216], - ['tree.children.child-2.carbon', 4000.0000020256216, 4000.0000020256216], - ]; - - const reformedContext = Object.assign({}, context, {outputs}); - const options: CsvOptions = { - tree, - context: reformedContext, - outputPath, - params: 'carbon', - }; - - await executeCsv(options); - - expect.assertions(1); - expect(fs.writeFile).toHaveBeenCalledWith( - 'output.csv', - stringify(matrix, {columns}) - ); + describe('getManifestData(): ', () => { + it('returns correct manifest data.', async () => { + process.env.MOCK_MANIFEST = 'true'; + const mockManifestPath = './src/__mocks__/mock-manifest.yaml'; + const result = await getManifestData(mockManifestPath); + + expect.assertions(2); + expect(result).toHaveProperty('description'); + expect(result).toHaveProperty('tree'); + }); + + it('throws an error when there is not `outputs` in the manifest.', async () => { + process.env.MOCK_MANIFEST = 'false'; + const mockManifestPath = ''; + const load = require('../../../common/lib/load'); + const spyLoad = jest.spyOn(load, 'load'); + + spyLoad.mockReturnValue({ + rawManifest: {tree: {children: {child: {}}}}, + }); + expect.assertions(1); + try { + await getManifestData(mockManifestPath); + } catch (error) { + expect(error).toEqual( + new ManifestValidationError('Manifest outputs are not available!') + ); + } + }); }); }); From 752c014fb454580871a178679efe325bdf53740b Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 10 Jul 2024 13:43:26 +0400 Subject: [PATCH 14/37] fix(package): add `if-csv` into package-loc.json --- package-lock.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package-lock.json b/package-lock.json index dd24c9128..f01b0fb13 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,7 @@ }, "bin": { "if-check": "build/if-check/index.js", + "if-csv": "build/if-csv/index.js", "if-diff": "build/if-diff/index.js", "if-env": "build/if-env/index.js", "if-run": "build/if-run/index.js" From 96f086c53dfa00831febeba7e7c27e421295acda Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 10 Jul 2024 13:57:39 +0400 Subject: [PATCH 15/37] fix(util): improve getManifestData function --- src/if-csv/util/helpers.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/if-csv/util/helpers.ts b/src/if-csv/util/helpers.ts index d9f7ec09d..8ae6dd0e5 100644 --- a/src/if-csv/util/helpers.ts +++ b/src/if-csv/util/helpers.ts @@ -15,11 +15,11 @@ const {ManifestValidationError} = ERRORS; * Gets the folder path of the manifest file, dependencies from manifest file and install argument from the given arguments. */ export const getManifestData = async (manifest: string) => { - const loadedManifest = await load(manifest); - const children = loadedManifest.rawManifest.tree.children; + const {rawManifest} = await load(manifest); + const children = rawManifest.tree.children; if ((children.child || children['child-0']).outputs) { - return loadedManifest.rawManifest; + return rawManifest; } throw new ManifestValidationError(FAILURE_MESSAGE_OUTPUTS); From 9d8d1fd8f906c325ea1c401a4b909ab97f153063 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 10 Jul 2024 14:03:26 +0400 Subject: [PATCH 16/37] fix(util): improve getManifestData function --- src/if-csv/util/helpers.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/if-csv/util/helpers.ts b/src/if-csv/util/helpers.ts index 8ae6dd0e5..397293dd8 100644 --- a/src/if-csv/util/helpers.ts +++ b/src/if-csv/util/helpers.ts @@ -12,13 +12,13 @@ const {FAILURE_MESSAGE_OUTPUTS} = STRINGS; const {ManifestValidationError} = ERRORS; /** - * Gets the folder path of the manifest file, dependencies from manifest file and install argument from the given arguments. + * Gets the manifest data if `outputs` is present in it. */ export const getManifestData = async (manifest: string) => { const {rawManifest} = await load(manifest); - const children = rawManifest.tree.children; + const {children} = rawManifest.tree; - if ((children.child || children['child-0']).outputs) { + if (children?.child?.outputs || children?.['child-0']?.outputs) { return rawManifest; } From 179238cd0eeb07b3cb23d28a202b4eff667e1749 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 10 Jul 2024 17:20:15 +0400 Subject: [PATCH 17/37] test(util): fix tests conflict after merging with the main --- src/__tests__/if-csv/util/helpers.test.ts | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/__tests__/if-csv/util/helpers.test.ts b/src/__tests__/if-csv/util/helpers.test.ts index 4712f5773..1859f4850 100644 --- a/src/__tests__/if-csv/util/helpers.test.ts +++ b/src/__tests__/if-csv/util/helpers.test.ts @@ -11,10 +11,9 @@ const {ManifestValidationError} = ERRORS; import { tree, context, - outputs, aggregated, aggregation, -} from '../../../__mocks__/builtins/export-csv'; +} from '../../../__mocks__/builtins/export-yaml'; jest.mock('fs/promises', () => { const originalModule = @@ -45,7 +44,6 @@ describe('if-csv/util/helpers: ', () => { 4000.0000020256216, ], ]; - const reformedContext = Object.assign({}, context, {outputs}); const reformedTree = Object.assign({}, tree, { children: { ...tree.children, @@ -62,7 +60,7 @@ describe('if-csv/util/helpers: ', () => { const options: CsvOptions = { tree: reformedTree, - context: reformedContext, + context, outputPath, params: 'carbon', }; @@ -142,12 +140,7 @@ describe('if-csv/util/helpers: ', () => { ], ]; - const reformedContext = Object.assign( - {}, - context, - {outputs}, - {aggregation} - ); + const reformedContext = Object.assign({}, context, {aggregation}); const reformedTree = Object.assign({}, tree, { children: { ...tree.children, @@ -221,10 +214,9 @@ describe('if-csv/util/helpers: ', () => { ], ]; - const reformedContext = Object.assign({}, context, {outputs}); const options: CsvOptions = { tree, - context: reformedContext, + context, outputPath, params: 'carbon', }; From e66b168688e87153326d1db0be2be907a39a6c7a Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 16:50:12 +0400 Subject: [PATCH 18/37] fix(util): remove `params` and update `aggregation` in the manifest schema --- src/common/util/validations.ts | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/common/util/validations.ts b/src/common/util/validations.ts index a4f0fcb5c..76b785d6d 100644 --- a/src/common/util/validations.ts +++ b/src/common/util/validations.ts @@ -3,8 +3,10 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {STRINGS} from '../../if-run/config'; -import {AGGREGATION_METHODS} from '../../if-run/types/aggregation'; -import {AGGREGATION_TYPES} from '../../if-run/types/parameters'; +import { + AGGREGATION_METHODS, + AGGREGATION_TYPES, +} from '../../if-run/types/aggregation'; const {ManifestValidationError, InputValidationError} = ERRORS; const {VALIDATING_MANIFEST} = STRINGS; @@ -38,22 +40,15 @@ export const manifestSchema = z.object({ .nullable(), aggregation: z .object({ - metrics: z.array(z.string()), - type: z.enum(AGGREGATION_METHODS), + metrics: z.record( + z.object({ + method: z.enum(AGGREGATION_METHODS), + }) + ), + type: z.enum(AGGREGATION_TYPES), }) .optional() .nullable(), - params: z - .array( - z.object({ - name: z.string(), - description: z.string(), - aggregation: z.enum(AGGREGATION_TYPES), - unit: z.string(), - }) - ) - .optional() - .nullable(), initialize: z.object({ plugins: z.record( z.string(), From 2cc439846da0e10182854e743bbe98f52bf5e614 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 16:54:14 +0400 Subject: [PATCH 19/37] fix(src): remove all files realted to params --- src/__mocks__/json.ts | 14 -- src/__tests__/if-run/lib/parameterize.test.ts | 114 ---------- src/__tests__/if-run/util/json.test.ts | 26 --- src/if-run/config/params.ts | 198 ------------------ src/if-run/lib/parameterize.ts | 79 ------- src/if-run/types/parameters.ts | 7 - src/if-run/util/json.ts | 10 - 7 files changed, 448 deletions(-) delete mode 100644 src/__mocks__/json.ts delete mode 100644 src/__tests__/if-run/lib/parameterize.test.ts delete mode 100644 src/__tests__/if-run/util/json.test.ts delete mode 100644 src/if-run/config/params.ts delete mode 100644 src/if-run/lib/parameterize.ts delete mode 100644 src/if-run/types/parameters.ts delete mode 100644 src/if-run/util/json.ts diff --git a/src/__mocks__/json.ts b/src/__mocks__/json.ts deleted file mode 100644 index d6bb24d2c..000000000 --- a/src/__mocks__/json.ts +++ /dev/null @@ -1,14 +0,0 @@ -export const readAndParseJson = async () => { - return { - 'mock-carbon': { - description: 'an amount of carbon emitted into the atmosphere', - unit: 'gCO2e', - aggregation: 'sum', - }, - 'mock-cpu': { - description: 'number of cores available', - unit: 'cores', - aggregation: 'none', - }, - }; -}; diff --git a/src/__tests__/if-run/lib/parameterize.test.ts b/src/__tests__/if-run/lib/parameterize.test.ts deleted file mode 100644 index ed3a4b948..000000000 --- a/src/__tests__/if-run/lib/parameterize.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import {LeveledLogMethod} from 'winston'; - -const mockLog = jest.fn((message: string) => message); - -jest.mock('../../../if-run/util/log-memoize', () => ({ - memoizedLog: mockLog, -})); -jest.mock('../../../common/util/logger', () => ({ - logger: { - warn: mockLog, - debug: mockLog, - }, -})); - -import {PARAMETERS} from '../../../if-run/config'; -import {parameterize} from '../../../if-run/lib/parameterize'; - -import {STRINGS} from '../../../if-run/config'; - -import {ManifestParameter} from '../../../common/types/manifest'; - -const {REJECTING_OVERRIDE, CHECKING_AGGREGATION_METHOD} = STRINGS; - -describe('lib/parameterize: ', () => { - afterEach(() => { - (mockLog as jest.Mock).mockReset(); - }); - - describe('getAggregationMethod(): ', () => { - it('returns method for average aggregation method metric.', () => { - const metric = 'cpu/utilization'; - const method = parameterize.getAggregationMethod(metric); - - const expectedMethod = 'avg'; - - expect(method).toEqual(expectedMethod); - }); - - it('returns method for unknown aggregation method metric.', () => { - const metric = 'mock/metric'; - const method = parameterize.getAggregationMethod(metric); - - const expectedMethod = 'sum'; - - expect(method).toEqual(expectedMethod); - expect(mockLog as unknown as LeveledLogMethod).toHaveBeenCalledTimes(2); - }); - - it('prints debug log for first input.', () => { - const unitName = 'timestamp'; - - parameterize.getAggregationMethod(unitName); - - expect(mockLog as typeof console.debug).toHaveBeenCalledWith( - console.debug, - CHECKING_AGGREGATION_METHOD(unitName) - ); - }); - }); - - describe('combine(): ', () => { - it('checks if return type is undefined.', () => { - const params = {}; - const response = parameterize.combine(null, params); - - expect(response).toBeUndefined(); - }); - - it('checks if uninitialized custom param is requested, then returns fallback `sum` method.', () => { - const name = 'mock-name'; - const method = parameterize.getAggregationMethod(name); - - const expectedMethodName = 'sum'; - expect(method).toEqual(expectedMethodName); - }); - - it('checks if custom params are inserted successfully.', () => { - const params = [ - { - name: 'mock-name', - description: 'mock-description', - unit: 'mock/sq', - aggregation: 'none', - }, - ] as ManifestParameter[]; - const object = {}; - - parameterize.combine(params, object); - const method = parameterize.getAggregationMethod(params[0].name); - - expect(method).toEqual(params[0].aggregation); - }); - - it('rejects on default param override.', () => { - const params = [ - { - name: 'carbon', - description: 'mock-description', - unit: 'mock/co', - aggregation: 'none', - }, - ] as ManifestParameter[]; - - parameterize.combine(params, PARAMETERS); - const method = parameterize.getAggregationMethod(params[0].name); - - const expectedMethodName = 'sum'; - const expectedMessage = REJECTING_OVERRIDE(params[0]); - - expect(method).toEqual(expectedMethodName); - expect(mockLog).toHaveBeenCalledWith(expectedMessage); - }); - }); -}); diff --git a/src/__tests__/if-run/util/json.test.ts b/src/__tests__/if-run/util/json.test.ts deleted file mode 100644 index 5836c677a..000000000 --- a/src/__tests__/if-run/util/json.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -jest.mock('fs/promises', () => require('../../../__mocks__/fs')); - -import {readAndParseJson} from '../../../if-run/util/json'; - -describe('util/json: ', () => { - describe('readAndParseJson(): ', () => { - it('returns file content from path.', async () => { - const path = 'mock/path/json'; - expect.assertions(1); - - const response = await readAndParseJson(path); - expect(response).toEqual(path); - }); - - it('throws error if path does not exist.', async () => { - const path = 'mock/path/json-reject'; - expect.assertions(1); - - try { - await readAndParseJson(path); - } catch (error) { - expect(error).toBeInstanceOf(Error); - } - }); - }); -}); diff --git a/src/if-run/config/params.ts b/src/if-run/config/params.ts deleted file mode 100644 index 76732e57b..000000000 --- a/src/if-run/config/params.ts +++ /dev/null @@ -1,198 +0,0 @@ -import {Parameters} from '../types/parameters'; - -export const PARAMETERS: Parameters = { - carbon: { - description: 'an amount of carbon emitted into the atmosphere', - unit: 'gCO2e', - aggregation: 'sum', - }, - 'cpu/number-cores': { - description: 'number of cores available', - unit: 'cores', - aggregation: 'none', - }, - 'cpu/utilization': { - description: 'refers to CPU utilization.', - unit: 'percentage', - aggregation: 'avg', - }, - 'disk-io': { - description: 'refers to GB of data written/read from disk', - unit: 'GB', - aggregation: 'sum', - }, - duration: { - description: 'refers to the duration of the input', - unit: 'seconds', - aggregation: 'sum', - }, - energy: { - description: 'amount of energy utilised by the component', - unit: 'kWh', - aggregation: 'sum', - }, - 'cpu/energy': { - description: 'Energy consumed by the CPU of the component', - unit: 'kWh', - aggregation: 'sum', - }, - 'device/expected-lifespan': { - description: 'Total Expected Lifespan of the Component in Seconds', - unit: 'seconds', - aggregation: 'sum', - }, - 'memory/energy': { - description: 'Energy consumed by the Memory of the component', - unit: 'kWh', - aggregation: 'sum', - }, - 'carbon-embodied': { - description: 'Embodied Emissions of the component', - unit: 'gCO2e', - aggregation: 'sum', - }, - 'network/energy': { - description: 'Energy consumed by the Network of the component', - unit: 'kWh', - aggregation: 'sum', - }, - 'functional-unit': { - description: - 'the name of the functional unit in which the final SCI value should be expressed, e.g. requests, users', - unit: 'none', - aggregation: 'sum', - }, - 'gpu-util': { - description: 'refers to CPU utilization.', - unit: 'percentage', - aggregation: 'avg', - }, - 'grid/carbon-intensity': { - description: 'Carbon intensity for the grid', - unit: 'gCO2eq/kWh', - aggregation: 'avg', - }, - 'cloud/instance-type': { - description: 'Type of Cloud Instance name used in the cloud provider APIs', - unit: 'None', - aggregation: 'none', - }, - geolocation: { - description: - 'Geographic location of provider as string (for watt-time model it is provided as latitude and longitude, comma separated, in decimal degrees)', - unit: 'None (decimal degrees for watt-time model)', - aggregation: 'none', - }, - 'carbon-operational': { - description: 'Operational Emissions of the component', - unit: 'gCO2e', - aggregation: 'sum', - }, - 'cpu/name': { - description: 'Name of the physical processor', - unit: 'None', - aggregation: 'none', - }, - 'cloud/region': { - description: 'region cloud instance runs in', - unit: 'None', - aggregation: 'none', - }, - 'cloud/vendor': { - description: - 'Name of the cloud service provider in the ccf model. Can be aws, gcp or azure', - unit: 'None', - aggregation: 'none', - }, - name: { - description: 'arbitrary name parameter.', - unit: 'None', - aggregation: 'none', - }, - 'ram-alloc': { - description: 'refers to GB of memory allocated.', - unit: 'GB', - aggregation: 'avg', - }, - 'ram-util': { - description: 'refers to percentage of memory utilized.', - unit: 'percentage', - aggregation: 'avg', - }, - 'resources-reserved': { - description: 'resources reserved for an application', - unit: 'count', - aggregation: 'none', - }, - 'cpu/thermal-design-power': { - description: 'thermal design power for a processor', - unit: 'kwh', - aggregation: 'avg', - }, - 'device/emissions-embodied': { - description: 'total embodied emissions of some component', - unit: 'gCO2e', - aggregation: 'sum', - }, - timestamp: { - description: 'refers to the time of occurrence of the input', - unit: 'RFC3339', - aggregation: 'none', - }, - 'time-reserved': { - description: 'time reserved for a component', - unit: 'seconds', - aggregation: 'avg', - }, - 'resources-total': { - description: 'total resources available', - unit: 'count', - aggregation: 'none', - }, - 'vcpus-allocated': { - description: 'number of vcpus allocated to particular resource', - unit: 'count', - aggregation: 'none', - }, - 'vcpus-total': { - description: 'total number of vcpus available on a particular resource', - unit: 'count', - aggregation: 'none', - }, - 'memory-available': { - description: 'total amount of memory available on a particular resource', - unit: 'GB', - aggregation: 'none', - }, - 'physical-processor': { - description: - 'name of the physical processor being used in a specific instance type', - unit: 'None', - aggregation: 'none', - }, - 'cloud/region-cfe': { - description: 'cloud region name in cfe format', - unit: 'None', - aggregation: 'none', - }, - 'cloud/region-em-zone-id': { - description: 'cloud region name in electricity maps format', - unit: 'None', - aggregation: 'none', - }, - 'cloud/region-wt-id': { - description: 'cloud region name in watt-time format', - unit: 'None', - aggregation: 'none', - }, - 'cloud/region-location': { - description: 'cloud region name in our IF format', - unit: 'None', - aggregation: 'none', - }, - 'cloud/region-geolocation': { - description: 'location expressed as decimal coordinates (lat/lon)', - unit: 'decimal degrees', - aggregation: 'none', - }, -}; diff --git a/src/if-run/lib/parameterize.ts b/src/if-run/lib/parameterize.ts deleted file mode 100644 index 1034e1ce2..000000000 --- a/src/if-run/lib/parameterize.ts +++ /dev/null @@ -1,79 +0,0 @@ -import {debugLogger} from '../../common/util/debug-logger'; -import {logger} from '../../common/util/logger'; -import {memoizedLog} from '../util/log-memoize'; - -import {STRINGS, PARAMETERS} from '../config'; - -import {Parameters} from '../types/parameters'; -import {ManifestParameter} from '../../common/types/manifest'; - -const { - REJECTING_OVERRIDE, - UNKNOWN_PARAM, - SYNCING_PARAMETERS, - CHECKING_AGGREGATION_METHOD, -} = STRINGS; - -/** - * Parameters manager. Provides get aggregation method and combine functionality. - */ -const Parameterize = () => { - let parametersStorage = PARAMETERS; - - /** - * Returns aggregation method for given `unitName`. If doesn't exist then returns value `sum`. - */ - const getAggregationMethod = (unitName: string) => { - debugLogger.setExecutingPluginName(); - memoizedLog(console.debug, CHECKING_AGGREGATION_METHOD(unitName)); - - if (`${unitName}` in parametersStorage) { - return parametersStorage[unitName as keyof typeof PARAMETERS].aggregation; - } - - memoizedLog(logger.warn, UNKNOWN_PARAM(unitName)); - - return 'sum'; - }; - - /** - * Checks if additional parameters are provided in context. - * If so, then checks if they are coincident with default ones and exits with warning message. - * Otherwise appends context based parameters to defaults. - */ - const combine = ( - contextParameters: ManifestParameter[] | null | undefined, - parameters: Parameters - ) => { - console.debug(SYNCING_PARAMETERS); - - if (contextParameters) { - contextParameters.forEach(param => { - if (`${param.name}` in parameters) { - logger.warn(REJECTING_OVERRIDE(param)); - - return; - } - - const {description, unit, aggregation, name} = param; - - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - parameters[name] = { - description, - unit, - aggregation, - }; - }); - } - - parametersStorage = parameters; - }; - - return { - combine, - getAggregationMethod, - }; -}; - -export const parameterize = Parameterize(); diff --git a/src/if-run/types/parameters.ts b/src/if-run/types/parameters.ts deleted file mode 100644 index 939ac2ba6..000000000 --- a/src/if-run/types/parameters.ts +++ /dev/null @@ -1,7 +0,0 @@ -import {ManifestParameter} from '../../common/types/manifest'; - -export const AGGREGATION_TYPES = ['sum', 'none', 'avg'] as const; - -type ParameterProps = Omit; - -export type Parameters = Record; diff --git a/src/if-run/util/json.ts b/src/if-run/util/json.ts deleted file mode 100644 index c9180a788..000000000 --- a/src/if-run/util/json.ts +++ /dev/null @@ -1,10 +0,0 @@ -import * as fs from 'fs/promises'; - -/** - * Reads and parses json file. - */ -export const readAndParseJson = async (paramPath: string): Promise => { - const file = await fs.readFile(paramPath, 'utf-8'); - - return JSON.parse(file) as T; -}; From c17a8f2d29dd7786a93034993de2a44e19ec2562 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 16:57:02 +0400 Subject: [PATCH 20/37] fix(src): remove `override-params` from config and from `IfRunArgs` interface --- src/if-run/config/config.ts | 6 ------ src/if-run/types/process-args.ts | 1 - 2 files changed, 7 deletions(-) diff --git a/src/if-run/config/config.ts b/src/if-run/config/config.ts index 99ca34193..91f366543 100644 --- a/src/if-run/config/config.ts +++ b/src/if-run/config/config.ts @@ -21,12 +21,6 @@ export const CONFIG = { alias: 'o', description: '[path to the output file]', }, - 'override-params': { - type: String, - optional: true, - alias: 'p', - description: '[path to a parameter file that overrides our defaults]', - }, 'no-output': { type: Boolean, optional: true, diff --git a/src/if-run/types/process-args.ts b/src/if-run/types/process-args.ts index a336b0af0..184877bd0 100644 --- a/src/if-run/types/process-args.ts +++ b/src/if-run/types/process-args.ts @@ -1,7 +1,6 @@ export interface IfRunArgs { manifest?: string; output?: string; - 'override-params'?: string; 'no-output'?: boolean; debug?: boolean; } From 793ee6d3fd43f2409a5ffd13bdbadf368f7cfea1 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 18:09:06 +0400 Subject: [PATCH 21/37] fix(src): remove redundant types and strings --- src/common/types/manifest.ts | 2 -- src/if-run/config/index.ts | 1 - src/if-run/config/strings.ts | 5 ----- 3 files changed, 8 deletions(-) diff --git a/src/common/types/manifest.ts b/src/common/types/manifest.ts index 2359a3433..1b003be2c 100644 --- a/src/common/types/manifest.ts +++ b/src/common/types/manifest.ts @@ -14,5 +14,3 @@ export type AggregationParamsSure = Extract; export type Context = Omit; export type ContextWithExec = Omit; - -export type ManifestParameter = Extract[number]; diff --git a/src/if-run/config/index.ts b/src/if-run/config/index.ts index 6aa6e0a98..4972b390b 100644 --- a/src/if-run/config/index.ts +++ b/src/if-run/config/index.ts @@ -1,3 +1,2 @@ export {CONFIG} from './config'; -export {PARAMETERS} from './params'; export {STRINGS} from './strings'; diff --git a/src/if-run/config/strings.ts b/src/if-run/config/strings.ts index 5f5387c1f..0a9c1a455 100644 --- a/src/if-run/config/strings.ts +++ b/src/if-run/config/strings.ts @@ -1,5 +1,3 @@ -import {ManifestParameter} from '../../common/types/manifest'; - export const STRINGS = { MISSING_METHOD: "Initalization param 'method' is missing.", MISSING_PATH: "Initalization param 'path' is missing.", @@ -28,8 +26,6 @@ export const STRINGS = { METRIC_MISSING: (metric: string, index: number) => `Aggregation metric ${metric} is not found in inputs[${index}].`, INVALID_GROUP_BY: (type: string) => `Invalid group ${type}.`, - REJECTING_OVERRIDE: (param: ManifestParameter) => - `Rejecting overriding of canonical parameter: ${param.name}.`, INVALID_EXHAUST_PLUGIN: (pluginName: string) => `Invalid exhaust plugin: ${pluginName}.`, UNKNOWN_PARAM: (name: string) => @@ -48,7 +44,6 @@ Note that for the '--output' option you also need to define the output type in y LOADING_MANIFEST: 'Loading manifest', VALIDATING_MANIFEST: 'Validating manifest', CAPTURING_RUNTIME_ENVIRONMENT_DATA: 'Capturing runtime environment data', - SYNCING_PARAMETERS: 'Syncing parameters', CHECKING_AGGREGATION_METHOD: (unitName: string) => `Checking aggregation method for ${unitName}`, INITIALIZING_PLUGINS: 'Initializing plugins', From 99e433611fa8f366b26ffd21126eb1c859e82072 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 18:13:26 +0400 Subject: [PATCH 22/37] fix(util): remove `override-params` from the args.ts --- src/if-run/util/args.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/if-run/util/args.ts b/src/if-run/util/args.ts index 632a17fd6..140c5d140 100644 --- a/src/if-run/util/args.ts +++ b/src/if-run/util/args.ts @@ -33,7 +33,7 @@ const validateAndParseProcessArgs = () => { }; /** - * 1. Parses process arguments like `manifest`, `output`, `override-params`, `help` and `debug`. + * 1. Parses process arguments like `manifest`, `output`, `help` and `debug`. * 2. Checks if `help` param is provided, then logs help message and exits. * 3. If output params are missing, warns user about it. * 3. Otherwise checks if `manifest` param is there, then processes with checking if it's a yaml file. @@ -44,7 +44,6 @@ export const parseIfRunProcessArgs = (): ProcessArgsOutputs => { const { manifest, output, - 'override-params': overrideParams, 'no-output': noOutput, debug, } = validateAndParseProcessArgs(); @@ -61,7 +60,6 @@ export const parseIfRunProcessArgs = (): ProcessArgsOutputs => { ...(output && {outputPath: prependFullFilePath(output)}), ...(noOutput && {noOutput}), }, - ...(overrideParams && {paramPath: overrideParams}), debug, }; } From 25be4913c82505d8c3b4a8ecef83a24125c82108 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 18:15:53 +0400 Subject: [PATCH 23/37] feat(types): update aggregation types --- src/if-run/types/aggregation.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/if-run/types/aggregation.ts b/src/if-run/types/aggregation.ts index d15322d29..a7e895da4 100644 --- a/src/if-run/types/aggregation.ts +++ b/src/if-run/types/aggregation.ts @@ -1,3 +1,9 @@ export type AggregationResult = Record; -export const AGGREGATION_METHODS = ['horizontal', 'vertical', 'both'] as const; +export const AGGREGATION_TYPES = ['horizontal', 'vertical', 'both'] as const; +export const AGGREGATION_METHODS = ['sum', 'avg', 'none'] as const; + +export type AggregationMetric = Record< + string, + {method: 'sum' | 'avg' | 'none'} +>; From 9e484b29b0e895eaf12f29a49a72696cf99db897 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 18:18:05 +0400 Subject: [PATCH 24/37] feat(util): update metrics type --- src/if-run/util/aggregation-helper.ts | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/if-run/util/aggregation-helper.ts b/src/if-run/util/aggregation-helper.ts index 408a4e9f7..87d3e15fc 100644 --- a/src/if-run/util/aggregation-helper.ts +++ b/src/if-run/util/aggregation-helper.ts @@ -1,11 +1,11 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {PluginParams} from '@grnsft/if-core/types'; -import {parameterize} from '../lib/parameterize'; - import {CONFIG, STRINGS} from '../config'; -import {AggregationResult} from '../types/aggregation'; +import {AggregationMetric, AggregationResult} from '../types/aggregation'; + +import {getAggregationMethod} from '../lib/aggregate'; const {InvalidAggregationMethodError, MissingAggregationParamError} = ERRORS; const {INVALID_AGGREGATION_METHOD, METRIC_MISSING} = STRINGS; @@ -15,9 +15,9 @@ const {AGGREGATION_ADDITIONAL_PARAMS} = CONFIG; * Validates metrics array before applying aggregator. * If aggregation method is `none`, then throws error. */ -const checkIfMetricsAreValid = (metrics: string[]) => { - metrics.forEach(metric => { - const method = parameterize.getAggregationMethod(metric); +const checkIfMetricsAreValid = (metrics: AggregationMetric) => { + Object.keys(metrics).forEach(metric => { + const method = metrics[metric].method; if (method === 'none') { throw new InvalidAggregationMethodError( @@ -33,11 +33,14 @@ const checkIfMetricsAreValid = (metrics: string[]) => { */ export const aggregateInputsIntoOne = ( inputs: PluginParams[], - metrics: string[], + metrics: AggregationMetric, isTemporal?: boolean ) => { checkIfMetricsAreValid(metrics); - const extendedMetrics = [...metrics, ...AGGREGATION_ADDITIONAL_PARAMS]; + const extendedMetrics = [ + ...Object.keys(metrics), + ...AGGREGATION_ADDITIONAL_PARAMS, + ]; return inputs.reduce((acc, input, index) => { for (const metric of extendedMetrics) { @@ -56,7 +59,7 @@ export const aggregateInputsIntoOne = ( /** Checks for the last iteration. */ if (index === inputs.length - 1) { - if (parameterize.getAggregationMethod(metric) === 'avg') { + if (getAggregationMethod(metric) === 'avg') { acc[metric] /= inputs.length; } } From 1aca2939259ec488cc7f437baa61e3b5a7925305 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 18:21:20 +0400 Subject: [PATCH 25/37] feat(src): add temp workaround to solve issue related to timeSync --- src/if-run/index.ts | 12 ++++--- src/if-run/lib/aggregate.ts | 70 ++++++++++++++++++++++++++++++++++--- 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/src/if-run/index.ts b/src/if-run/index.ts index fa5d17fa2..5ffb62ded 100644 --- a/src/if-run/index.ts +++ b/src/if-run/index.ts @@ -1,10 +1,9 @@ #!/usr/bin/env node -import {aggregate} from './lib/aggregate'; +import {aggregate, storeAggregateMetrics} from './lib/aggregate'; import {compute} from './lib/compute'; import {injectEnvironment} from './lib/environment'; import {exhaust} from './lib/exhaust'; import {initialize} from './lib/initialize'; -import {parameterize} from './lib/parameterize'; import {load} from '../common/lib/load'; import {parseIfRunProcessArgs} from './util/args'; @@ -21,19 +20,22 @@ const {DISCLAIMER_MESSAGE} = COMMON_STRINGS; const impactEngine = async () => { const options = parseIfRunProcessArgs(); - const {inputPath, paramPath, outputOptions, debug} = options; + const {inputPath, outputOptions, debug} = options; debugLogger.overrideConsoleMethods(!!debug); logger.info(DISCLAIMER_MESSAGE); console.info(STARTING_IF); - const {rawManifest, parameters} = await load(inputPath, paramPath); + const {rawManifest} = await load(inputPath); const envManifest = await injectEnvironment(rawManifest); try { const {tree, ...context} = validateManifest(envManifest); - parameterize.combine(context.params, parameters); + + // TODO: remove this after resolving timeSync to be a builtin functionality. + storeAggregateMetrics(context.aggregation); + const pluginStorage = await initialize(context.initialize.plugins); const computedTree = await compute(tree, {context, pluginStorage}); const aggregatedTree = aggregate(computedTree, context.aggregation); diff --git a/src/if-run/lib/aggregate.ts b/src/if-run/lib/aggregate.ts index 4c16a0d0d..7ca0d3f42 100644 --- a/src/if-run/lib/aggregate.ts +++ b/src/if-run/lib/aggregate.ts @@ -1,14 +1,25 @@ import {PluginParams} from '@grnsft/if-core/types'; -import {aggregateInputsIntoOne} from '../util/aggregation-helper'; - -import {STRINGS} from '../config/strings'; +import {debugLogger} from '../../common/util/debug-logger'; +import {logger} from '../../common/util/logger'; import { AggregationParams, AggregationParamsSure, } from '../../common/types/manifest'; -const {AGGREGATING_NODE, AGGREGATING_OUTPUTS} = STRINGS; +import {aggregateInputsIntoOne} from '../util/aggregation-helper'; +import {memoizedLog} from '../util/log-memoize'; + +import {AggregationMetric} from '../types/aggregation'; + +import {STRINGS} from '../config/strings'; + +const { + AGGREGATING_NODE, + AGGREGATING_OUTPUTS, + UNKNOWN_PARAM, + CHECKING_AGGREGATION_METHOD, +} = STRINGS; /** * Gets `i`th element from all children outputs and collects them in single array. @@ -27,7 +38,7 @@ const getIthElementsFromChildren = (children: any, i: number) => { * 1. Gets the i'th element from each childrens outputs (treating children as rows and we are after a column of data). * 2. Now we just aggregate over the `ithSliceOfOutputs` the same as we did for the normal outputs. */ -const temporalAggregation = (node: any, metrics: string[]) => { +const temporalAggregation = (node: any, metrics: AggregationMetric) => { const outputs: PluginParams[] = []; const values: any = Object.values(node.children); @@ -91,3 +102,52 @@ export const aggregate = (tree: any, aggregationParams: AggregationParams) => { return copyOfTree; }; + +/** + * Gets or stores aggregation metrics. + * @todo Remove these functions after resolving timeSync to be a builtin functionality. + */ +export const storeAggregateMetrics = ( + aggregationParams?: AggregationParams +) => { + if (aggregationParams?.metrics) { + metricManager.metrics = aggregationParams?.metrics; + } + + return metricManager.metrics; +}; + +/** + * Creates an encapsulated object to retrieve the metrics. + */ +const metricManager = (() => { + let metric: AggregationMetric; + + const manager = { + get metrics() { + return metric; + }, + set metrics(value: AggregationMetric) { + metric = value; + }, + }; + + return manager; +})(); + +/** + * Returns aggregation method for given `unitName`. If doesn't exist then returns value `sum`. + */ +export const getAggregationMethod = (unitName: string) => { + debugLogger.setExecutingPluginName(); + memoizedLog(console.debug, CHECKING_AGGREGATION_METHOD(unitName)); + const aggregationMetricsStorage = storeAggregateMetrics(); + + if (`${unitName}` in aggregationMetricsStorage) { + return aggregationMetricsStorage[unitName].method; + } + + memoizedLog(logger.warn, UNKNOWN_PARAM(unitName)); + + return 'sum'; +}; From 061df462eaaeb24e915aabdc8a118940dfbbee63 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 18:22:49 +0400 Subject: [PATCH 26/37] feat(src): update load function to not deal with params --- src/common/lib/load.ts | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/common/lib/load.ts b/src/common/lib/load.ts index 02b44aef1..ade17795d 100644 --- a/src/common/lib/load.ts +++ b/src/common/lib/load.ts @@ -1,30 +1,18 @@ import {openYamlFileAsObject} from '../util/yaml'; -import {readAndParseJson} from '../../if-run/util/json'; -import {PARAMETERS} from '../../if-run/config'; import {STRINGS} from '../../if-run/config'; -import {Parameters} from '../../if-run/types/parameters'; - const {LOADING_MANIFEST} = STRINGS; /** - * Parses manifest file as an object. Checks if parameter file is passed via CLI, then loads it too. - * Returns context, tree and parameters (either the default one, or from CLI). + * Parses manifest file as an object. */ -export const load = async (inputPath: string, paramPath?: string) => { +export const load = async (inputPath: string) => { console.debug(LOADING_MANIFEST); const rawManifest = await openYamlFileAsObject(inputPath); - const parametersFromCli = - paramPath && - (await readAndParseJson(paramPath)); /** @todo validate json */ - const parameters = - parametersFromCli || - PARAMETERS; /** @todo PARAMETERS should be specified in parameterize only */ return { rawManifest, - parameters, }; }; From c0556cc52ddfb17abd31c96282a72ee6d5b6f3fd Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 18:24:25 +0400 Subject: [PATCH 27/37] fix(util): remove `SYNCING_PARAMETERS` from debugger log --- src/common/util/debug-logger.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/common/util/debug-logger.ts b/src/common/util/debug-logger.ts index 7babaaf3f..35dc77ea7 100644 --- a/src/common/util/debug-logger.ts +++ b/src/common/util/debug-logger.ts @@ -6,7 +6,6 @@ const logMessagesKeys: (keyof typeof STRINGS)[] = [ 'LOADING_MANIFEST', 'VALIDATING_MANIFEST', 'CAPTURING_RUNTIME_ENVIRONMENT_DATA', - 'SYNCING_PARAMETERS', 'CHECKING_AGGREGATION_METHOD', 'INITIALIZING_PLUGINS', 'INITIALIZING_PLUGIN', From a4a9fc50c76124ad3fe2e653b737b6720a4c0bb1 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 18:25:51 +0400 Subject: [PATCH 28/37] feat(src): update `getAggregationMethod` function call --- src/if-run/builtins/time-sync.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/if-run/builtins/time-sync.ts b/src/if-run/builtins/time-sync.ts index 78b293e75..9c14b63a4 100644 --- a/src/if-run/builtins/time-sync.ts +++ b/src/if-run/builtins/time-sync.ts @@ -11,11 +11,10 @@ import { TimeParams, } from '@grnsft/if-core/types'; -import {parameterize} from '../lib/parameterize'; - import {validate} from '../../common/util/validations'; import {STRINGS} from '../config'; +import {getAggregationMethod} from '../lib/aggregate'; Settings.defaultZone = 'utc'; @@ -200,7 +199,7 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { const inputKeys = Object.keys(input); return inputKeys.reduce((acc, key) => { - const method = parameterize.getAggregationMethod(key); + const method = getAggregationMethod(key); if (key === 'timestamp') { const perSecond = normalizeTimePerSecond(input.timestamp, i); @@ -254,7 +253,7 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { return acc; } - const method = parameterize.getAggregationMethod(metric); + const method = getAggregationMethod(metric); if (method === 'avg' || method === 'sum') { acc[metric] = 0; @@ -313,7 +312,7 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { const metrics = Object.keys(input); metrics.forEach(metric => { - const method = parameterize.getAggregationMethod(metric); + const method = getAggregationMethod(metric); acc[metric] = acc[metric] ?? 0; if (metric === 'timestamp') { From 84cfc8f64508c47ba3cedf667241543b8b55447d Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 18:28:09 +0400 Subject: [PATCH 29/37] test(src): update tests according to changes --- src/__tests__/common/lib/load.test.ts | 35 +--------------- src/__tests__/if-diff/lib/load.test.ts | 3 -- .../if-run/builtins/time-sync.test.ts | 17 ++++++++ src/__tests__/if-run/lib/aggregate.test.ts | 21 ++++++++-- .../if-run/util/aggregation-helper.test.ts | 41 +++++++++++++++---- src/__tests__/if-run/util/args.test.ts | 21 ---------- 6 files changed, 70 insertions(+), 68 deletions(-) diff --git a/src/__tests__/common/lib/load.test.ts b/src/__tests__/common/lib/load.test.ts index 43f6ee2af..a4f0289a4 100644 --- a/src/__tests__/common/lib/load.test.ts +++ b/src/__tests__/common/lib/load.test.ts @@ -1,6 +1,3 @@ -jest.mock('../../../if-run/util/json', () => - require('../../../__mocks__/json') -); jest.mock( 'mockavizta', () => ({ @@ -42,45 +39,17 @@ jest.mock('../../../common/util/yaml', () => ({ import {PluginParams} from '@grnsft/if-core/types'; -import {PARAMETERS} from '../../../if-run/config'; import {load} from '../../../common/lib/load'; describe('lib/load: ', () => { describe('load(): ', () => { - it('loads yaml with default parameters.', async () => { - const inputPath = 'load-default.yml'; - const paramPath = undefined; - - const result = await load(inputPath, paramPath); - - const expectedValue = { - rawManifest: 'raw-manifest', - parameters: PARAMETERS, - }; - - expect(result).toEqual(expectedValue); - }); - - it('loads yaml with custom parameters.', async () => { + it('successfully loads yaml.', async () => { const inputPath = 'load-default.yml'; - const paramPath = 'param-mock.json'; - const result = await load(inputPath, paramPath); + const result = await load(inputPath); const expectedValue = { rawManifest: 'raw-manifest', - parameters: { - 'mock-carbon': { - description: 'an amount of carbon emitted into the atmosphere', - unit: 'gCO2e', - aggregation: 'sum', - }, - 'mock-cpu': { - description: 'number of cores available', - unit: 'cores', - aggregation: 'none', - }, - }, }; expect(result).toEqual(expectedValue); diff --git a/src/__tests__/if-diff/lib/load.test.ts b/src/__tests__/if-diff/lib/load.test.ts index f722757e5..ab69d7d76 100644 --- a/src/__tests__/if-diff/lib/load.test.ts +++ b/src/__tests__/if-diff/lib/load.test.ts @@ -1,6 +1,3 @@ -jest.mock('../../../if-run/util/json', () => - require('../../../__mocks__/json') -); jest.mock( 'mockavizta', () => ({ diff --git a/src/__tests__/if-run/builtins/time-sync.test.ts b/src/__tests__/if-run/builtins/time-sync.test.ts index ab58b3e34..e216690df 100644 --- a/src/__tests__/if-run/builtins/time-sync.test.ts +++ b/src/__tests__/if-run/builtins/time-sync.test.ts @@ -1,6 +1,9 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {Settings, DateTime} from 'luxon'; +import {AggregationParams} from '../../../common/types/manifest'; + +import {storeAggregateMetrics} from '../../../if-run/lib/aggregate'; import {TimeSync} from '../../../if-run/builtins/time-sync'; import {STRINGS} from '../../../if-run/config'; @@ -51,6 +54,20 @@ jest.mock('luxon', () => { }); describe('builtins/time-sync:', () => { + beforeAll(() => { + const metricStorage: AggregationParams = { + metrics: { + carbon: {method: 'sum'}, + 'cpu/utilization': {method: 'sum'}, + 'time-reserved': {method: 'avg'}, + 'resources-total': {method: 'none'}, + }, + type: 'horizontal', + }; + + storeAggregateMetrics(metricStorage); + }); + describe('time-sync: ', () => { const basicConfig = { 'start-time': '2023-12-12T00:01:00.000Z', diff --git a/src/__tests__/if-run/lib/aggregate.test.ts b/src/__tests__/if-run/lib/aggregate.test.ts index 76de6300c..56488eea2 100644 --- a/src/__tests__/if-run/lib/aggregate.test.ts +++ b/src/__tests__/if-run/lib/aggregate.test.ts @@ -1,8 +1,21 @@ /* eslint-disable @typescript-eslint/ban-ts-comment */ -import {aggregate} from '../../../if-run/lib/aggregate'; +import {AggregationParams} from '../../../common/types/manifest'; + +import {aggregate, storeAggregateMetrics} from '../../../if-run/lib/aggregate'; describe('lib/aggregate: ', () => { + beforeAll(() => { + const metricStorage: AggregationParams = { + metrics: { + carbon: {method: 'sum'}, + }, + type: 'horizontal', + }; + + storeAggregateMetrics(metricStorage); + }); + describe('aggregate(): ', () => { it('returns tree if aggregation is missing.', () => { const tree = {}; @@ -44,7 +57,7 @@ describe('lib/aggregate: ', () => { }; const aggregatedTree = aggregate(tree, { - metrics: ['carbon'], + metrics: {carbon: {method: 'sum'}}, type: 'horizontal', }); const expectedAggregated = { @@ -92,7 +105,7 @@ describe('lib/aggregate: ', () => { }; const aggregatedTree = aggregate(tree, { - metrics: ['carbon'], + metrics: {carbon: {method: 'sum'}}, type: 'vertical', }); const expectedOutputs = [ @@ -153,7 +166,7 @@ describe('lib/aggregate: ', () => { }; const aggregatedTree = aggregate(tree, { - metrics: ['carbon'], + metrics: {carbon: {method: 'sum'}}, type: 'both', }); diff --git a/src/__tests__/if-run/util/aggregation-helper.test.ts b/src/__tests__/if-run/util/aggregation-helper.test.ts index 78c8c4aeb..5f8bc3325 100644 --- a/src/__tests__/if-run/util/aggregation-helper.test.ts +++ b/src/__tests__/if-run/util/aggregation-helper.test.ts @@ -1,7 +1,11 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {PluginParams} from '@grnsft/if-core/types'; +import {AggregationParams} from '../../../common/types/manifest'; + import {aggregateInputsIntoOne} from '../../../if-run/util/aggregation-helper'; +import {AggregationMetric} from '../../../if-run/types/aggregation'; +import {storeAggregateMetrics} from '../../../if-run/lib/aggregate'; import {STRINGS} from '../../../if-run/config'; @@ -9,10 +13,23 @@ const {InvalidAggregationMethodError, MissingAggregationParamError} = ERRORS; const {INVALID_AGGREGATION_METHOD, METRIC_MISSING} = STRINGS; describe('util/aggregation-helper: ', () => { + beforeAll(() => { + const metricStorage: AggregationParams = { + metrics: { + carbon: {method: 'sum'}, + 'cpu/number-cores': {method: 'none'}, + 'cpu/utilization': {method: 'sum'}, + }, + type: 'horizontal', + }; + + storeAggregateMetrics(metricStorage); + }); + describe('aggregateInputsIntoOne(): ', () => { it('throws error if aggregation method is none.', () => { const inputs: PluginParams[] = []; - const metrics: string[] = ['cpu/number-cores']; + const metrics: AggregationMetric = {'cpu/number-cores': {method: 'none'}}; const isTemporal = false; expect.assertions(2); @@ -23,14 +40,16 @@ describe('util/aggregation-helper: ', () => { expect(error).toBeInstanceOf(InvalidAggregationMethodError); if (error instanceof InvalidAggregationMethodError) { - expect(error.message).toEqual(INVALID_AGGREGATION_METHOD(metrics[0])); + expect(error.message).toEqual( + INVALID_AGGREGATION_METHOD('cpu/number-cores') + ); } } }); it('throws error if aggregation criteria is not found in input.', () => { const inputs: PluginParams[] = [{timestamp: '', duration: 10}]; - const metrics: string[] = ['cpu/utilization']; + const metrics: AggregationMetric = {'cpu/utilization': {method: 'sum'}}; const isTemporal = false; expect.assertions(2); @@ -41,7 +60,7 @@ describe('util/aggregation-helper: ', () => { expect(error).toBeInstanceOf(MissingAggregationParamError); if (error instanceof MissingAggregationParamError) { - expect(error.message).toEqual(METRIC_MISSING(metrics[0], 0)); + expect(error.message).toEqual(METRIC_MISSING('cpu/utilization', 0)); } } }); @@ -51,7 +70,7 @@ describe('util/aggregation-helper: ', () => { {timestamp: '', duration: 10, carbon: 10}, {timestamp: '', duration: 10, carbon: 20}, ]; - const metrics: string[] = ['carbon']; + const metrics: AggregationMetric = {carbon: {method: 'sum'}}; const isTemporal = true; const expectedValue = { @@ -68,7 +87,7 @@ describe('util/aggregation-helper: ', () => { {timestamp: '', duration: 10, carbon: 10}, {timestamp: '', duration: 10, carbon: 20}, ]; - const metrics: string[] = ['carbon']; + const metrics: AggregationMetric = {carbon: {method: 'sum'}}; const isTemporal = false; const expectedValue = { @@ -79,11 +98,19 @@ describe('util/aggregation-helper: ', () => { }); it('calculates average of metrics.', () => { + const metricStorage: AggregationParams = { + metrics: { + 'cpu/utilization': {method: 'avg'}, + }, + type: 'horizontal', + }; + + storeAggregateMetrics(metricStorage); const inputs: PluginParams[] = [ {timestamp: '', duration: 10, 'cpu/utilization': 10}, {timestamp: '', duration: 10, 'cpu/utilization': 90}, ]; - const metrics: string[] = ['cpu/utilization']; + const metrics: AggregationMetric = {'cpu/utilization': {method: 'avg'}}; const isTemporal = false; const expectedValue = { diff --git a/src/__tests__/if-run/util/args.test.ts b/src/__tests__/if-run/util/args.test.ts index 5ea3dd03b..c43d49eb9 100644 --- a/src/__tests__/if-run/util/args.test.ts +++ b/src/__tests__/if-run/util/args.test.ts @@ -21,11 +21,6 @@ jest.mock('ts-command-line-args', () => ({ manifest: 'manifest-mock.yml', output: 'output-mock.yml', }; - case 'override-params': - return { - manifest: 'manifest-mock.yml', - 'override-params': 'override-params-mock.yml', - }; case 'not-yaml': return { manifest: 'mock.notyaml', @@ -119,22 +114,6 @@ describe('if-run/util/args: ', () => { expect(result).toEqual(expectedResult); }); - it('returns manifest with `paramPath`.', () => { - expect.assertions(1); - - process.env.result = 'override-params'; - - const result = parseIfRunProcessArgs(); - const manifestPath = 'manifest-mock.yml'; - const expectedResult = { - inputPath: path.normalize(`${processRunningPath}/${manifestPath}`), - paramPath: 'override-params-mock.yml', - outputOptions: {}, - }; - - expect(result).toEqual(expectedResult); - }); - it('returns manifest and output path.', () => { expect.assertions(1); From a83cc42e04ae6fb700361bf235633d680dcb9ef7 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 18:32:00 +0400 Subject: [PATCH 30/37] fix(config): remove message --- src/if-run/config/strings.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/if-run/config/strings.ts b/src/if-run/config/strings.ts index 0a9c1a455..cdbc7c8c2 100644 --- a/src/if-run/config/strings.ts +++ b/src/if-run/config/strings.ts @@ -3,8 +3,6 @@ export const STRINGS = { MISSING_PATH: "Initalization param 'path' is missing.", UNSUPPORTED_PLUGIN: "Plugin interface doesn't implement 'execute' or 'metadata' methods.", - OVERRIDE_WARNING: - '\n**WARNING**: You are overriding the IF default parameters file. Please be extremely careful of unintended side-effects in your plugin pipeline!\n', NOT_NATIVE_PLUGIN: (path: string) => ` You are using plugin ${path} which is not part of the Impact Framework standard library. You should do your own research to ensure the plugins are up to date and accurate. They may not be actively maintained.`, From 608bd8ee43006e34132d6e1435903bc27ba75222 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 19:17:42 +0400 Subject: [PATCH 31/37] fix(lib): fix aggregation method check --- src/if-run/lib/aggregate.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/if-run/lib/aggregate.ts b/src/if-run/lib/aggregate.ts index 7ca0d3f42..a3d2d2bad 100644 --- a/src/if-run/lib/aggregate.ts +++ b/src/if-run/lib/aggregate.ts @@ -143,7 +143,7 @@ export const getAggregationMethod = (unitName: string) => { memoizedLog(console.debug, CHECKING_AGGREGATION_METHOD(unitName)); const aggregationMetricsStorage = storeAggregateMetrics(); - if (`${unitName}` in aggregationMetricsStorage) { + if (aggregationMetricsStorage && `${unitName}` in aggregationMetricsStorage) { return aggregationMetricsStorage[unitName].method; } From ff605134fff235b04ee26f6884ed438a27948a4b Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 19:23:33 +0400 Subject: [PATCH 32/37] docs(src): update documentations --- Refactor-migration-guide.md | 5 ++++- src/if-run/builtins/README.md | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Refactor-migration-guide.md b/Refactor-migration-guide.md index 0e662110a..f8da3d8b8 100644 --- a/Refactor-migration-guide.md +++ b/Refactor-migration-guide.md @@ -201,6 +201,8 @@ This is a builtin feature of IF, meaning it does not have to be initialized as a - `metrics`: which metrics do you want to aggregate? Every metric you provide here must exist in the output array. + - `method`: the aggregation method for the specied metric + - `type`: the options are `horizontal`, `vertical` or both. Horizontal aggregation is the type that condenses each time series into a single summary value. Vertical aggregation is aggregated across components. Here's what the config block should look like: @@ -208,7 +210,8 @@ Here's what the config block should look like: ```yaml aggregation: metrics: - - 'carbon' + 'carbon': + method: 'sum' type: 'both' ``` diff --git a/src/if-run/builtins/README.md b/src/if-run/builtins/README.md index df743e93c..ccacf83e5 100644 --- a/src/if-run/builtins/README.md +++ b/src/if-run/builtins/README.md @@ -165,7 +165,7 @@ Note that when `error-on-padding` is `true` no padding is performed and the plug ##### Resampling rules -Now we have synchronized, continuous, high resolution time series data, we can resample. To achieve this, we use `interval`, which sets the global temporal resolution for the final, processed time series. `intervalk` is expressed in units of seconds, which means we can simply batch `observations` together in groups of size `interval`. For each value in each object we either sum, average or copy the values into one single summary object representing each time bucket of size `interval` depending on their `aggregation-method` defined in `params.ts`. The returned array is the final, synchronized time series at the desired temporal resolution. +Now we have synchronized, continuous, high resolution time series data, we can resample. To achieve this, we use `interval`, which sets the global temporal resolution for the final, processed time series. `interval` is expressed in units of seconds, which means we can simply batch `observations` together in groups of size `interval`. For each value in each object we either sum, average or copy the values into one single summary object representing each time bucket of size `interval` depending on their `aggregation-method` defined in `aggregation` section in the manifest file. The returned array is the final, synchronized time series at the desired temporal resolution. #### Assumptions and limitations From 4a5e15deff68d2d7ef5bc092a2b67c2ecbbb4578 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 19:24:32 +0400 Subject: [PATCH 33/37] test(mocks): update mocks --- src/__mocks__/builtins/export-yaml.ts | 2 +- src/__mocks__/fs/index.ts | 20 -------------------- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/src/__mocks__/builtins/export-yaml.ts b/src/__mocks__/builtins/export-yaml.ts index 399bbdb28..85f54e966 100644 --- a/src/__mocks__/builtins/export-yaml.ts +++ b/src/__mocks__/builtins/export-yaml.ts @@ -162,6 +162,6 @@ export const aggregated = { }; export const aggregation = { - metrics: ['carbon'], + metrics: {carbon: {method: 'sum'}}, type: 'both', }; diff --git a/src/__mocks__/fs/index.ts b/src/__mocks__/fs/index.ts index 6d4f61b8b..0f37a4291 100644 --- a/src/__mocks__/fs/index.ts +++ b/src/__mocks__/fs/index.ts @@ -10,27 +10,7 @@ export const readFile = async (filePath: string) => { return fs.readFileSync(updatedPath, 'utf8'); } - /** mock for util/json */ - if (filePath.includes('json-reject')) { - return Promise.reject(new Error('rejected')); - } - if (filePath.includes('json')) { - if (filePath.includes('param')) { - return JSON.stringify({ - 'mock-carbon': { - description: 'an amount of carbon emitted into the atmosphere', - unit: 'gCO2e', - aggregation: 'sum', - }, - 'mock-cpu': { - description: 'number of cores available', - unit: 'cores', - aggregation: 'none', - }, - }); - } - return JSON.stringify(filePath); } From 6dd3984a7548b6318c82f3fea7a721cf50338e81 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 11 Jul 2024 19:29:53 +0400 Subject: [PATCH 34/37] fix(manifests): update aggregation section in the manifests --- manifests/examples/pipelines/nesting.yml | 4 ++-- .../examples/pipelines/pipeline-with-aggregate.yml | 11 ++++++----- manifests/examples/pipelines/pipeline-with-mocks.yml | 3 ++- .../outputs/bugs/aggregation-error-wrong-metric.yaml | 3 ++- .../features/aggregate-failure-invalid-metrics.yaml | 3 ++- .../aggregate-failure-missing-metric-in-inputs.yaml | 3 ++- manifests/outputs/features/aggregate-horizontal.yaml | 3 ++- manifests/outputs/features/aggregate-vertical.yaml | 3 ++- manifests/outputs/features/aggregate.yaml | 3 ++- 9 files changed, 22 insertions(+), 14 deletions(-) diff --git a/manifests/examples/pipelines/nesting.yml b/manifests/examples/pipelines/nesting.yml index ac413d1c6..9f034092f 100644 --- a/manifests/examples/pipelines/nesting.yml +++ b/manifests/examples/pipelines/nesting.yml @@ -6,9 +6,9 @@ tags: category: on-premise aggregation: metrics: - - "carbon" + "carbon": + method: sum type: "both" -params: initialize: plugins: "interpolate": diff --git a/manifests/examples/pipelines/pipeline-with-aggregate.yml b/manifests/examples/pipelines/pipeline-with-aggregate.yml index 7b689f1d8..a833bcb9b 100644 --- a/manifests/examples/pipelines/pipeline-with-aggregate.yml +++ b/manifests/examples/pipelines/pipeline-with-aggregate.yml @@ -3,19 +3,20 @@ description: a full pipeline with the aggregate feature enabled tags: aggregation: metrics: - - "carbon" + "carbon": + method: sum type: "both" initialize: plugins: "interpolate": method: Interpolation - path: 'builtin' + path: "builtin" global-config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] - input-parameter: 'cpu/utilization' - output-parameter: 'cpu-factor' + input-parameter: "cpu/utilization" + output-parameter: "cpu-factor" "cpu-factor-to-wattage": method: Multiply path: builtin @@ -182,4 +183,4 @@ tree: cpu/utilization: 33 cloud/instance-type: A1 cloud/region: uk-west - requests: 180 \ No newline at end of file + requests: 180 diff --git a/manifests/examples/pipelines/pipeline-with-mocks.yml b/manifests/examples/pipelines/pipeline-with-mocks.yml index 2cd23920e..9e4292f51 100644 --- a/manifests/examples/pipelines/pipeline-with-mocks.yml +++ b/manifests/examples/pipelines/pipeline-with-mocks.yml @@ -3,7 +3,8 @@ description: a full pipeline seeded with data from mock-observations feature tags: aggregation: metrics: - - "carbon" + "carbon": + method: sum type: "both" initialize: plugins: diff --git a/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml b/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml index 2bbfba839..ebf55ad73 100644 --- a/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml +++ b/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml @@ -5,7 +5,8 @@ description: >- tags: null aggregation: metrics: - - dummy-param + "dummy-param": + method: sum type: both initialize: plugins: diff --git a/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml b/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml index ad145646f..870be308a 100644 --- a/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml +++ b/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml @@ -2,7 +2,8 @@ name: Aggregation description: Fails with invalid metric. aggregation: metrics: - - test + "test": + method: sum type: both initialize: plugins: diff --git a/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml b/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml index 09b2e9338..4a3da85ae 100644 --- a/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml +++ b/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml @@ -2,7 +2,8 @@ name: Aggregation description: Fails with missing metric in inputs. aggregation: metrics: - - cpu/utilization + "cpu/utilization": + method: sum type: both initialize: plugins: diff --git a/manifests/outputs/features/aggregate-horizontal.yaml b/manifests/outputs/features/aggregate-horizontal.yaml index 2241815d2..f870bb4d2 100644 --- a/manifests/outputs/features/aggregate-horizontal.yaml +++ b/manifests/outputs/features/aggregate-horizontal.yaml @@ -2,7 +2,8 @@ name: Aggregation description: Apply `horizontal` aggregation aggregation: metrics: - - cpu/utilization + "cpu/utilization": + method: sum type: horizontal initialize: plugins: diff --git a/manifests/outputs/features/aggregate-vertical.yaml b/manifests/outputs/features/aggregate-vertical.yaml index 0c7b32b6a..0fd5b170a 100644 --- a/manifests/outputs/features/aggregate-vertical.yaml +++ b/manifests/outputs/features/aggregate-vertical.yaml @@ -2,7 +2,8 @@ name: Aggregation description: Apply `vertical` aggregation aggregation: metrics: - - cpu/utilization + "cpu/utilization": + method: sum type: vertical initialize: plugins: diff --git a/manifests/outputs/features/aggregate.yaml b/manifests/outputs/features/aggregate.yaml index b58ac9031..35ab21423 100644 --- a/manifests/outputs/features/aggregate.yaml +++ b/manifests/outputs/features/aggregate.yaml @@ -2,7 +2,8 @@ name: Aggregation description: Apply both `horizontal` and `vertical` aggregations aggregation: metrics: - - cpu/utilization + "cpu/utilization": + method: sum type: both initialize: plugins: From 03275e34b7af97774bac05e6445860aa87409383 Mon Sep 17 00:00:00 2001 From: mouhamadalmounayar Date: Wed, 10 Jul 2024 12:59:49 +0200 Subject: [PATCH 35/37] feat(builtins): handle multiple matches in extractMatching function - Update output parameter documentation - Added corresponding test case Signed-off-by: mouhamadalmounayar --- src/__tests__/if-run/builtins/regex.test.ts | 27 +++++++++++++++++++++ src/if-run/builtins/regex/README.md | 3 ++- src/if-run/builtins/regex/index.ts | 6 ++--- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/__tests__/if-run/builtins/regex.test.ts b/src/__tests__/if-run/builtins/regex.test.ts index 6ac6bd4e9..ac0594cd6 100644 --- a/src/__tests__/if-run/builtins/regex.test.ts +++ b/src/__tests__/if-run/builtins/regex.test.ts @@ -49,6 +49,33 @@ describe('builtins/regex: ', () => { expect(result).toStrictEqual(expectedResult); }); + it('successfully applies regex strategy with multiple matches', async () => { + const globalConfig = { + parameter: 'cloud/instance-type', + match: '/(?<=_)[^_]+?(?=_|$)/g', + output: 'cloud/instance-type', + }; + const regex = Regex(globalConfig); + + const expectedResult = [ + { + timestamp: '2023-08-06T00:00', + duration: 3600, + 'cloud/instance-type': 'DS1 v2', + }, + ]; + + const result = await regex.execute([ + { + timestamp: '2023-08-06T00:00', + duration: 3600, + 'cloud/instance-type': 'Standard_DS1_v2', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + it('returns a result when regex is not started and ended with ``.', async () => { const physicalProcessor = 'Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz'; diff --git a/src/if-run/builtins/regex/README.md b/src/if-run/builtins/regex/README.md index a0fa5d624..e424efca0 100644 --- a/src/if-run/builtins/regex/README.md +++ b/src/if-run/builtins/regex/README.md @@ -22,7 +22,8 @@ Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-26 ## Returns -- `output`: the first match of `parameter` with the parameter name with `match` defined in global config. +- `output`: The match of the `parameter` value using the `match` regex defined in the global config. If the `match` regex includes the global flag (`g`), a string containing all matches separated by spaces. + ## Implementation diff --git a/src/if-run/builtins/regex/index.ts b/src/if-run/builtins/regex/index.ts index 45c65d6f6..bb97e8358 100644 --- a/src/if-run/builtins/regex/index.ts +++ b/src/if-run/builtins/regex/index.ts @@ -80,13 +80,13 @@ export const Regex = (globalConfig: ConfigParams): ExecutePlugin => { } const regex = eval(match); - const matchedItem = input[parameter].match(regex); + const matchedItems = input[parameter].match(regex); - if (!matchedItem || !matchedItem[0]) { + if (!matchedItems || matchedItems.length === 0) { throw new RegexMismatchError(REGEX_MISMATCH(input[parameter], match)); } - return matchedItem[0]; + return matchedItems.join(' '); }; return { From e609d0c194e402d60a10e606b91fa7c7e7230977 Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 16 Jul 2024 11:13:29 +0400 Subject: [PATCH 36/37] fix(src): improve manifestData check --- src/if-csv/index.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/if-csv/index.ts b/src/if-csv/index.ts index 94ede0bb4..72c33379c 100644 --- a/src/if-csv/index.ts +++ b/src/if-csv/index.ts @@ -18,14 +18,11 @@ const IfCsv = async () => { const pipedManifest = await parseManifestFromStdin(); const {manifest, output, params} = await parseIfCsvArgs(); const resolvedManifest = manifest || pipedManifest; - let manifestData: Manifest; if (resolvedManifest) { - if (pipedManifest) { - manifestData = (await YAML.load(pipedManifest!)) as Manifest; - } else { - manifestData = await getManifestData(manifest!); - } + const manifestData = pipedManifest + ? ((await YAML.load(pipedManifest!)) as Manifest) + : await getManifestData(manifest!); const options: CsvOptions = { tree: manifestData.tree, From 669a422f5ea0adb839cf68712ac6bdb784a1f593 Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 16 Jul 2024 11:25:41 +0400 Subject: [PATCH 37/37] fix(src): rename the `executeCsv` function into `generateCsv` --- src/__tests__/if-csv/util/helpers.test.ts | 14 +++++++------- src/if-csv/index.ts | 4 ++-- src/if-csv/util/helpers.ts | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/__tests__/if-csv/util/helpers.test.ts b/src/__tests__/if-csv/util/helpers.test.ts index 1859f4850..1781f5fbc 100644 --- a/src/__tests__/if-csv/util/helpers.test.ts +++ b/src/__tests__/if-csv/util/helpers.test.ts @@ -3,7 +3,7 @@ import * as fs from 'fs/promises'; import {stringify} from 'csv-stringify/sync'; import {ERRORS} from '@grnsft/if-core/utils'; -import {executeCsv, getManifestData} from '../../../if-csv/util/helpers'; +import {generateCsv, getManifestData} from '../../../if-csv/util/helpers'; import {CsvOptions} from '../../../if-csv/types/csv'; const {ManifestValidationError} = ERRORS; @@ -26,7 +26,7 @@ jest.mock('fs/promises', () => { }); describe('if-csv/util/helpers: ', () => { - describe('executeCsv(): ', () => { + describe('generateCsv(): ', () => { it('generates CSV file with correct data.', async () => { const outputPath = 'output'; const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; @@ -65,7 +65,7 @@ describe('if-csv/util/helpers: ', () => { params: 'carbon', }; - await executeCsv(options); + await generateCsv(options); expect(fs.writeFile).toHaveBeenCalledWith( 'output.csv', @@ -112,7 +112,7 @@ describe('if-csv/util/helpers: ', () => { params: 'carbon', }; - await executeCsv(options); + await generateCsv(options); expect.assertions(1); @@ -162,7 +162,7 @@ describe('if-csv/util/helpers: ', () => { params: 'carbon', }; - await executeCsv(options); + await generateCsv(options); expect.assertions(1); expect(fs.writeFile).toHaveBeenCalledWith( @@ -190,7 +190,7 @@ describe('if-csv/util/helpers: ', () => { params: 'carbon', }; - const result = await executeCsv(options); + const result = await generateCsv(options); expect.assertions(1); expect(result).toEqual(stringify(matrix, {columns})); @@ -221,7 +221,7 @@ describe('if-csv/util/helpers: ', () => { params: 'carbon', }; - await executeCsv(options); + await generateCsv(options); expect.assertions(1); expect(fs.writeFile).toHaveBeenCalledWith( diff --git a/src/if-csv/index.ts b/src/if-csv/index.ts index 72c33379c..602aabe1e 100644 --- a/src/if-csv/index.ts +++ b/src/if-csv/index.ts @@ -7,7 +7,7 @@ import {debugLogger} from '../common/util/debug-logger'; import {Manifest} from '../common/types/manifest'; import {logger} from '../common/util/logger'; -import {executeCsv, getManifestData} from './util/helpers'; +import {generateCsv, getManifestData} from './util/helpers'; import {parseIfCsvArgs} from './util/args'; import {CsvOptions} from './types/csv'; @@ -30,7 +30,7 @@ const IfCsv = async () => { outputPath: output, params, }; - const result = await executeCsv(options); + const result = await generateCsv(options); if (!output && result) { console.log(result); diff --git a/src/if-csv/util/helpers.ts b/src/if-csv/util/helpers.ts index 397293dd8..655611ff3 100644 --- a/src/if-csv/util/helpers.ts +++ b/src/if-csv/util/helpers.ts @@ -26,9 +26,9 @@ export const getManifestData = async (manifest: string) => { }; /** - * Executes a CSV generation based on the provided tree structure, context, output path, and params. + * Generates a CSV file based on the provided tree structure, context, output path, and params. */ -export const executeCsv = async (options: CsvOptions) => { +export const generateCsv = async (options: CsvOptions) => { const {tree, context, outputPath, params} = options; const columns = ['Path']; const matrix = [columns];