more core testing

test-standalone/core/arbitrary-objects/Crasher.py

@@ -0,0 +1,20 @@
+
+from ConfigValidator.Config.RunnerConfig import RunnerConfig as OriginalRunnerConfig
+from ProgressManager.Output.CSVOutputManager import CSVOutputManager
+from ProgressManager.RunTable.Models.RunProgress import RunProgress
+
+import TestUtilities
+
+if __name__ == '__main__':
+    TEST_DIR = TestUtilities.get_test_dir(__file__)
+
+    config_file = TestUtilities.load_and_get_config_file_as_module(TEST_DIR)
+    RunnerConfig: OriginalRunnerConfig = config_file.RunnerConfig
+
+    csv_data_manager = CSVOutputManager(RunnerConfig.results_output_path / RunnerConfig.name)
+    run_table = csv_data_manager.read_run_table()
+    for row in run_table:
+        if row['__run_id'] == 'run_1':
+            row['__done']  = RunProgress.TODO
+            row['avg_cpu'] = 0
+    csv_data_manager.write_run_table(run_table)

test-standalone/core/arbitrary-objects/RunnerConfig.py

@@ -0,0 +1,109 @@
+from EventManager.Models.RunnerEvents import RunnerEvents
+from EventManager.EventSubscriptionController import EventSubscriptionController
+from ConfigValidator.Config.Models.RunTableModel import RunTableModel
+from ConfigValidator.Config.Models.FactorModel import FactorModel
+from ConfigValidator.Config.Models.RunnerContext import RunnerContext
+from ConfigValidator.Config.Models.OperationType import OperationType
+from ExtendedTyping.Typing import SupportsStr
+from ProgressManager.Output.OutputProcedure import OutputProcedure as output
+
+from typing import Dict, List, Any, Optional
+from pathlib import Path
+from os.path import dirname, realpath
+
+'''
+Test Description:
+
+Test functionality for arbitrary objects as factor levels
+  * When recovering from a crash, the generated objects from `create_run_table()` should be used
+    instead of the `str` values found in the stored csv.
+'''
+
+
+class CustomObject:
+    def __init__(self, x):
+        self._x = x+10
+
+    @property
+    def x(self):
+        return self._x
+
+    def __str__(self):
+        return f'{self._x}'
+
+
+class RunnerConfig:
+    ROOT_DIR = Path(dirname(realpath(__file__)))
+
+    # ================================ USER SPECIFIC CONFIG ================================
+    name:                       str             = "new_runner_experiment"
+    results_output_path:        Path             = ROOT_DIR / 'experiments'
+    operation_type:             OperationType   = OperationType.AUTO
+    time_between_runs_in_ms:    int             = 100
+
+    def __init__(self):
+        """Executes immediately after program start, on config load"""
+
+        EventSubscriptionController.subscribe_to_multiple_events([
+            (RunnerEvents.BEFORE_EXPERIMENT, self.before_experiment),
+            (RunnerEvents.BEFORE_RUN       , self.before_run       ),
+            (RunnerEvents.START_RUN        , self.start_run        ),
+            (RunnerEvents.START_MEASUREMENT, self.start_measurement),
+            (RunnerEvents.INTERACT         , self.interact         ),
+            (RunnerEvents.STOP_MEASUREMENT , self.stop_measurement ),
+            (RunnerEvents.STOP_RUN         , self.stop_run         ),
+            (RunnerEvents.POPULATE_RUN_DATA, self.populate_run_data),
+            (RunnerEvents.AFTER_EXPERIMENT , self.after_experiment )
+        ])
+        self.run_table_model = None  # Initialized later
+
+        output.console_log("Custom config loaded")
+
+    def create_run_table(self) -> List[Dict]:
+        x1, x2, x3 = CustomObject(1), CustomObject(2), CustomObject(3)
+
+        factor1 = FactorModel("example_factor1", [x1, x2, x3])
+        factor2 = FactorModel("example_factor2", [True, False])
+        self.run_table_model = RunTableModel(
+            factors=[factor1, factor2],
+            exclude_variations=[
+                {factor1: [x1]},
+                {factor1: [x2], factor2: [True]},
+            ],
+            data_columns=['avg_cpu', 'avg_mem']
+        )
+        return self.run_table_model.generate_experiment_run_table()
+
+    def before_experiment(self) -> None:
+        output.console_log("Config.before_experiment() called!")
+
+    def before_run(self) -> None:
+        output.console_log("Config.before_run() called!")
+
+    def start_run(self, context: RunnerContext) -> None:
+        output.console_log("Config.start_run() called!")
+
+    def start_measurement(self, context: RunnerContext) -> None:
+        output.console_log("Config.start_measurement() called!")
+
+    def interact(self, context: RunnerContext) -> None:
+        output.console_log("Config.interact() called!")
+
+    def stop_measurement(self, context: RunnerContext) -> None:
+        output.console_log("Config.stop_measurement called!")
+
+    def stop_run(self, context: RunnerContext) -> None:
+        output.console_log("Config.stop_run() called!")
+
+    def populate_run_data(self, context: RunnerContext) -> Optional[Dict[str, SupportsStr]]:
+        output.console_log("Config.populate_run_data() called!")
+        return {
+            'avg_cpu': context.run_variation['example_factor1'].x,
+            'avg_mem': 18.1
+        }
+
+    def after_experiment(self) -> None:
+        output.console_log("Config.after_experiment() called!")
+
+    # ================================ DO NOT ALTER BELOW THIS LINE ================================
+    experiment_path:            Path             = None

test-standalone/core/arbitrary-objects/Validator.py

@@ -0,0 +1,23 @@
+
+import csv
+
+from ConfigValidator.Config.RunnerConfig import RunnerConfig as OriginalRunnerConfig
+from ProgressManager.Output.CSVOutputManager import CSVOutputManager
+from ProgressManager.RunTable.Models.RunProgress import RunProgress
+
+import TestUtilities
+
+if __name__ == 'main':
+    TEST_DIR = TestUtilities.get_test_dir(__file__)
+
+    config_file = TestUtilities.load_and_get_config_file_as_module(TEST_DIR)
+    RunnerConfig: OriginalRunnerConfig = config_file.RunnerConfig
+
+    csv_data_manager = CSVOutputManager(RunnerConfig.results_output_path / RunnerConfig.name)
+    run_table = csv_data_manager.read_run_table()
+
+    for row in run_table:
+        assert(row['avg_cpu'] == row['example_factor1'])
+        assert(row['__done']) == RunProgress.DONE.name
+        if row['__run_id'] == 'run_1':
+            assert(int(row['avg_cpu'])) == 13

test-standalone/plugins/CodecarbonWrapper/combined/RunnerConfig.py

@@ -14,6 +14,12 @@
 from Plugins import CodecarbonWrapper
 from Plugins.CodecarbonWrapper import DataColumns as CCDataCols
 
+'''
+Test Description:
+
+Test Plugins.CodecarbonWrapper class decorator
+'''
+
 @CodecarbonWrapper.emission_tracker(
     data_columns=[CCDataCols.EMISSIONS, CCDataCols.ENERGY_CONSUMED],
     country_iso_code="NLD" # your country code
@@ -25,7 +31,7 @@ class RunnerConfig:
     name:                       str             = "new_runner_experiment"
     results_output_path:        Path             = ROOT_DIR / 'experiments'
     operation_type:             OperationType   = OperationType.AUTO
-    time_between_runs_in_ms:    int             = 1000
+    time_between_runs_in_ms:    int             = 100
 
     def __init__(self):
         """Executes immediately after program start, on config load"""

test-standalone/plugins/CodecarbonWrapper/individual/RunnerConfig.py

@@ -14,14 +14,20 @@
 from Plugins import CodecarbonWrapper
 from Plugins.CodecarbonWrapper import DataColumns as CCDataCols
 
+'''
+Test Description:
+
+Test Plugins.CodecarbonWrapper individual decorators
+'''
+
 class RunnerConfig:
     ROOT_DIR = Path(dirname(realpath(__file__)))
 
     # ================================ USER SPECIFIC CONFIG ================================
     name:                       str             = "new_runner_experiment"
     results_output_path:        Path             = ROOT_DIR / 'experiments'
     operation_type:             OperationType   = OperationType.AUTO
-    time_between_runs_in_ms:    int             = 1000
+    time_between_runs_in_ms:    int             = 100
 
     def __init__(self):
         """Executes immediately after program start, on config load"""

test-standalone/runner.sh

@@ -15,6 +15,7 @@ fi
 
 set -e
 tests=( # TODO: gather_tests recursively
+  "${PROJECT_DIR}/test-standalone/core/arbitrary-objects"
   "${PROJECT_DIR}/test-standalone/plugins/CodecarbonWrapper/individual"
   "${PROJECT_DIR}/test-standalone/plugins/CodecarbonWrapper/combined"
 )
@@ -27,7 +28,16 @@ for test in "${tests[@]}"; do
     rm -rf "$test/experiments"
 
     echo " [*] Executing"
-    python experiment-runner/ "${test}/RunnerConfig.py"
+    python experiment-runner/ "$test/RunnerConfig.py"
+
+    # Simulate a crash
+    if [ -f "$test/Crasher.py" ]; then
+        echo " [*] Modifying generated run table to simulate a crash"
+        python "$test/Crasher.py"
+
+        echo " [*] Re-running experiment"
+        python experiment-runner/ "$test/RunnerConfig.py"
+    fi
 
     echo " [*] Validating"
     python "$test/Validator.py"