diff --git a/Robot-Framework/config/variables.robot b/Robot-Framework/config/variables.robot
index f2c59d7..b65edd0 100644
--- a/Robot-Framework/config/variables.robot
+++ b/Robot-Framework/config/variables.robot
@@ -4,18 +4,6 @@
*** Settings ***
Library OperatingSystem
-*** Variables ***
-${DEVICE} ${DEVICE}
-
-${SERIAL_PORT} ${EMPTY}
-${DEVICE_IP_ADDRESS} ${EMPTY}
-${SOCKET_IP_ADDRESS} ${EMPTY}
-${LOGIN} ${EMPTY}
-${PASSWORD} ${EMPTY}
-${PLUG_USERNAME} ${EMPTY}
-${PLUG_PASSWORD} ${EMPTY}
-${netvm_service} ${EMPTY}
-
*** Keywords ***
Set Variables
@@ -25,7 +13,8 @@ Set Variables
Set Global Variable ${SERIAL_PORT} ${config['addresses']['${DEVICE}']['serial_port']}
Set Global Variable ${DEVICE_IP_ADDRESS} ${config['addresses']['${DEVICE}']['device_ip_address']}
Set Global Variable ${SOCKET_IP_ADDRESS} ${config['addresses']['${DEVICE}']['socket_ip_address']}
- Set Global Variable ${netvm_service} microvm@netvm.service
+ Set Global Variable ${NETVM_SERVICE} microvm@netvm.service
+ Set Global Variable ${THREADS_NUMBER} ${config['addresses']['${DEVICE}']['threads']}
Read Config
[Arguments] ${file_path}=../config/test_config.json
diff --git a/Robot-Framework/lib/PerformanceDataProcessing.py b/Robot-Framework/lib/PerformanceDataProcessing.py
new file mode 100644
index 0000000..dce5521
--- /dev/null
+++ b/Robot-Framework/lib/PerformanceDataProcessing.py
@@ -0,0 +1,196 @@
+# SPDX-FileCopyrightText: 2022-2023 Technology Innovation Institute (TII)
+# SPDX-License-Identifier: Apache-2.0
+
+import csv
+import os
+import matplotlib.pyplot as plt
+import logging
+from robot.api.deco import keyword
+
+
+class PerformanceDataProcessing:
+
+ def __init__(self, device):
+ # Initialize the instance variable with the global variable value
+ self.data_dir = "../../../Performance_test_results/"
+ self.device = device
+
+ def _write_to_csv(self, test_name, data):
+ file_path = os.path.join(self.data_dir, f"{self.device}_{test_name}.csv")
+ logging.info(f"Writing data to {file_path}")
+ with open(file_path, 'a', newline='') as csvfile:
+ csvwriter = csv.writer(csvfile)
+ csvwriter.writerow(data)
+
+ @keyword
+ def write_cpu_to_csv(self, test_name, build_number, cpu_data):
+ data = [build_number,
+ cpu_data['cpu_events_per_second'],
+ cpu_data['min_latency'],
+ cpu_data['avg_latency'],
+ cpu_data['max_latency'],
+ cpu_data['cpu_events_per_thread'],
+ cpu_data['cpu_events_per_thread_stddev'],
+ self.device]
+ self._write_to_csv(test_name, data)
+
+ @keyword
+ def write_mem_to_csv(self, test_name, build_number, mem_data):
+ data = [build_number,
+ mem_data['operations_per_second'],
+ mem_data['data_transfer_speed'],
+ mem_data['min_latency'],
+ mem_data['avg_latency'],
+ mem_data['max_latency'],
+ mem_data['avg_events_per_thread'],
+ mem_data['events_per_thread_stddev'],
+ self.device]
+ self._write_to_csv(test_name, data)
+
+ @keyword
+ def read_cpu_csv_and_plot(self, test_name):
+ build_numbers = []
+ cpu_events_per_second = []
+ min_latency = []
+ avg_latency = []
+ max_latency = []
+ cpu_events_per_thread = []
+ cpu_events_per_thread_stddev = []
+
+ with open(f"{self.data_dir}{self.device}_{test_name}.csv", 'r') as csvfile:
+ csvreader = csv.reader(csvfile)
+ logging.info("Reading data from csv file...")
+ for row in csvreader:
+ if row[7] == self.device:
+ build_numbers.append(str(row[0]))
+ cpu_events_per_second.append(float(row[1]))
+ min_latency.append(float(row[2]))
+ avg_latency.append(float(row[3]))
+ max_latency.append(float(row[4]))
+ cpu_events_per_thread.append(float(row[5]))
+ cpu_events_per_thread_stddev.append(float(row[6]))
+
+ plt.figure(figsize=(20, 10))
+ plt.set_loglevel('WARNING')
+
+ # Plot 1: CPU Events per Second
+ plt.subplot(3, 1, 1)
+ plt.ticklabel_format(axis='y', style='plain')
+ plt.plot(build_numbers, cpu_events_per_second, marker='o', linestyle='-', color='b')
+ plt.title('CPU Events per Second', loc='right', fontweight="bold")
+ plt.ylabel('CPU Events per Second')
+ plt.grid(True)
+ plt.xticks(build_numbers)
+
+ # Plot 2: CPU Events per Thread
+ plt.subplot(3, 1, 2)
+ plt.ticklabel_format(axis='y', style='plain')
+ plt.plot(build_numbers, cpu_events_per_thread, marker='o', linestyle='-', color='b')
+ plt.title('CPU Events per Thread', loc='right', fontweight="bold")
+ plt.ylabel('CPU Events per Thread')
+ plt.grid(True)
+ plt.xticks(build_numbers)
+ # Create line chart with error bars on the same subplot
+ plt.errorbar(build_numbers, cpu_events_per_thread,
+ yerr=cpu_events_per_thread_stddev,
+ capsize=4)
+
+ # Plot 3: Latency
+ plt.subplot(3, 1, 3)
+ plt.ticklabel_format(axis='y', style='plain')
+ plt.plot(build_numbers, avg_latency, marker='o', linestyle='-', color='b', label='Avg')
+ plt.ylabel('Avg Latency (ms)')
+ plt.legend(loc='upper left')
+ plt.xlabel('Build Number')
+ plt.twinx()
+ plt.plot(build_numbers, max_latency, marker='o', linestyle='-', color='r', label='Max')
+ plt.plot(build_numbers, min_latency, marker='o', linestyle='-', color='g', label='Min')
+ plt.ylabel('Max/Min Latency (ms)')
+ plt.legend(loc='upper right')
+ plt.title('Latency', loc='right', fontweight="bold")
+ plt.grid(True)
+ plt.xticks(build_numbers)
+
+ plt.suptitle(f'{test_name} ({self.device})', fontsize=16, fontweight='bold')
+
+ plt.tight_layout()
+ plt.savefig(f'../test-suites/{self.device}_{test_name}.png') # Save the plot as an image file
+
+ @keyword
+ def read_mem_csv_and_plot(self, test_name):
+ build_numbers = []
+ operations_per_second = []
+ data_transfer_speed = []
+ min_latency = []
+ avg_latency = []
+ max_latency = []
+ avg_events_per_thread = []
+ events_per_thread_stddev = []
+
+ with open(f"{self.data_dir}{self.device}_{test_name}.csv", 'r') as csvfile:
+ csvreader = csv.reader(csvfile)
+ logging.info("Reading data from csv file...")
+ for row in csvreader:
+ if row[8] == self.device:
+ build_numbers.append(str(row[0]))
+ operations_per_second.append(float(row[1]))
+ data_transfer_speed.append(float(row[2]))
+ min_latency.append(float(row[3]))
+ avg_latency.append(float(row[4]))
+ max_latency.append(float(row[5]))
+ avg_events_per_thread.append(float(row[6]))
+ events_per_thread_stddev.append(float(row[7]))
+
+ plt.figure(figsize=(20, 10))
+ plt.set_loglevel('WARNING')
+
+ # Plot 1: Operations Per Second
+ plt.subplot(3, 1, 1)
+ plt.ticklabel_format(axis='y', style='sci', useMathText=True)
+ plt.plot(build_numbers, operations_per_second, marker='o', linestyle='-', color='b')
+ plt.title('Operations per Second', loc='right', fontweight="bold")
+ plt.ylabel('Operations per Second')
+ plt.grid(True)
+ plt.xticks(build_numbers)
+
+ # Plot 2: Data Transfer Speed
+ plt.subplot(3, 1, 2)
+ plt.ticklabel_format(axis='y', style='plain')
+ plt.plot(build_numbers, data_transfer_speed, marker='o', linestyle='-', color='b')
+ plt.title('Data Transfer Speed', loc='right', fontweight="bold")
+ plt.ylabel('Data Transfer Speed (MiB/sec)')
+ plt.grid(True)
+ plt.xticks(build_numbers)
+
+ # Plot 3: Latency
+ plt.subplot(3, 1, 3)
+ plt.ticklabel_format(axis='y', style='plain')
+ plt.plot(build_numbers, avg_latency, marker='o', linestyle='-', color='b', label='Avg')
+ plt.ylabel('Avg Latency (ms)')
+ plt.legend(loc='upper left')
+ plt.grid(True)
+ plt.xlabel('Build Number')
+ plt.twinx()
+ plt.plot(build_numbers, max_latency, marker='o', linestyle='-', color='r', label='Max')
+ plt.plot(build_numbers, min_latency, marker='o', linestyle='-', color='g', label='Min')
+ plt.ylabel('Max/Min Latency (ms)')
+ plt.legend(loc='upper right')
+ plt.title('Latency', loc='right', fontweight="bold")
+ plt.xticks(build_numbers)
+
+ plt.suptitle(f'{test_name} ({self.device})', fontsize=16, fontweight='bold')
+
+ plt.tight_layout()
+ plt.savefig(f'../test-suites/{self.device}_{test_name}.png') # Save the plot as an image file
+
+ @keyword
+ def save_cpu_data(self, test_name, build_number, cpu_data):
+
+ self.write_cpu_to_csv(test_name, build_number, cpu_data)
+ self.read_cpu_csv_and_plot(test_name)
+
+ @keyword
+ def save_memory_data(self, test_name, build_number, cpu_data):
+
+ self.write_mem_to_csv(test_name, build_number, cpu_data)
+ self.read_mem_csv_and_plot(test_name)
diff --git a/Robot-Framework/lib/output_parser.py b/Robot-Framework/lib/output_parser.py
index 3aa6edb..bfb36c2 100644
--- a/Robot-Framework/lib/output_parser.py
+++ b/Robot-Framework/lib/output_parser.py
@@ -48,3 +48,62 @@ def verify_date_format(date_string):
datetime.strptime(date_string, '%Y%m%d')
except ValueError:
raise Exception("Wrong date format in version date field")
+
+def parse_cpu_results(output):
+ def extract_value(pattern, output):
+ match = re.search(pattern, output)
+ if match:
+ return match.group(1)
+ else:
+ raise Exception(f"Couldn't parse result of the test with pattern: {pattern}")
+
+ output = re.sub(r'\033\[.*?m', '', output) # remove colors from serial console output
+
+ cpu_events_per_second = extract_value(r'events per second:\s*([.\d]+)', output)
+ min_latency = extract_value(r'min:\s+([.\d]+)', output)
+ max_latency = extract_value(r'max:\s+([.\d]+)', output)
+ avg_latency = extract_value(r'avg:\s+([.\d]+)', output)
+ cpu_events_per_thread = extract_value(r'events \(avg\/stddev\):\s+([.\d]+)', output)
+ cpu_events_per_thread_stddev = extract_value(r'events \(avg\/stddev\):\s+[.\d]+\/([.\d]+)', output)
+
+ cpu_data = {
+ 'cpu_events_per_second': cpu_events_per_second,
+ 'min_latency': min_latency,
+ 'max_latency': max_latency,
+ 'avg_latency': avg_latency,
+ 'cpu_events_per_thread': cpu_events_per_thread,
+ 'cpu_events_per_thread_stddev': cpu_events_per_thread_stddev
+ }
+
+ return cpu_data
+
+def parse_memory_results(output):
+ def extract_value(pattern, output):
+ match = re.search(pattern, output)
+ if match:
+ return match.group(1)
+ else:
+ raise Exception(f"Couldn't parse result of the test with pattern: {pattern}")
+
+ output = re.sub(r'\033\[.*?m', '', output) # remove colors from serial console output
+
+ operations_per_second = extract_value(r'Total operations:\s*\d+ \(([.\d]+) per second', output)
+ data_transfer_speed = extract_value(r'\(([.\d]+) MiB\/sec\)', output)
+ min_latency = extract_value(r'min:\s+([.\d]+)', output)
+ max_latency = extract_value(r'max:\s+([.\d]+)', output)
+ avg_latency = extract_value(r'avg:\s+([.\d]+)', output)
+ avg_events_per_thread = extract_value(r'events \(avg\/stddev\):\s+([.\d]+)', output)
+ events_per_thread_stddev = extract_value(r'events \(avg\/stddev\):\s+[.\d]+\/([.\d]+)', output)
+
+ mem_data = {
+ 'operations_per_second': operations_per_second,
+ 'data_transfer_speed': data_transfer_speed,
+ 'min_latency': min_latency,
+ 'max_latency': max_latency,
+ 'avg_latency': avg_latency,
+ 'avg_events_per_thread': avg_events_per_thread,
+ 'events_per_thread_stddev': events_per_thread_stddev
+ }
+
+ return mem_data
+
diff --git a/Robot-Framework/resources/ssh_keywords.resource b/Robot-Framework/resources/ssh_keywords.resource
index 8326a8a..fc4ffc7 100644
--- a/Robot-Framework/resources/ssh_keywords.resource
+++ b/Robot-Framework/resources/ssh_keywords.resource
@@ -223,3 +223,16 @@ Check if ssh is ready on netvm
Sleep 1
END
IF ${status} == False FAIL Port 22 of NetVM is not ready after ${timeout}
+
+Install sysbench tool
+ ${command_output}= Execute Command nix-env --query --installed
+ ${not_installed} = Run Keyword And Return Status Should Not Contain ${command_output} sysbench
+ IF ${not_installed}
+ Execute Command nix-env -i sysbench
+ ${command_output}= Execute Command nix-env --query --installed
+ Log To Console ${\n}Installed packages:${\n}${command_output}
+ Should Contain ${command_output} sysbench sysbench tool was not installed
+ Log To Console sysbench tool was succesfully installed
+ ELSE
+ Log To Console ${\n}sysbench tool was already installed
+ END
diff --git a/Robot-Framework/test-suites/performance/performance.robot b/Robot-Framework/test-suites/performance/performance.robot
new file mode 100644
index 0000000..6768739
--- /dev/null
+++ b/Robot-Framework/test-suites/performance/performance.robot
@@ -0,0 +1,93 @@
+# SPDX-FileCopyrightText: 2022-2023 Technology Innovation Institute (TII)
+# SPDX-License-Identifier: Apache-2.0
+
+*** Settings ***
+Documentation Gathering performance data
+Force Tags performance
+Resource ../../resources/ssh_keywords.resource
+Resource ../../config/variables.robot
+Library ../../lib/output_parser.py
+Library ../../lib/PerformanceDataProcessing.py ${DEVICE}
+Suite Setup Common Setup
+Suite Teardown Close All Connections
+
+*** Test Cases ***
+
+CPU One thread test
+ [Documentation] Run a CPU benchmark using Sysbench with a duration of 10 seconds and a SINGLE thread.
+ ... The benchmark records to csv CPU events per second, events per thread, and latency data.
+ ... Create visual plots to represent these metrics comparing to previous tests.
+ [Tags] performance cpu SP-T67-1
+ ${output} Execute Command sysbench cpu --time=10 --threads=1 --cpu-max-prime=20000 run
+ Log ${output}
+ &{cpu_data} Parse Cpu Results ${output}
+ Save Cpu Data ${TEST NAME} ${buildID} ${cpu_data}
+ Log 
HTML
+
+CPU multimple threads test
+ [Documentation] Run a CPU benchmark using Sysbench with a duration of 10 seconds and MULTIPLE threads.
+ ... The benchmark records to csv CPU events per second, events per thread, and latency data.
+ ... Create visual plots to represent these metrics comparing to previous tests.
+ [Tags] performance cpu SP-T67-2
+ ${output} Execute Command sysbench cpu --time=10 --threads=${threads_number} --cpu-max-prime=20000 run
+ Log ${output}
+ &{cpu_data} Parse Cpu Results ${output}
+ Save Cpu Data ${TEST NAME} ${buildID} ${cpu_data}
+ Log HTML
+
+Memory Read One thread test
+ [Documentation] Run a memory benchmark using Sysbench for 60 seconds with a SINGLE thread.
+ ... The benchmark records Operations Per Second, Data Transfer Speed, Average Events per Thread,
+ ... and Latency for READ operations.
+ ... Create visual plots to represent these metrics comparing to previous tests.
+ [Tags] performance cpu SP-T67-3
+ ${output} Execute Command sysbench memory --time=60 --memory-oper=read --threads=1 run
+ Log ${output}
+ &{cpu_data} Parse Memory Results ${output}
+ Save Memory Data ${TEST NAME} ${buildID} ${cpu_data}
+ Log HTML
+
+Memory Write One thread test
+ [Documentation] Run a memory benchmark using Sysbench for 60 seconds with a SINGLE thread.
+ ... The benchmark records Operations Per Second, Data Transfer Speed, Average Events per Thread,
+ ... and Latency for WRITE operations.
+ ... Create visual plots to represent these metrics comparing to previous tests.
+ [Tags] performance cpu SP-T67-4
+ ${output} Execute Command sysbench memory --time=60 --memory-oper=write --threads=1 run
+ Log ${output}
+ &{cpu_data} Parse Memory Results ${output}
+ Save Memory Data ${TEST NAME} ${buildID} ${cpu_data}
+ Log HTML
+
+Memory Read multimple threads test
+ [Documentation] Run a memory benchmark using Sysbench for 60 seconds with MULTIPLE threads.
+ ... The benchmark records Operations Per Second, Data Transfer Speed, Average Events per Thread,
+ ... and Latency for READ operations.
+ ... Create visual plots to represent these metrics comparing to previous tests.
+ [Tags] performance cpu SP-T67-5
+ ${output} Execute Command sysbench memory --time=60 --memory-oper=read --threads=${threads_number} run
+ Log ${output}
+ &{cpu_data} Parse Memory Results ${output}
+ Save Memory Data ${TEST NAME} ${buildID} ${cpu_data}
+ Log HTML
+
+Memory Write multimple threads test
+ [Documentation] Run a memory benchmark using Sysbench for 60 seconds with MULTIPLE threads.
+ ... The benchmark records Operations Per Second, Data Transfer Speed, Average Events per Thread,
+ ... and Latency for WRITE operations.
+ ... Create visual plots to represent these metrics comparing to previous tests.
+ [Tags] performance cpu SP-T67-6
+ ${output} Execute Command sysbench memory --time=60 --memory-oper=write --threads=${threads_number} run
+ Log ${output}
+ &{cpu_data} Parse Memory Results ${output}
+ Save Memory Data ${TEST NAME} ${buildID} ${cpu_data}
+ Log HTML
+
+
+
+*** Keywords ***
+
+Common Setup
+ Set Variables ${DEVICE}
+ Connect
+ Install sysbench tool
diff --git a/requirements b/requirements
index f237bf3..449e49a 100644
--- a/requirements
+++ b/requirements
@@ -6,3 +6,4 @@ robotframework==5.0.1
PyP100==0.1.2
python-kasa~=0.5.1
robotframework-retryfailed==0.2.0
+matplotlib==3.7.2