Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add CO2 formula values to Report #13

Closed
wants to merge 25 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
c6284ef
add energy consumpiton plots
mirpedrol Jun 13, 2023
8a03833
add PackageScope for all fields
mirpedrol Jun 14, 2023
1ff29ab
show energy plot
mirpedrol Jun 14, 2023
f2277c3
add energy consumpiton plots
mirpedrol Jun 13, 2023
d1da4aa
add PackageScope for all fields
mirpedrol Jun 14, 2023
ab3e9c5
show energy plot
mirpedrol Jun 14, 2023
cef204f
add functions to obtain values from CO2Report in readable units as st…
mirpedrol Jun 15, 2023
69770d9
remove redundant code in report template JS
mirpedrol Jun 15, 2023
6ac9ba0
remove original report values from table
mirpedrol Jun 15, 2023
4cb7594
add table values for co2
mirpedrol Jun 16, 2023
a5cd1ec
remove log print
mirpedrol Jun 16, 2023
8ad4237
Merge branch 'update-report' of https://github.com/mirpedrol/nf-co2fo…
mirpedrol Jun 16, 2023
765ae2f
add human readable units to report table
mirpedrol Jun 16, 2023
463700f
change plot units
mirpedrol Jun 16, 2023
1a3002c
don't remove zeros from plots and don't round
mirpedrol Jun 16, 2023
c0ff38e
add readable units plots in a new tab
mirpedrol Jun 16, 2023
85a8f49
add energy unit index
mirpedrol Jun 19, 2023
ac89ca6
add energy unit index
mirpedrol Jun 19, 2023
c91c46f
add all formula values to output table
mirpedrol Jun 19, 2023
17e7a42
add formula values to report table - not workign yet
mirpedrol Jun 19, 2023
514ddd8
change FIELDS order to fix output
mirpedrol Jun 20, 2023
b94c243
fix time humamize
mirpedrol Jun 20, 2023
eb0a10b
round all values
mirpedrol Jun 20, 2023
5d20ae7
do not show constant values in report table
mirpedrol Jun 21, 2023
d4cea62
Merge branch 'dev' into add-report-formula-values
mirpedrol Aug 8, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ class CO2FootprintFactory implements TraceObserverFactory {
e = e * 1000000
c = c * 1000

return [e, c]
return [e, c, t as Double, nc as Double, pc as Double, uc as Double, nm as Double, pm as Double, pue as Double, ci as Double]
}


Expand Down Expand Up @@ -277,7 +277,7 @@ class CO2FootprintFactory implements TraceObserverFactory {
writer = new Agent<PrintWriter>(co2eFile)
summaryWriter = new Agent<PrintWriter>(co2eSummaryFile)

writer.send { co2eFile.println("task_id\tenergy_consumption\tCO2e"); co2eFile.flush() }
writer.send { co2eFile.println("task_id\tenergy_consumption\tCO2e\ttime\tnumber_of_cores\tpower_draw_of_a_computing_core\tcore_usage_factor\tsize_of_memory_available\tpower_draw_of_memory\tefficiency_coefficient_of_the_data_center\tcarbon_intensity"); co2eFile.flush() }
}

/**
Expand Down Expand Up @@ -346,15 +346,24 @@ class CO2FootprintFactory implements TraceObserverFactory {
def computation_results = computeTaskCO2footprint(trace)
def eConsumption = computation_results[0]
def co2 = computation_results[1]



co2eRecords[taskId] = new CO2Record((Double) eConsumption, (Double) co2, trace.get('name').toString())
def t = computation_results[2]
def nc = computation_results[3]
def pc = computation_results[4]
def uc = computation_results[5]
def nm = computation_results[6]
def pm = computation_results[7]
def pue = computation_results[8]
def ci = computation_results[9]

def Map valuesMap = new LinkedHashMap<>(['energy': eConsumption, 'co2e': co2, 'name': trace.get('name').toString(), 'time': t, 'cores': nc, 'core_power': pc, 'core_usage': uc, 'memory': nm, 'memory_power': pm, 'pue': pue, 'ci': ci])

co2eRecords[taskId] = new CO2Record((Double) eConsumption, (Double) co2, trace.get('name').toString(), valuesMap)
total_energy += eConsumption
total_co2 += co2

// save to the file
writer.send { PrintWriter it -> it.println("${taskId}\t${HelperFunctions.convertToReadableUnits(eConsumption,3)}Wh\t${HelperFunctions.convertToReadableUnits(co2,3)}g"); it.flush() }
writer.send { PrintWriter it -> it.println("${taskId}\t${HelperFunctions.convertToReadableUnits(eConsumption,3)}Wh\t${HelperFunctions.convertToReadableUnits(co2,3)}g\t\
${t} hours\t${nc}\t${pc}\t${uc}\t${HelperFunctions.convertToReadableUnits(nm,8)}B\t${HelperFunctions.convertToReadableUnits(pm)}W\t${pue}\t${HelperFunctions.convertToReadableUnits(ci)}g/kWh"); it.flush() }
}


Expand All @@ -370,12 +379,24 @@ class CO2FootprintFactory implements TraceObserverFactory {
def computation_results = computeTaskCO2footprint(trace)
def eConsumption = computation_results[0]
def co2 = computation_results[1]
co2eRecords[taskId] = new CO2Record((Double) eConsumption, (Double) co2, trace.get('name').toString())
def t = computation_results[2]
def nc = computation_results[3]
def pc = computation_results[4]
def uc = computation_results[5]
def nm = computation_results[6]
def pm = computation_results[7]
def pue = computation_results[8]
def ci = computation_results[9]

def Map valuesMap = new LinkedHashMap<>(['energy': eConsumption, 'co2e': co2, 'name': trace.get('name').toString(), 'time': t, 'cores': nc, 'core_power': pc, 'core_usage': uc, 'memory': nm, 'memory_power': pm, 'pue': pue, 'ci': ci])

co2eRecords[taskId] = new CO2Record((Double) eConsumption, (Double) co2, trace.get('name').toString(), valuesMap)
total_energy += eConsumption
total_co2 += co2

// save to the file
writer.send { PrintWriter it -> it.println("${taskId}\t${HelperFunctions.convertToReadableUnits(eConsumption,3)}Wh\t${HelperFunctions.convertToReadableUnits(co2,3)}g"); it.flush() }
writer.send { PrintWriter it -> it.println("${taskId}\t${HelperFunctions.convertToReadableUnits(eConsumption,3)}Wh\t${HelperFunctions.convertToReadableUnits(co2,3)}g\t\
${t} hours\t${nc}\t${pc}\t${uc}\t${HelperFunctions.convertToReadableUnits(nm,8)}B\t${HelperFunctions.convertToReadableUnits(pm)}W\t${pue}\t${HelperFunctions.convertToReadableUnits(ci)}g/kWh"); it.flush() }
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import groovy.util.logging.Slf4j
import nextflow.trace.TraceRecord
import groovy.json.StringEscapeUtils


import nextflow.co2footprint.HelperFunctions

/**
Expand All @@ -21,17 +20,25 @@ class CO2Record extends TraceRecord {
private String name
// final? or something? to make sure for key value can be set only once?

CO2Record(Double energy, Double co2e, String name) {
CO2Record(Double energy, Double co2e, String name, Map store=[:]) {
this.energy = energy
this.co2e = co2e
this.name = name
this.store = new LinkedHashMap<>(['energy': energy, 'co2e': co2e, 'name': name])
}

final public static Map<String,String> FIELDS = [
co2e: 'num',
energy: 'num',
name: 'str'
co2e: 'num',
energy: 'num',
time: 'num',
cores: 'num',
core_power: 'num',
core_usage: 'num',
memory: 'num',
memory_power: 'num',
pue: 'num',
ci: 'num',
name: 'str'
]

// TODO implement accordingly to TraceRecord
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package nextflow.co2footprint

public class HelperFunctions {

static public String convertToReadableUnits(double value, int unitIndex=4) {
static public String convertToReadableUnits(Number value, int unitIndex=4) {
def units = ['p', 'n', 'u', 'm', ' ', 'K', 'M', 'G', 'T', 'P', 'E'] // Units: pico, nano, micro, mili, 0, Kilo, Mega, Giga, Tera, Peta, Exa

while (value >= 1000 && unitIndex < units.size() - 1) {
Expand All @@ -14,7 +14,7 @@ public class HelperFunctions {
unitIndex--
}

return "${value}${units[unitIndex]}"
return "${value} ${units[unitIndex]}"
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -56,16 +56,30 @@ $(function() {
window.data_byprocess[proc][key].push(metrics[key].q3);
window.data_byprocess[proc][key].push(metrics[key].max);
}
if (key == "time") {
window.data_byprocess[proc][key] = window.data_byprocess[proc][key].map(function(d,i){
return moment.duration(d).asMinutes().toFixed(1);
});
}
}
}
}

// Convert to readable units for plots
function readable_units_value(value, unit_index) {
units = ['p', 'n', 'u', 'm', ' ', 'K', 'M', 'G', 'T', 'P', 'E'] // Units: pico, nano, micro, mili, 0, Kilo, Mega, Giga, Tera, Peta, Exa

while (value >= 1000 && unit_index < units.length - 1) {
value /= 1000;
unit_index++;
}
while (value <= 1 && unit_index > 0) {
value *= 1000;
unit_index--;
}

return [ value, units[unit_index] ];
}

// Plot histograms of resource usage
//// Co2e
var co2e_data = [];
var energy_data = [];
for(var pname in window.data_byprocess){
Expand Down Expand Up @@ -126,11 +140,18 @@ $(function() {
if (duration.days() > 0) {
return duration.days() + "d " + duration.hours() + "h"
}
if (duration.hours() > 0) {
return duration.hours() + "h " + duration.minutes() + "m"
if (duration >= 1) {
minutes = Math.floor(hours % 60);
return Math.floor(duration) + "h" + minutes + "m";
}
if (duration.minutes() > 0) {
return duration.minutes() + "m " + duration.seconds() + "s"
if (duration < 1) {
minutes = Math.floor(duration * 60);
if (minutes >= 1) {
seconds = Math.floor(duration * 60 % 60);
return minutes + "m " + seconds + "s";
}
seconds = Math.floor(duration * 60 * 60);
return seconds.toFixed(1) + "s";
}
return duration.asSeconds().toFixed(1) + "s"
}*/
Expand Down Expand Up @@ -170,9 +191,9 @@ $(function() {
if (ms == '-' || ms == 0){
return ms;
}
return humanize(moment.duration( parseInt(ms) ));
return readable_units(ms, 4) + 'g';
}
function make_date(ms, type){
function make_energy(ms, type){
if (type === 'sort') {
return parseInt(ms);
}
Expand All @@ -182,22 +203,23 @@ $(function() {
if (ms == '-' || ms == 0){
return ms;
}
return moment( parseInt(ms) ).format();
return readable_units(ms, 5) + 'Wh';
}
function make_memory(bytes, type){
function make_time(ms, type){
if (type === 'sort') {
return parseInt(bytes);
return parseInt(ms);
}
if($('#nf-table-humanreadable').val() == 'false'){
return bytes;
return ms;
}
if (bytes == '-' || bytes == 0){
return bytes;
if (ms == '-' || ms == 0){
return ms;
}
// https://stackoverflow.com/a/14919494
var thresh = 1024;
if(Math.abs(bytes) < thresh) {
return bytes + ' B';
return humanize(ms);
}
function make_memory(ms, type){
if (type === 'sort') {
return parseInt(ms);
}
var units = ['kB','MB','GB','TB','PB','EB','ZB','YB'];
var u = -1;
Expand Down Expand Up @@ -254,14 +276,22 @@ $(function() {
},
{ title: co2EmissionsTitle, data: 'co2e', render: make_co2e },
{ title: energyConsumptionTitle, data: 'energy', render: make_energy },
{ title: 'Time', data: 'time', render: make_time },
{ title: 'Number of cores', data: 'cores' },
{ title: 'Power draw of a computing core', data: 'core_power' },
{ title: 'Core usage factor', data: 'core_usage' },
{ title: 'Size of memory available', data: 'memory', render: make_memory },
//{ title: 'Power draw of memory', data: 'memory_power', render: make_index0 },
//{ title: 'Efficiency coefficient of the data center', data: 'pue' },
//{ title: 'Carbon intensity', data: 'ci', render: make_index0 },
],
"deferRender": true,
"lengthMenu": [[25, 50, 100, -1], [25, 50, 100, "All"]],
"scrollX": true,
"colReorder": true,
"columnDefs": [
{ className: "id", "targets": [ 0,1,2,3 ] },
{ className: "meta", "targets": [ 4 ] },
{ className: "meta", "targets": [ 4,7,8,9,10,11 ] },
{ className: "metrics", "targets": [ 5,6 ] }
],
"buttons": [
Expand Down
Loading