Merge branch 'dev' into add-report-formula-values

buildSrc/src/main/groovy/io.nextflow.groovy-common-conventions.gradle

@@ -14,7 +14,7 @@ repositories {
 
 java {
     toolchain {
-        languageVersion = JavaLanguageVersion.of(17)
+        languageVersion = JavaLanguageVersion.of(19)
     }
 }
 

plugins/nf-co2footprint/build.gradle

@@ -50,7 +50,7 @@ sourceSets {
 }
 
 ext{
-    nextflowVersion = '22.10.0'
+    nextflowVersion = '23.07.0-edge'
 }
 
 dependencies {
@@ -61,10 +61,10 @@ dependencies {
     // add here plugins depepencies
 
     // test configuration
-    testImplementation "org.codehaus.groovy:groovy:3.0.10"
-    testImplementation "org.codehaus.groovy:groovy-nio:3.0.10"
+    testImplementation "org.codehaus.groovy:groovy:3.0.17"
+    testImplementation "org.codehaus.groovy:groovy-nio:3.0.17"
     testImplementation "io.nextflow:nextflow:$nextflowVersion"
-    testImplementation ("org.codehaus.groovy:groovy-test:3.0.10") { exclude group: 'org.codehaus.groovy' }
+    testImplementation ("org.codehaus.groovy:groovy-test:3.0.17") { exclude group: 'org.codehaus.groovy' }
     testImplementation ("cglib:cglib-nodep:3.3.0")
     testImplementation ("org.objenesis:objenesis:3.1")
     testImplementation ("org.spockframework:spock-core:2.2-groovy-3.0") { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' }

plugins/nf-co2footprint/src/main/nextflow/co2footprint/CO2FootprintConfig.groovy

@@ -11,28 +11,70 @@ import groovy.transform.PackageScope
  * co2footprint {
  *     file = "co2footprint.txt"
  *     summaryFile = "co2footprint.summary.txt"
+ *     ci = 300
+ *     pue = 1.4
+ *     powerdrawMem = 0.67
  * }
  *
  *
  * We anotate this class as @PackageScope to restrict the access of their methods only to class in the
  * same package
  *
- * @author : Sabrina Krakau <[email protected]>
+ * @author Júlia Mir Pedrol <[email protected]>, Sabrina Krakau <[email protected]>
  *
  */
 @PackageScope
 class CO2FootprintConfig {
 
     final private String  file
     final private String  summaryFile
+    final private String  reportFile
+    final private Double  ci    // CI: carbon intensity
+    final private Double  pue   // PUE: power usage effectiveness efficiency, coefficient of the data centre
+    final private Double  powerdrawMem  // Power draw of memory [W per GB]
+
+    // Retrieve CI value from file containing CI values for different locations
+    protected Double retrieveCi(String country) {
+        def dataReader = new InputStreamReader(this.class.getResourceAsStream('/ci_values.csv'))
+
+        Double localCi = 0.0
+        String line
+        while ( line = dataReader.readLine() ) {
+            def row = line.split(",")
+            if (row[0] == country) {
+                localCi = row[1].toFloat()
+                break
+            }
+        }
+        dataReader.close()
+        if (localCi == 0.0)
+            throw new IllegalArgumentException("Invalid 'country' parameter: $country. Could not be found in 'ci_values.csv'.")
+
+        return localCi
+    }
 
     CO2FootprintConfig(Map map){
         def config = map ?: Collections.emptyMap()
         file = config.file ?: CO2FootprintFactory.CO2FootprintTextFileObserver.DEF_FILE_NAME
         summaryFile = config.summaryFile ?: CO2FootprintFactory.CO2FootprintTextFileObserver.DEF_SUMMARY_FILE_NAME
+        reportFile = config.reportFile ?: CO2FootprintFactory.CO2FootprintReportObserver.DEF_REPORT_FILE_NAME
+
+        ci = 475
+        if (config.ci && config.country)
+            throw new IllegalArgumentException("Invalid combination of 'ci' and 'country' parameters specified for the CO2Footprint plugin. Please specify either 'ci' and 'country'!")
+        if (config.ci)
+            ci = config.ci
+        if (config.country)
+            ci = retrieveCi(config.country)
+
+        pue = config.pue ?: 1.67
+        powerdrawMem = config.powerdrawMem ?: 0.3725
     }
 
     String getFile() { file }
-
     String getSummaryFile() { summaryFile }
+    String getReportFile() { reportFile }
+    Double getCI() { ci }
+    Double getPUE() { pue }
+    Double getPowerdrawMem() { powerdrawMem }
 }

plugins/nf-co2footprint/src/main/nextflow/co2footprint/CO2FootprintFactory.groovy

@@ -42,7 +42,7 @@ import java.util.concurrent.ConcurrentHashMap
 /**
  * Implements the CO2Footprint observer factory
  *
- * @author Sabrina Krakau <[email protected]>
+ * @author Júlia Mir Pedrol <[email protected]>, Sabrina Krakau <[email protected]>
  */
 @Slf4j
 @CompileStatic
@@ -61,12 +61,14 @@ class CO2FootprintFactory implements TraceObserverFactory {
 
     // Load file containing TDP values for different CPU models
     protected void loadCpuTdpData(Map<String, Float> data) {
-        def inData = new InputStreamReader(this.class.getResourceAsStream('/cpu_tdp_values.csv')).text
+        def dataReader = new InputStreamReader(this.class.getResourceAsStream('/cpu_tdp_values.csv'))
 
-        for (String line : inData.readLines()) {
+        String line
+        while ( line = dataReader.readLine() ) {
             def h = line.split(",")
             if (h[0] != 'model_name') data[h[0]] = h[3].toFloat()
         }
+        dataReader.close()
         log.info "$data"
     }
 
@@ -84,7 +86,7 @@ class CO2FootprintFactory implements TraceObserverFactory {
         result.add( new CO2FootprintTextFileObserver(co2eFile, co2eSummaryFile) )
 
         // Generate CO2 footprint report with box-plot
-        def co2eReport = (CO2FootprintReportObserver.DEF_FILE_NAME as Path).complete()
+        def co2eReport = (this.config.getReportFile() as Path).complete()
         result.add( new CO2FootprintReportObserver(co2eReport) )
 
         return result
@@ -168,15 +170,15 @@ class CO2FootprintFactory implements TraceObserverFactory {
         // TODO handle if more memory/cpus used than requested?
 
         // Pm: power draw of memory [W per GB]
-        def pm  = 0.3725
+        def pm  = config.getPowerdrawMem()
 
         /**
          * Remaining factors
          */
         // PUE: efficiency coefficient of the data centre
-        def pue = 1.67
+        def pue = config.getPUE()
         // CI: carbon intensity [gCO2e kWh−1]
-        def ci  = 475
+        def ci  = config.getCI()
 
         /**
          * Calculate energy consumption [kWh]
@@ -190,6 +192,10 @@ class CO2FootprintFactory implements TraceObserverFactory {
         def Double c = (e * ci) as Double
         log.info "CO2: $c"
 
+        // Return values in mWh and mg
+        e = e * 1000000
+        c = c * 1000
+
         return [e, c, t as Double, nc as Double, pc as Double, uc as Double, nm as Double, pm as Double, pue as Double, ci as Double]
     }
 
@@ -286,8 +292,8 @@ class CO2FootprintFactory implements TraceObserverFactory {
 
             //writer.send { co2eFile.println("Test CO2 emission is:"); co2eFile.flush() }
             //writer.send { PrintWriter it -> it.println("Test CO2 emission is:"); it.flush() }
-            co2eSummaryFile.println("The total CO2 emission is: ${HelperFunctions.convertToReadableUnits(total_co2)}g")
-            co2eSummaryFile.println("The total energy consumption is: ${HelperFunctions.convertToReadableUnits(total_energy,5)}Wh")
+            co2eSummaryFile.println("The total CO2 emission is: ${HelperFunctions.convertToReadableUnits(total_co2,3)}g")
+            co2eSummaryFile.println("The total energy consumption is: ${HelperFunctions.convertToReadableUnits(total_energy,3)}Wh")
             co2eSummaryFile.flush()
             co2eSummaryFile.close()
 
@@ -356,7 +362,7 @@ class CO2FootprintFactory implements TraceObserverFactory {
             total_co2 += co2
 
             // save to the file
-            writer.send { PrintWriter it -> it.println("${taskId}\t${HelperFunctions.convertToReadableUnits(eConsumption,5)}Wh\t${HelperFunctions.convertToReadableUnits(co2)}g\t\
+            writer.send { PrintWriter it -> it.println("${taskId}\t${HelperFunctions.convertToReadableUnits(eConsumption,3)}Wh\t${HelperFunctions.convertToReadableUnits(co2,3)}g\t\
             ${t} hours\t${nc}\t${pc}\t${uc}\t${HelperFunctions.convertToReadableUnits(nm,8)}B\t${HelperFunctions.convertToReadableUnits(pm)}W\t${pue}\t${HelperFunctions.convertToReadableUnits(ci)}g/kWh"); it.flush() }
         }
 
@@ -389,7 +395,7 @@ class CO2FootprintFactory implements TraceObserverFactory {
             total_co2 += co2
 
             // save to the file
-            writer.send { PrintWriter it -> it.println("${taskId}\t${HelperFunctions.convertToReadableUnits(eConsumption,5)}Wh\t${HelperFunctions.convertToReadableUnits(co2)}g\t\
+            writer.send { PrintWriter it -> it.println("${taskId}\t${HelperFunctions.convertToReadableUnits(eConsumption,3)}Wh\t${HelperFunctions.convertToReadableUnits(co2,3)}g\t\
             ${t} hours\t${nc}\t${pc}\t${uc}\t${HelperFunctions.convertToReadableUnits(nm,8)}B\t${HelperFunctions.convertToReadableUnits(pm)}W\t${pue}\t${HelperFunctions.convertToReadableUnits(ci)}g/kWh"); it.flush() }
         }
     }
@@ -400,7 +406,7 @@ class CO2FootprintFactory implements TraceObserverFactory {
      */
     class CO2FootprintReportObserver implements TraceObserver {
 
-        static final public String DEF_FILE_NAME = "CO2Footprint-report-${TraceHelper.launchTimestampFmt()}.html"
+        static final public String DEF_REPORT_FILE_NAME = "co2footprint-report-${TraceHelper.launchTimestampFmt()}.html"
 
         static final public int DEF_MAX_TASKS = 10_000
 
@@ -642,6 +648,7 @@ class CO2FootprintFactory implements TraceObserverFactory {
                             readTemplate('assets/CO2FootprintReportTemplate.js')
                     ]
             ]
+            //log.info "${tpl_fields['payload']}"
             final tpl = readTemplate('CO2FootprintReportTemplate.html')
             def engine = new GStringTemplateEngine()
             def html_template = engine.createTemplate(tpl)

plugins/nf-co2footprint/src/main/nextflow/co2footprint/CO2FootprintReportSummary.groovy

@@ -24,7 +24,7 @@ import nextflow.trace.TraceRecord
 /**
  * Model a process summary data used to render box-plots in the execution HTML report
  *
- * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
+ * @author Júlia Mir Pedrol <[email protected]>, Sabrina Krakau <sabrinakrakau@gmail.com>
  */
 
 @Slf4j
@@ -202,9 +202,12 @@ class CO2FootprintReportSummary {
             result.q3 = quantile(sorted, 75)
             result.max = quantile(sorted, 100)
 
-            // discard entry with all zero 
-            //if( result.min == 0 && result.min == result.max  )
-            //    return null
+            /* 
+                Unlike the Nextflow Report, we are not rounding the results nor discarding entries with all zeros.
+                This decision is taken to avoid the loss of information in the report. 
+                Plots will show values of 0, making the representation of all processes consistent.
+                This class reports all values in mili-unit to increase precision. Values are converted to the required units by CO2FootprintReportTemplate.js
+            */           
 
             result.minLabel = minLabel
             result.maxLabel = maxLabel

plugins/nf-co2footprint/src/main/nextflow/co2footprint/CO2FootprintResourcesAggregator.groovy

@@ -14,7 +14,7 @@ import java.util.concurrent.Future
 /**
  * Collect and aggregate execution metrics used by execution report
  * 
- * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
+ * @author Júlia Mir Pedrol <[email protected]>, Sabrina Krakau <sabrinakrakau@gmail.com>
  */
 @Slf4j
 @CompileStatic

plugins/nf-co2footprint/src/main/nextflow/co2footprint/CO2Record.groovy

@@ -5,9 +5,12 @@ import groovy.util.logging.Slf4j
 import nextflow.trace.TraceRecord
 import groovy.json.StringEscapeUtils
 
-
 import nextflow.co2footprint.HelperFunctions
 
+/**
+ *
+ * @author Júlia Mir Pedrol <[email protected]>, Sabrina Krakau <[email protected]>
+ */
 @Slf4j
 @CompileStatic
 class CO2Record extends TraceRecord {
@@ -21,7 +24,7 @@ class CO2Record extends TraceRecord {
         this.energy = energy
         this.co2e = co2e
         this.name = name
-        this.store = store
+        this.store = new LinkedHashMap<>(['energy': energy, 'co2e': co2e, 'name': name])
     }
 
     final public static Map<String,String> FIELDS = [
@@ -40,8 +43,8 @@ class CO2Record extends TraceRecord {
 
     // TODO implement accordingly to TraceRecord
     Double getEnergyConsumption() { energy }
-    String getEnergyConsumptionReadable() { HelperFunctions.convertToReadableUnits(energy,5) }
-    String getCO2eReadable() { HelperFunctions.convertToReadableUnits(co2e) }
+    String getEnergyConsumptionReadable() { HelperFunctions.convertToReadableUnits(energy,3) }
+    String getCO2eReadable() { HelperFunctions.convertToReadableUnits(co2e,3) }
     Double getCO2e() { co2e }
     String getName() { name }