From 6d77587019d3b9a8f1c4c90cba556859b97e4d9c Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 23 Sep 2024 18:27:24 +0200 Subject: [PATCH] fix: nextflow_poll_period_secdonds now minutes. And better nextflow ansible task names --- .../tasks/nextflow-dedup-file-type.yaml | 20 ++++++++++++------- .../tasks/nextflow-merge-file-type.yaml | 20 ++++++++++++------- ansible/roles/combine/vars/main.yaml | 2 +- .../roles/fragment/tasks/fragmentation.yaml | 20 ++++++++----------- .../tasks/gen-isomol-inchi-database.yaml | 15 +++++++------- .../tasks/gen-nonisomol-inchi-database.yaml | 10 +++++----- .../standardise/tasks/standardise-file.yaml | 20 ++++++++----------- 7 files changed, 56 insertions(+), 51 deletions(-) diff --git a/ansible/roles/combine/tasks/nextflow-dedup-file-type.yaml b/ansible/roles/combine/tasks/nextflow-dedup-file-type.yaml index 14fe580..df9f302 100644 --- a/ansible/roles/combine/tasks/nextflow-dedup-file-type.yaml +++ b/ansible/roles/combine/tasks/nextflow-dedup-file-type.yaml @@ -81,7 +81,7 @@ when: - not combined_file.stat.exists -- name: Set nextflow command +- name: Set nextflow command (Deduplicate) set_fact: nextflow_cmd: >- nextflow run -c {{ reppath }}/nextflow/nextflow.config @@ -91,13 +91,19 @@ when: - not combined_file.stat.exists -- name: Display nextflow command +- name: Display nextflow command (Deduplicate) debug: var: nextflow_cmd when: - not combined_file.stat.exists -- name: Run nextflow (async) +- name: Display nextflow command timeout (Deduplicate) + debug: + msg: nextflow_timeout_minutes={{ nextflow_timeout_minutes }} nextflow_poll_period_minutes={{ nextflow_poll_period_minutes }} + when: + - not combined_file.stat.exists + +- name: Run nextflow (Deduplicate) (async) command: "{{ nextflow_cmd }}" async: "{{ nextflow_timeout_minutes|int * 60 }}" poll: 0 @@ -105,13 +111,13 @@ when: - not combined_file.stat.exists -- name: Wait for nextflow (async) +- name: Wait for nextflow (Deduplicate) async_status: jid: "{{ nextflow_async.ansible_job_id }}" register: nextflow_result until: nextflow_result.finished - delay: "{{ nextflow_poll_period_seconds|int }}" - retries: "{{ (nextflow_timeout_minutes|int / (nextflow_poll_period_seconds|int / 60))|int }}" + delay: "{{ nextflow_poll_period_minutes | int * 60 }}" + retries: "{{ (nextflow_timeout_minutes | int / nextflow_poll_period_minutes | int) | int }}" when: - not combined_file.stat.exists @@ -149,7 +155,7 @@ register: compress_task when: not file_type_file.stat.exists -- name: Add this tasy to the list of compress tasks +- name: Add this task to the list of compress tasks set_fact: compress_tasks: "{{ compress_tasks + [ compress_task ] }}" when: not file_type_file.stat.exists \ No newline at end of file diff --git a/ansible/roles/combine/tasks/nextflow-merge-file-type.yaml b/ansible/roles/combine/tasks/nextflow-merge-file-type.yaml index 5f67cd2..ae25056 100644 --- a/ansible/roles/combine/tasks/nextflow-merge-file-type.yaml +++ b/ansible/roles/combine/tasks/nextflow-merge-file-type.yaml @@ -81,7 +81,7 @@ when: - not combined_file.stat.exists -- name: Set nextflow command +- name: Set nextflow command (Merge) set_fact: nextflow_cmd: >- nextflow run -c {{ reppath }}/nextflow/nextflow.config @@ -91,13 +91,19 @@ when: - not combined_file.stat.exists -- name: Display nextflow command +- name: Display nextflow command (Merge) debug: var: nextflow_cmd when: - not combined_file.stat.exists -- name: Run nextflow (async) +- name: Display nextflow command timeout (Merge) + debug: + msg: nextflow_timeout_minutes={{ nextflow_timeout_minutes }} nextflow_poll_period_minutes={{ nextflow_poll_period_minutes }} + when: + - not combined_file.stat.exists + +- name: Run nextflow (Merge) (async) command: "{{ nextflow_cmd }}" async: "{{ nextflow_timeout_minutes|int * 60 }}" poll: 0 @@ -105,13 +111,13 @@ when: - not combined_file.stat.exists -- name: Wait for nextflow (async) +- name: Wait for nextflow (Merge) async_status: jid: "{{ nextflow_async.ansible_job_id }}" register: nextflow_result until: nextflow_result.finished - delay: "{{ nextflow_poll_period_seconds|int }}" - retries: "{{ (nextflow_timeout_minutes|int / (nextflow_poll_period_seconds|int / 60))|int }}" + delay: "{{ nextflow_poll_period_minutes | int * 60 }}" + retries: "{{ (nextflow_timeout_minutes | int / nextflow_poll_period_minutes | int) | int }}" when: - not combined_file.stat.exists @@ -149,7 +155,7 @@ register: compress_task when: not file_type_file.stat.exists -- name: Add this tasy to the list of compress tasks +- name: Add this task to the list of compress tasks set_fact: compress_tasks: "{{ compress_tasks + [ compress_task ] }}" when: not file_type_file.stat.exists \ No newline at end of file diff --git a/ansible/roles/combine/vars/main.yaml b/ansible/roles/combine/vars/main.yaml index c8d8c06..0a5773e 100644 --- a/ansible/roles/combine/vars/main.yaml +++ b/ansible/roles/combine/vars/main.yaml @@ -39,7 +39,7 @@ shatterfile_large_threshold_kb: 1048576 # and poll period for the (asynchronous) Nextflow process # Timeouts (minutes): 1440 (24 hours) nextflow_timeout_minutes: 1440 -nextflow_poll_period_seconds: 30 +nextflow_poll_period_minutes: 1 compress_timeout_minutes: 1440 compress_poll_period_minutes: 1 diff --git a/ansible/roles/fragment/tasks/fragmentation.yaml b/ansible/roles/fragment/tasks/fragmentation.yaml index 400382c..299a61f 100644 --- a/ansible/roles/fragment/tasks/fragmentation.yaml +++ b/ansible/roles/fragment/tasks/fragmentation.yaml @@ -60,10 +60,6 @@ that: - nextflow_timeout_minutes | int > 0 -- name: Display nextflow timeout (minutes) - debug: - var: nextflow_timeout_minutes - # Set nextflow_poll_period_minutes based on nextflow_timeout_minutes/30. # If this is development then this defaults. - name: Set nextflow_poll_period_minutes @@ -71,10 +67,6 @@ nextflow_poll_period_minutes: "{{ (nextflow_timeout_minutes|int/60)|int }}" when: deployment=="production" -- name: Display nextflow poll period (minutes) - debug: - var: nextflow_poll_period_minutes - - name: Log line to report blockinfile: marker: "" @@ -99,7 +91,7 @@ - "fragmentation.nf.j2" - "{{ deployment }}/nextflow.config.j2" -- name: Set nextflow command +- name: Set nextflow command (Fragmentation) set_fact: nextflow_cmd: >- nextflow run -c {{ reppath }}/nextflow/nextflow.config @@ -112,17 +104,21 @@ --max_hac {{ vendors[vendor].fraghac }} --max_frag {{ vendors[vendor].fragmaxfrags }} -- name: Display nextflow command +- name: Display nextflow command (Fragmentation) debug: var: nextflow_cmd -- name: Run nextflow (async) +- name: Display nextflow command timeout (Fragmentation) + debug: + msg: nextflow_timeout_minutes={{ nextflow_timeout_minutes }} nextflow_poll_period_minutes={{ nextflow_poll_period_minutes }} + +- name: Run nextflow (Fragmentation) (async) command: "{{ nextflow_cmd }}" async: "{{ nextflow_timeout_minutes|int * 60 }}" poll: 0 register: nextflow_async -- name: Wait for nextflow (async) +- name: Wait for nextflow (Fragmentation) async_status: jid: "{{ nextflow_async.ansible_job_id }}" register: nextflow_result diff --git a/ansible/roles/inchi/tasks/gen-isomol-inchi-database.yaml b/ansible/roles/inchi/tasks/gen-isomol-inchi-database.yaml index 4f98de5..6d13b27 100644 --- a/ansible/roles/inchi/tasks/gen-isomol-inchi-database.yaml +++ b/ansible/roles/inchi/tasks/gen-isomol-inchi-database.yaml @@ -63,9 +63,6 @@ nextflow_timeout_minutes: "{{ nextflow_timeout_calc|int }}" when: nextflow_timeout_calc|int>nextflow_timeout_minutes -- debug: - var: nextflow_timeout_minutes - # Set nextflow_poll_period_minutes based on nextflow_timeout_minutes/30. # If this is development then this defaults. - name: Set nextflow_poll_period_minutes @@ -88,7 +85,7 @@ # python -m frag.network.scripts.generate_inchi -i $FRAGPATH/fragment/$FRAGNODEFILE -o $FRAGPATH/fragment/$INCHITAB -n # -- name: Set nextflow command +- name: Set nextflow command (Generate Isomol Inchi) set_fact: nextflow_cmd: >- nextflow run -c {{ reppath }}/nextflow/nextflow.config @@ -100,17 +97,21 @@ --outputfile {{ isoinchitab }} --chunk_size {{ inchigenchunk|int }} -- name: Display nextflow command +- name: Display nextflow command (Generate Isomol Inchi) debug: var: nextflow_cmd -- name: Run nextflow (async) +- name: Display nextflow command timeout (Generate Isomol Inchi) + debug: + msg: nextflow_timeout_minutes={{ nextflow_timeout_minutes }} nextflow_poll_period_minutes={{ nextflow_poll_period_minutes }} + +- name: Run nextflow (Generate Isomol Inchi) (async) command: "{{ nextflow_cmd }}" async: "{{ nextflow_timeout_minutes|int * 60 }}" poll: 0 register: nextflow_async -- name: Wait for nextflow (async) +- name: Wait for nextflow (Generate Isomol Inchi) async_status: jid: "{{ nextflow_async.ansible_job_id }}" register: nextflow_result diff --git a/ansible/roles/inchi/tasks/gen-nonisomol-inchi-database.yaml b/ansible/roles/inchi/tasks/gen-nonisomol-inchi-database.yaml index c6d237f..39311fa 100644 --- a/ansible/roles/inchi/tasks/gen-nonisomol-inchi-database.yaml +++ b/ansible/roles/inchi/tasks/gen-nonisomol-inchi-database.yaml @@ -69,7 +69,7 @@ # Move the nextflow workflow and config into place (from templates) # so they move from "/{{ deployment }}" to "{{ reppath }}/nextflow"... -- name: Translate nextflow files +- name: Translate nextflow files (Generate Inchi) template: src: "{{ item }}" dest: "{{ reppath }}/nextflow/{{ item|regex_replace(deployment + '/')|regex_replace('.j2') }}" @@ -81,7 +81,7 @@ # python -m frag.network.scripts.generate_inchi -i $FRAGPATH/fragment/$FRAGNODEFILE -o $FRAGPATH/fragment/$INCHITAB -n -s # -- name: Set nextflow command +- name: Set nextflow command (Generate Non-Isomol Inchi) set_fact: nextflow_cmd: >- nextflow run -c {{ reppath }}/nextflow/nextflow.config @@ -94,17 +94,17 @@ --chunk_size {{ inchigenchunk|int }} --standard '{{ standard }}' -- name: Display nextflow command +- name: Display nextflow command (Generate Non-Isomol Inchi) debug: var: nextflow_cmd -- name: Run nextflow (async) +- name: Run nextflow (Generate Non-Isomol Inchi) (async) command: "{{ nextflow_cmd }}" async: "{{ nextflow_timeout_minutes|int * 60 }}" poll: 0 register: nextflow_async -- name: Wait for nextflow (async) +- name: Wait for nextflow (Generate Non-Isomol Inchi) async_status: jid: "{{ nextflow_async.ansible_job_id }}" register: nextflow_result diff --git a/ansible/roles/standardise/tasks/standardise-file.yaml b/ansible/roles/standardise/tasks/standardise-file.yaml index a06d111..a31176f 100644 --- a/ansible/roles/standardise/tasks/standardise-file.yaml +++ b/ansible/roles/standardise/tasks/standardise-file.yaml @@ -79,10 +79,6 @@ that: - nextflow_timeout_minutes | int > 0 - - name: Display nextflow timeout (minutes) - debug: - var: nextflow_timeout_minutes - # Set nextflow_poll_period_minutes based on nextflow_timeout_minutes/30. # If this is development then this defaults. - name: Set nextflow_poll_period_minutes @@ -90,10 +86,6 @@ nextflow_poll_period_minutes: "{{ (nextflow_timeout_minutes|int/30)|int }}" when: deployment == "production" - - name: Display nextflow poll period (minutes) - debug: - var: nextflow_poll_period_minutes - - name: Log line to report blockinfile: marker: "" @@ -118,7 +110,7 @@ - "standardizer.nf.j2" - "{{ deployment }}/nextflow.config.j2" - - name: Set nextflow command + - name: Set nextflow command (Standardise) set_fact: nextflow_cmd: >- nextflow run -c {{ reppath }}/nextflow/nextflow.config @@ -133,17 +125,21 @@ --compound_id_field {{ standard_compound_id_field }} --split_token {{ standsplittoken }} - - name: Display nextflow command + - name: Display nextflow command (Standardise) debug: var: nextflow_cmd - - name: Run nextflow (async) + - name: Display nextflow command timeout (Standardise) + debug: + msg: nextflow_timeout_minutes={{ nextflow_timeout_minutes }} nextflow_poll_period_minutes={{ nextflow_poll_period_minutes }} + + - name: Run nextflow (Standardize) (async) command: "{{ nextflow_cmd }}" async: "{{ nextflow_timeout_minutes|int * 60 }}" poll: 0 register: nextflow_async - - name: Wait for nextflow (async) + - name: Wait for nextflow (Standardise) async_status: jid: "{{ nextflow_async.ansible_job_id }}" register: nextflow_result