.gitlab-ci.yml

# GCG Gitlab CI script
# Can be skipped using `git push --push-option=ci.skip`

.init_ssh2: &init_ssh2 |
  eval $(ssh-agent -s)
  chmod 700 "$PRIVATE_KEY"
  ls -la "$PRIVATE_KEY"
  ssh-add "$PRIVATE_KEY" > /dev/null
  mkdir -p ~/.ssh
  chmod 700 ~/.ssh
  [[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config

stages:
  - "Build"
  - "Test"
  - "Test Report"
  - "Experiment"
  - "Report"
  - "Documentation"

# default, loaded with docker image for Linux
default:
  image: registry.git.or.rwth-aachen.de/docker/gcg-doc:latest
  before_script:
    # initialize SSH key to access repositories
    - *init_ssh2
    # configure git
    #- git config --global user.email "gcg@or.rwth-aachen.de"
    #- git config --global user.name "GCG CI"
    # update submodules
    #- git submodule sync --recursive
    #- git submodule foreach 'git stash'
    #- git submodule update --init --recursive
    # set links to SCIP and SoPLEX
    - ln -sfn ../lib/scip-git lib/scip
    # SoPlex links inside SCIP lib folder
    - mkdir -p lib/scip/lib/include/spxinc
    - ln -sfn $PWD/lib/soplex-git/src/* $PWD/lib/scip/lib/include/spxinc/
    - mkdir -p lib/scip/lib/static/
    - ln -sfn $PWD/lib/soplex-git/lib/libsoplex.linux.x86_64.gnu.opt.a $PWD/lib/scip/lib/static/libsoplex.linux.x86_64.gnu.opt.a


###################################
############ Cleanup Job ##########
###################################

# remove deprecated feature branch runs from file system
cleanup:master:
  stage: "Build"
  rules:
    #  execute automatically inside a scheduled master job
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "master"'
      when: always
    - if: '$CI_COMMIT_BRANCH == "ci_performance_experiments"'
      when: always
    - when: never
  allow_failure: true
  before_script:
    # initialize SSH key to access repositories
    - *init_ssh2
  script:
    # execute cleanup script
    - FOLDER="gcg_experiment"
    - ssh gcgci@clustor "rm -rf scripts; mkdir -p scripts"
    - scp scripts/git_ci_cleanup.sh gcgci@clustor:~/scripts
    - ssh gcgci@clustor "./scripts/git_ci_cleanup.sh $FOLDER $PRIVATE_TOKEN"


###################################
############ Build Jobs ###########
###################################

# Make compilation jobs
build:make_release:
  stage: "Build"
  variables:
    GIT_STRATEGY: fetch
    GIT_SUBMODULE_STRATEGY: recursive
  when: always
  #except:
  #  - schedules
  script:
    # prepare dependencies
    - mkdir -p lib/include
    - mkdir -p lib/static
    ## Cliquer ##
    # obtain
    - wget http://users.aalto.fi/~pat/cliquer/cliquer-1.21.tar.gz
    - tar xvfz cliquer-1.21.tar.gz
    - ln -sfn ../cliquer-1.21/ lib/cliquer-git
    # link
    - ln -sfn ../cliquer-git/ lib/include/cliquer
    ## hMetis ##
    # link
    - wget https://or.rwth-aachen.de/hmetis/hmetis-2.0pre1.tar.gz
    - tar xvfz hmetis-2.0pre1.tar.gz
    - ln -sfn hmetis-2.0pre1/Linux-x86_64/hmetis2.0pre1 hmetis
    - sed -i.bak "s/HMETIS_EXECUTABLE \"hmetis\"/HMETIS_EXECUTABLE \"\.\/hmetis\"/g" src/gcg/dec_h*partition.cpp
    # generate 'opt' binary
    - make deps OPT=opt CLIQUER=true BLISS=true
    - ln -sfn ../cliquer-git/libcliquer.a lib/static/libcliquer.a
    - make OPT=opt CLIQUER=true BLISS=true
  artifacts:
    expire_in: 3 days
    paths:
      - $CI_PROJECT_DIR
    exclude:
      - $CI_PROJECT_DIR/.git/*

build:make_debug:
  stage: "Build"
  variables:
    GIT_STRATEGY: fetch
    GIT_SUBMODULE_STRATEGY: recursive
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" && ($CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH =~ /^v[0-9]+-bugfix$/)'
      when: always
    - if: '$CI_PIPELINE_SOURCE == "push"'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: manual
      allow_failure: true
  script:
    # prepare dependencies
    - mkdir -p lib/include
    - mkdir -p lib/static
    ## Cliquer ##
    # obtain
    - wget http://users.aalto.fi/~pat/cliquer/cliquer-1.21.tar.gz
    - tar xvfz cliquer-1.21.tar.gz
    - ln -sfn ../cliquer-1.21/ lib/cliquer-git
    # link
    - ln -sfn ../cliquer-git/ lib/include/cliquer
    ## hMetis ##
    # link
    - wget https://or.rwth-aachen.de/hmetis/hmetis-2.0pre1.tar.gz
    - tar xvfz hmetis-2.0pre1.tar.gz
    - ln -sfn hmetis-2.0pre1/Linux-x86_64/hmetis2.0pre1 hmetis
    - sed -i.bak "s/HMETIS_EXECUTABLE \"hmetis\"/HMETIS_EXECUTABLE \"\.\/hmetis\"/g" src/gcg/dec_h*partition.cpp
    # generate 'dbg' binary
    - make deps OPT=dbg CLIQUER=true BLISS=true
    - ln -sfn ../cliquer-git/libcliquer.a lib/static/libcliquer.a
    - make OPT=dbg CLIQUER=true BLISS=true
  artifacts:
    expire_in: 3 days
    paths:
      - $CI_PROJECT_DIR
    exclude:
      - $CI_PROJECT_DIR/.git/*

# CMake compilation jobs
build:cmake_release:
  stage: "Build"
  variables:
    GIT_STRATEGY: fetch
    GIT_SUBMODULE_STRATEGY: recursive
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" && ($CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH =~ /^v[0-9]+-bugfix$/)'
      when: always
    - if: '$CI_PIPELINE_SOURCE == "push"'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: manual
      allow_failure: true
  script:
    - mkdir build
    - cd build
    # main compilation (build type: debug)
    - cmake -DGCG_DEV_BUILD=ON -DZIMPL=OFF -DIPOPT=OFF -DPAPILO=OFF -DCLIQUER_DIR=cliquer-1.21/ ..
    - make -j 4
    - cd ..
  artifacts:
    expire_in: 3 days
    paths:
      - $CI_PROJECT_DIR
    exclude:
      - $CI_PROJECT_DIR/.git/*

build:cmake_debug:
  stage: "Build"
  variables:
    GIT_STRATEGY: fetch
    GIT_SUBMODULE_STRATEGY: recursive
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "push"'
      when: always
  script:
    - mkdir build
    - cd build
    # download and unpack cliquer (bliss and hmetis already exist)
    - wget http://users.aalto.fi/~pat/cliquer/cliquer-1.21.tar.gz
    - tar xvfz cliquer-1.21.tar.gz
    # main compilation (build type: debug)
    - cmake -DGCG_DEV_BUILD=ON -DZIMPL=OFF -DIPOPT=OFF -DPAPILO=OFF -DCLIQUER_DIR=cliquer-1.21/ -DCMAKE_BUILD_TYPE=Debug ..
    - make -j 4
    - cd ..
  artifacts:
    expire_in: 3 days
    paths:
      - $CI_PROJECT_DIR
    exclude:
      - $CI_PROJECT_DIR/.git/*

# Windows compilation jobs
build:windows_release:
  stage: "Build"
  variables:
    GIT_STRATEGY: fetch
    GIT_SUBMODULE_STRATEGY: recursive
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "push"'
      when: always
  tags:
    - windows
  before_script:
    - echo "on windows"
  script:
    - cmake --workflow --preset gcg-windows-ci-release
  artifacts:
    expire_in: 3 days
    paths:
      - $CI_PROJECT_DIR
    exclude:
      - $CI_PROJECT_DIR/.git/*

build:windows_debug:
  stage: "Build"
  variables:
    GIT_STRATEGY: fetch
    GIT_SUBMODULE_STRATEGY: recursive
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "push"'
      when: always
  tags:
    - windows
  before_script:
    - echo "on windows"
  script:
    - cmake --workflow --preset gcg-windows-ci-debug
  artifacts:
    expire_in: 3 days
    paths:
      - $CI_PROJECT_DIR
    exclude:
      - $CI_PROJECT_DIR/.git/*


###################################
############## Tests ##############
###################################

# Short test for makefiles release build (test set: short)
test:make_release:
  stage: "Test"
  when: always
  #except:
  #  - schedules
  needs: ["build:make_release"]
  before_script: []
  script:
    - make SETTINGS=default_feature test
    # check if test failed
    - ls check/results/*.res # resfile exists
    - if [[ $(tail -n1 check/results/*.res) == "@01 GCG(?)SCIP(?)?(?):default" ]]; then echo "Fatal linking error."; exit 1; fi # no linking error
    - if [[ $(grep " shifted geom\. \[" -B1 check/results/*.res | head -n1 | tr -s ' ' | cut -d" " -f5) > 0 ]]; then echo "Some instances failed."; exit 1; fi # no failed instances
  artifacts:
    paths:
      - check/results/*
    expire_in: 3 days

# Short test for makefiles debug build (test set: short)
test:make_debug:
  stage: "Test"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" && ($CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH =~ /^v[0-9]+-bugfix$/)'
      when: always
    - if: '$CI_PIPELINE_SOURCE == "push"'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: manual
      allow_failure: true
  needs: ["build:make_debug"]
  before_script: []
  script:
    - make DETECTIONSTATISTICS=true OPT=dbg SETTINGS=default_feature test
    # check if test failed
    - ls check/results/*.res # resfile exists
    - if [[ $(tail -n1 check/results/*.res) == "@01 GCG(?)SCIP(?)?(?):default" ]]; then echo "Fatal linking error."; exit 1; fi # no linking error
    - if [[ $(grep " shifted geom\. \[" -B1 check/results/*.res | head -n1 | tr -s ' ' | cut -d" " -f5) > 0 ]]; then echo "Some instances failed."; exit 1; fi # no failed instances
  artifacts:
    paths:
      - check/results/*
    expire_in: 3 days

# Short test for CMake release build (test set: short)
test:cmake_release:
  stage: "Test"
  needs: ["build:cmake_release"]
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" && ($CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH =~ /^v[0-9]+-bugfix$/)'
      when: always
    - if: '$CI_PIPELINE_SOURCE == "push"'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: manual
      allow_failure: true
  before_script: []
  script:
    - cd build
    - make gcg_check SETTINGS=default_feature
    - cd ..
    # check if test failed
    - ls build/check/Testing/Temporary/LastTest.log # logfile exists
  artifacts:
    paths:
      - build/check/Testing/Temporary/LastTest.log
    expire_in: 3 days

# Short test for CMake debug build (test set: short)
test:cmake_debug:
  stage: "Test"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "push"'
      when: always
  needs: ["build:cmake_debug"]
  before_script: []
  script:
    - cd build
    - make gcg_check SETTINGS=default_feature
    - cd ..
    # check if test failed
    - ls build/check/Testing/Temporary/LastTest.log # logfile exists
  artifacts:
    paths:
      - build/check/Testing/Temporary/LastTest.log
    expire_in: 3 days


###################################
########## Documentation ##########
###################################

# Compile documentation and upload to master preview on gcg.or.rwth-aachen.de/doc-master
# Only executed on branch `master`
docu:master:
  when: always
  stage: "Documentation"
  needs: ["build:cmake_release"]
  only:
    - master
  except:
    - schedules
  script:
    # build documentation
    - cd build
    - make gcg_doc -j 4
    - cd ..
    # upload documentation to gcg.or.rwth-aachen.de/doc-master
    - scp -r doc/html/. gitGCGdoc@orweb.or.rwth-aachen.de:/var/www/gcg-doc-master
    - ssh gitGCGdoc@orweb.or.rwth-aachen.de "cd /var/www/gcg-doc-master; chmod -R 755 ./* ; "
  artifacts:
    paths:
      - doc/html

# Compile documentation and upload to dev preview on gcg.or.rwth-aachen.de/doc-preview
# Only executed on branch `docu`
docu:dev_preview:
  stage: "Documentation"
  when: always
  needs: ["build:cmake_release"]
  only:
    - docu
  script:
    # build documentation
    - cd build
    - make gcg_doc -j 4
    - cd ..
    # upload documentation to gcg.or.rwth-aachen.de/doc-preview
    - scp -r doc/html/. gitGCGdoc@orweb.or.rwth-aachen.de:/var/www/gcg-doc-preview
    - ssh gitGCGdoc@orweb.or.rwth-aachen.de "cd /var/www/gcg-doc-preview; chmod -R 755 ./* ; "
  artifacts:
    paths:
      - doc/html


###################################
###### Benchmark/Experiment #######
###################################

# Master branch runtime data generation
benchmark:master:
  stage: "Experiment"
  when: always
  needs: ["build:make_release"]
  # if the cluster is very busy, testing might take some time
  timeout: 48 hours
  # *automated* exection upon new commits,
  # to keep runtime data located in /shared/gcg_runtime_data/master_branch up-to-date
  only:
    - master
  except:
    - schedules
  before_script:
    - *init_ssh2
  script:
    # copy over the whole repository
    - FOLDER="gcg_experiment/${CI_JOB_ID}"
    - ssh gcgci@clustor "rm -rf ~/$FOLDER"
    - ssh gcgci@clustor "mkdir -p ~/$FOLDER/check/instances && ln -sfn /opt/instances/striplibn ~/$FOLDER/check/instances/striplib"
    - scp -r * gcgci@clustor:~/$FOLDER
    # execute "experiment.test" testset. use settings to mark runtime (there should not exist "default_master.set" different to "default.set"!)
    - PAT='^([0-9]+) job.*cluster ([0-9]+)\.'
    - CONDOR_RES=$(ssh gcgci@clustor "cd ~/$FOLDER && make testcluster STATISTICS=true TEST=experiment SETTINGS=default_master TIME=10800 MEM=8000 | grep 'submitted to'")
    - if [[ $CONDOR_RES =~ $PAT ]]; then echo $CONDOR_RES; else echo "Submitting jobs to cluster failed"; exit 1; fi
    - CLUSTER_ID=${BASH_REMATCH[2]}
    # check continuously if all cluster jobs are done. if so, exit while loop
    - >-
      ssh gcgci@clustor "while [[ ! \$(condor_q $CLUSTER_ID) =~ 'Total for query: 0 jobs' ]]; do sleep 5s; done; echo 'All jobs done'"
    - ssh gcgci@clustor "cd ~/$FOLDER/check && ./evalcheck_cluster.sh -r results/*.eval"
    # make the "current master link" a "previous master" link
    - ssh gcgci@clustor 'cd /shared/gcg_runtime_data/ && ln -sfn master_branch/$(basename $(readlink -f current_master)) previous_master && echo "Set link to previous master (hash $(basename $(readlink -f previous_master)))."'
    # rename old default_master setting to previous_master
    - ssh gcgci@clustor 'cd /shared/gcg_runtime_data/previous_master && for i in *default_master*; do mv $i "`echo $i | sed "s/default_master/previous_master/"`"; done'
    # create new folder for master runtime data
    - ssh gcgci@clustor "cd /shared/gcg_runtime_data/ && mkdir -p master_branch/${CI_COMMIT_SHORT_SHA} && ln -sfn master_branch/${CI_COMMIT_SHORT_SHA} current_master"
    # copy runtime data to newly created folder
    - ssh gcgci@clustor "cp -fr ~/$FOLDER/check/results/* /shared/gcg_runtime_data/master_branch/${CI_COMMIT_SHORT_SHA} && echo 'Adding new master branch run (hash ${CI_COMMIT_SHORT_SHA}).'"
    # clean up old entries (keep latest 5 runs)
    - ssh gcgci@clustor 'cd /shared/gcg_runtime_data/master_branch/ && for folder in $(ls -td */ | tail -n +6); do echo "Removing old master branch run with hash ${folder:0:8} (last modified $(stat ${folder} | grep Modify | cut -d\  -f2- | cut -d. -f1))."; rm -rf $folder; done'
    # copy experiment runtime data here to save as artifacts
    - mkdir -p check/results && scp -r gcgci@clustor:~/$FOLDER/check/results/* check/results
    # delete master runtime data (for later comparisons, use "previous_master" and "current_master" in /shared/gcg_runtime_data/)
    - ssh gcgci@clustor "rm -rf ~/$FOLDER"
  # artifacts needed for reporting stages testset_report:experiment_master and comparison_report:experiment_master
  artifacts:
    paths:
      - check/results/*
    expire_in: 3 days

# Master branch runtime data generation (experiment: use pricing statistics)
experiment:master:
  stage: "Experiment"
  when: always
  needs: ["build:make_release"]
  # if the cluster is very busy, testing might take some time
  timeout: 48 hours
  # *automated* exection upon new commits,
  # to keep runtime data located in user "gcgci"'s ~/gcg_master/${CI_JOB_ID}/check/results up-to-date
  only:
    - master
  except:
    - schedules
  before_script:
    - *init_ssh2
  script:
    # recompile with STATISTICS=true to also print pricing statistics
    - make STATISTICS=true OPT=opt CLIQUER=true BLISS=true
    # copy over the whole repository
    - FOLDER="gcg_experiment/${CI_JOB_ID}"
    - ssh gcgci@clustor "rm -rf ~/$FOLDER"
    - ssh gcgci@clustor "mkdir -p ~/$FOLDER/check/instances && ln -sfn /opt/instances/striplibn ~/$FOLDER/check/instances/striplib"
    - scp -r * gcgci@clustor:~/$FOLDER
    # execute "experiment.test" testset. use settings to mark runtime (there should not exist "default_master.set" different to "default.set"!)
    - PAT='^([0-9]+) job.*cluster ([0-9]+)\.'
    - CONDOR_RES=$(ssh gcgci@clustor "cd ~/$FOLDER && make testcluster STATISTICS=true TEST=experiment SETTINGS=default_master TIME=10800 MEM=8000 | grep 'submitted to'")
    - if [[ $CONDOR_RES =~ $PAT ]]; then echo $CONDOR_RES; else echo "Submitting jobs to cluster failed"; exit 1; fi
    - CLUSTER_ID=${BASH_REMATCH[2]}
    # check continuously if all cluster jobs are done. if so, exit while loop
    - >-
      ssh gcgci@clustor "while [[ ! \$(condor_q $CLUSTER_ID) =~ 'Total for query: 0 jobs' ]]; do sleep 5s; done; echo 'All jobs done'"
    - ssh gcgci@clustor "cd ~/$FOLDER/check && ./evalcheck_cluster.sh -r results/*.eval"
    # copy experiment runtime data here to save as artifacts
    - mkdir -p check/results && scp -r gcgci@clustor:~/$FOLDER/check/results/* check/results
    # delete master runtime data (for later comparisons, use "previous_master" and "current_master" in /shared/gcg_runtime_data/)
    - ssh gcgci@clustor "rm -rf ~/$FOLDER"
  # artifacts needed for reporting stages testset_report:experiment_master and comparison_report:experiment_master
  artifacts:
    paths:
      - check/results/*
    expire_in: 3 days

# Feature branch runtime data generation
benchmark:feature:
  stage: "Experiment"
  when: manual
  allow_failure: true
  needs: ["build:make_release"]
  # if the cluster is very busy, testing might take some time
  timeout: 48 hours
  except:
    - master
  before_script:
    - *init_ssh2
  script:
    # benchmark stage does not use bliss, cliquer or htmetis
    # copy over the whole repository
    - FOLDER="gcg_experiment/${CI_JOB_ID}"
    - ssh gcgci@clustor "rm -rf ~/$FOLDER"
    - ssh gcgci@clustor "mkdir -p ~/$FOLDER/check/instances && ln -sfn /opt/instances/striplibn ~/$FOLDER/check/instances/striplib"
    - scp -r * gcgci@clustor:~/$FOLDER
    # execute "experiment.test" testset
    - PAT='^([0-9]+) job.*cluster ([0-9]+)\.'
    - CONDOR_RES=$(ssh gcgci@clustor "cd ~/$FOLDER && make testcluster STATISTICS=true TEST=experiment SETTINGS=default_feature TIME=10800 MEM=8000 | grep 'submitted to'")
    - if [[ $CONDOR_RES =~ $PAT ]]; then echo $CONDOR_RES; else echo "Submitting jobs to cluster failed"; exit 1; fi
    - CLUSTER_ID=${BASH_REMATCH[2]}
    # check continuously if all cluster jobs are done. if so, exit while loop
    - >-
      ssh gcgci@clustor "while [[ ! \$(condor_q $CLUSTER_ID) =~ 'Total for query: 0 jobs' ]]; do sleep 5s; done; echo 'All jobs done'"
    - ssh gcgci@clustor "cd ~/$FOLDER/check && ./evalcheck_cluster.sh -r results/*.eval"
    # copy back runtime data
    - mkdir -p check/results
    # copy experiment runtime data here to save as artifacts
    - scp -r gcgci@clustor:~/$FOLDER/check/results/* check/results
    # remove the copied over data, in case branch is deleted or merged, to not keep the data under user "gcgci"
    - ssh gcgci@clustor "rm -rf ~/$FOLDER"
  artifacts:
    paths:
      - check/results/*
    expire_in: 3 days

# Feature branch runtime data generation (experiment: use pricing statistics)
experiment:feature:
  stage: "Experiment"
  when: manual
  allow_failure: true
  needs: ["build:make_release"]
  # if the cluster is very busy, testing might take some time
  timeout: 48 hours
  except:
    - master
  before_script:
    - *init_ssh2
  script:
    # recompile with STATISTICS=true to also print pricing statistics
    - make STATISTICS=true OPT=opt CLIQUER=true BLISS=true
    # copy over the whole repository
    - FOLDER="gcg_experiment/${CI_JOB_ID}"
    - ssh gcgci@clustor "rm -rf ~/$FOLDER"
    - ssh gcgci@clustor "mkdir -p ~/$FOLDER/check/instances && ln -sfn /opt/instances/striplibn ~/$FOLDER/check/instances/striplib"
    - scp -r * gcgci@clustor:~/$FOLDER
    # execute "experiment.test" testset
    - PAT='^([0-9]+) job.*cluster ([0-9]+)\.'
    - CONDOR_RES=$(ssh gcgci@clustor "cd ~/$FOLDER && make testcluster STATISTICS=true TEST=experiment SETTINGS=default_feature TIME=10800 MEM=8000 | grep 'submitted to'")
    - if [[ $CONDOR_RES =~ $PAT ]]; then echo $CONDOR_RES; else echo "Submitting jobs to cluster failed"; exit 1; fi
    - CLUSTER_ID=${BASH_REMATCH[2]}
    # check continuously if all cluster jobs are done. if so, exit while loop
    - >-
      ssh gcgci@clustor "while [[ ! \$(condor_q $CLUSTER_ID) =~ 'Total for query: 0 jobs' ]]; do sleep 5s; done; echo 'All jobs done'"
    - ssh gcgci@clustor "cd ~/$FOLDER/check && ./evalcheck_cluster.sh -r results/*.eval"
    # copy back runtime data
    - mkdir -p check/results
    # copy experiment runtime data here to save as artifacts
    - scp -r gcgci@clustor:~/$FOLDER/check/results/* check/results
    # remove the copied over data, in case branch is deleted or merged, to not keep the data under user "gcgci"
    - ssh gcgci@clustor "rm -rf ~/$FOLDER"
  artifacts:
    paths:
      - check/results/*
    expire_in: 3 days


###################################
############ Reports ##############
###################################

testset_report:benchmark_master:
  stage: "Report"
  needs: ["benchmark:master"]
  # since runtime data is large, generating reports will take longer as well
  timeout: 48 hours
  only:
    - master
  except:
    - schedules
  script:
    # rename hmetis executable back
    - sed -i.bak "s/HMETIS_EXECUTABLE \"\.\/hmetis\"/HMETIS_EXECUTABLE \"hmetis\"/g" src/gcg/dec_h*partition.cpp
    # generate test set report
    - printf "DEBUG=true\nDETECTION=false" > v.vset
    - make visu DATADIR=results/ VISUSETTINGS=../v.vset
    - cp $(ls -d check/reports/testsetreport*)/experiment.*.testsetreport.pdf testsetreport.pdf
  artifacts:
    name: "testsetreport_$CI_JOB_ID"
    paths:
      - check/reports/*
      - testsetreport.pdf
    expire_in: 14 days

# Runtime data comparison
comparison_report:benchmark_master:
  stage: "Report"
  needs: ["benchmark:master"]
  # since runtime data is large, generating reports will take longer as well
  timeout: 48 hours
  only:
    - master
  except:
    - schedules
  # get default before_script to do linking to SCIP
  script:
    # rename hmetis executable back
    - sed -i.bak "s/HMETIS_EXECUTABLE \"\.\/hmetis\"/HMETIS_EXECUTABLE \"hmetis\"/g" src/gcg/dec_h*partition.cpp
    # get runtime data from latest release
    # commented out, since the runtime data in the docu is just the short test set
    #- cp doc/resources/misc/runtime_data/*.* check/results
    #- cp doc/resources/misc/runtime_data/vbc/*.* check/results/vbc
    # get runtime data from previous master (experiment test set), since current master is already this one
    - ssh gcgci@clustor 'echo "Getting data from previous master (hash $(basename $(readlink -f previous_master)))."'
    - scp -r gcgci@clustor:/shared/gcg_runtime_data/previous_master/* check/results/
    # generate comparison report
    - printf "DEBUG=true\nDETECTION=false" > v.vset
    - echo "c" | make visu DATADIR=results/ VISUSETTINGS=../v.vset
    - cp $(ls -d check/reports/comparisonreport*)/comparisonreport.pdf comparisonreport.pdf
  artifacts:
    name: "comparisonreport_exp_$CI_JOB_ID"
    paths:
      - check/reports/*
      - comparisonreport.pdf
    expire_in: 14 days

testset_report:benchmark_feature:
  stage: "Report"
  when: manual
  allow_failure: true
  needs: ["benchmark:feature"]
  # since runtime data is large, generating reports will take longer as well
  timeout: 48 hours
  except:
    - master
  script:
    # rename hmetis executable back
    - sed -i.bak "s/HMETIS_EXECUTABLE \"\.\/hmetis\"/HMETIS_EXECUTABLE \"hmetis\"/g" src/gcg/dec_h*partition.cpp
    # generate test set report
    - printf "DEBUG=true\nDETECTION=false" > v.vset
    - make visu DATADIR=results/ VISUSETTINGS=../v.vset
    - cp $(ls -d check/reports/testsetreport*)/experiment.*.testsetreport.pdf testsetreport.pdf
  artifacts:
    name: "testsetreport_$CI_JOB_ID"
    paths:
      - check/reports/*
      - testsetreport.pdf
    expire_in: 14 days

# Runtime data comparison
comparison_report:benchmark_feature:
  stage: "Report"
  when: manual
  allow_failure: true
  needs: ["benchmark:feature"]
  # since runtime data is large, generating reports will take longer as well
  timeout: 24 hours
  except:
    - master
  # get default before_script to do linking to SCIP
  script:
    # rename hmetis executable back
    - sed -i.bak "s/HMETIS_EXECUTABLE \"\.\/hmetis\"/HMETIS_EXECUTABLE \"hmetis\"/g" src/gcg/dec_h*partition.cpp
    # get runtime data from latest release
    # commented out, since the runtime data in the docu is just the short test set
    #- cp doc/resources/misc/runtime_data/*.* check/results
    #- cp doc/resources/misc/runtime_data/vbc/*.* check/results/vbc
    # get runtime data from master
    - scp -r gcgci@clustor:/shared/gcg_runtime_data/current_master/* check/results/
    # generate comparison report
    - printf "DEBUG=true\nDETECTION=false" > v.vset
    - echo "c" | make visu DATADIR=results/ VISUSETTINGS=../v.vset
    - cp $(ls -d check/reports/comparisonreport*)/comparisonreport.pdf comparisonreport.pdf
  artifacts:
    name: "comparisonreport_exp_$CI_JOB_ID"
    paths:
      - check/reports/*
      - comparisonreport.pdf
    expire_in: 14 days

testset_report:experiment_master:
  stage: "Report"
  needs: ["experiment:master"]
  # since runtime data is large, generating reports will take longer as well
  timeout: 48 hours
  only:
    - master
  except:
    - schedules
  script:
    # rename hmetis executable back
    - sed -i.bak "s/HMETIS_EXECUTABLE \"\.\/hmetis\"/HMETIS_EXECUTABLE \"hmetis\"/g" src/gcg/dec_h*partition.cpp
    # generate test set report
    - printf "DEBUG=true" > v.vset
    - make visu DATADIR=results/ VISUSETTINGS=../v.vset
    - cp $(ls -d check/reports/testsetreport*)/experiment.*.testsetreport.pdf testsetreport.pdf
  artifacts:
    name: "testsetreport_$CI_JOB_ID"
    paths:
      - check/reports/*
      - testsetreport.pdf
    expire_in: 14 days

testset_report:experiment_feature:
  stage: "Report"
  when: manual
  allow_failure: true
  needs: ["experiment:feature"]
  # since runtime data is large, generating reports will take longer as well
  timeout: 48 hours
  except:
    - master
  script:
    # rename hmetis executable back
    - sed -i.bak "s/HMETIS_EXECUTABLE \"\.\/hmetis\"/HMETIS_EXECUTABLE \"hmetis\"/g" src/gcg/dec_h*partition.cpp
    # generate test set report
    - printf "DEBUG=true" > v.vset
    - make visu DATADIR=results/ VISUSETTINGS=../v.vset
    - cp $(ls -d check/reports/testsetreport*)/experiment.*.testsetreport.pdf testsetreport.pdf
  artifacts:
    name: "testsetreport_$CI_JOB_ID"
    paths:
      - check/reports/*
      - testsetreport.pdf
    expire_in: 14 days