.gitlab-ci.yml

image: eicweb.phy.anl.gov:4567/containers/eic_container/jug_xl:nightly

default:
  timeout: 3 hours
  artifacts:
    expire_in: 1 week
    paths:
      - results/
  before_script:
    - shopt -s expand_aliases
    - alias parallel='parallel -k --lb -j 1 --colsep ","'

stages:
  - nevents
  - timings
  - collect

.nevents:
  stage: nevents
  script:
    - mkdir -p $(dirname results/datasets/glob/$DATA)
    - grep -v "^\#" $DATA | parallel scripts/glob.sh results/datasets/glob/$DATA {}
    - sort -o results/datasets/glob/$DATA results/datasets/glob/$DATA
    - mkdir -p $(dirname results/datasets/nevents/$DATA)
    - grep -v "^\#" results/datasets/glob/$DATA | parallel scripts/count_events.sh results/datasets/nevents/$DATA {}
    - sort -o results/datasets/nevents/$DATA results/datasets/nevents/$DATA
    - sort -o $DATA $DATA

.timings:
  stage: timings
  script:
    - mkdir -p $(dirname results/datasets/timings/$DATA)
    # Use sed '1!d1' instead of head -n 1 to avoid pipefail issues
    - grep -v "^\#" results/datasets/nevents/$DATA | sed '1!d' | parallel scripts/determine_timing.sh results/datasets/timings/$DATA {}
    - |
      IFS="," read file ext nevents dt0 dt1 < results/datasets/timings/$DATA
      export dt0 dt1
      grep -v "^\#" results/datasets/nevents/$DATA | sed '1d' | parallel scripts/determine_timing.sh results/datasets/timings/$DATA {}
    - sort -o results/datasets/timings/$DATA results/datasets/timings/$DATA

.timings_all:
  stage: timings
  script:
    - mkdir -p $(dirname results/datasets/timings/$DATA)
    - grep -v "^\#" results/datasets/nevents/$DATA | parallel scripts/determine_timing.sh results/datasets/timings/$DATA {}
    - sort -o results/datasets/timings/$DATA results/datasets/timings/$DATA

.collect:
  stage: collect
  script:
    - rm -rf results/logs/
    - find results/datasets/
    - find results/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$5+$4} END {print(":",sum/3600,"core-hours")}' {} \;
    - for d in `find results/datasets/timings/ -type d` ; do
        echo -n $d ;
        find $d -name "*.csv" -exec cat {} \; | awk 'BEGIN {FS=","} {sum+=$3*$5+$4} END {print(":",sum/3600,"core-hours")}' ;
      done
    - find results/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$7+$6} END {print(":",sum/1048576,"GB (full)")}' {} \;
    - for d in `find results/datasets/timings/ -type d` ; do
        echo -n $d ;
        find $d -name "*.csv" -exec cat {} \; | awk 'BEGIN {FS=","} {sum+=$3*$7+$6} END {print(":",sum/1048576,"GB (full)")}' ;
      done
    - find results/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$9+$8} END {print(":",sum/1048576,"GB (reco)")}' {} \;
    - for d in `find results/datasets/timings/ -type d` ; do
        echo -n $d ;
        find $d -name "*.csv" -exec cat {} \; | awk 'BEGIN {FS=","} {sum+=$3*$9+$8} END {print(":",sum/1048576,"GB (reco)")}' ;
      done

include:
  - local: 'BACKGROUNDS/config.yml'
  - local: 'DIS/config.yml'
  - local: 'EXCLUSIVE/config.yml'
  - local: 'SIDIS/config.yml'
  - local: 'SINGLE/config.yml'
#  - local: 'SR/config.yml'

collect:
  extends: .collect
  needs:
    - "BACKGROUNDS:collect"
    - "DIS:collect"
    - "EXCLUSIVE:collect"
    - "SIDIS:collect"
    - "SINGLE:collect"
#    - "SR:collect"