Skip to content
Continuous Integration

Rerunning CosmoDC2 with recent changes #1006

Sign in to view logs

Rerunning CosmoDC2 with recent changes

Rerunning CosmoDC2 with recent changes #1006

Workflow file for this run

.github/workflows/ci.yml at 8df57ea
name: Continuous Integration
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
schedule:
# run this every week at 12 noon on Monday
- cron: '0 12 * * 1'
env:
EXAMPLE_DATA_FILE_VERSION: v3
# Setting this did not appear to work. Instead will need to
# find/replace when changing it.
# CONTAINER_IMAGE: ghcr.io/lsstdesc/txpipe-dev:latest
jobs:
# Run a download step first so that it is in
# the cache, because all the other jobs will be running
# at the same time and so might miss it. The other jobs
# still do try the download themselves in case something
# doesn't quite work
Download_Data:
runs-on: ubuntu-latest
steps:
- name: Cache example data
id: cache-example
uses: actions/cache@v3
env:
cache-name: cache-example-data
with:
path: example.tar.gz
# update this when we change package contents and want
# to force an update
key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}
- name: Download test data
if: steps.cache-example.outputs.cache-hit != 'true'
run: |
wget -O example.tar.gz "https://portal.nersc.gov/cfs/lsst/txpipe/data/example.tar.gz"
Unit_Tests:
runs-on: ubuntu-latest
container:
image: ghcr.io/lsstdesc/txpipe:v1.0
steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
submodules: true
- name: Test with pytest
run: |
ceci --version
pytest txpipe
Metacal_Pipeline:
runs-on: ubuntu-latest
needs: Download_Data
container:
image: ghcr.io/lsstdesc/txpipe:v1.0
steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
submodules: true
- name: Restore example data
uses: actions/cache/restore@v3
with:
path: example.tar.gz
# update this when we change package contents and want
# to force an update
key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}
fail-on-cache-miss: true
- name: Extract test data
run: |
tar -zxvf example.tar.gz
- name: Run metacal pipeline
run: |
mkdir -p cache/workspaces
ceci examples/metacal/pipeline.yml
test -f data/example/outputs_metacal/shear_xi_plus.png
- name: Show logs
if: ${{ always() }}
run: |
tail -n +1 data/example/logs_metacal/*
Metadetect_Pipeline:
runs-on: ubuntu-latest
needs: Download_Data
container:
image: ghcr.io/lsstdesc/txpipe:v1.0
steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
submodules: true
- name: Restore example data
uses: actions/cache/restore@v3
with:
path: example.tar.gz
# update this when we change package contents and want
# to force an update
key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}
fail-on-cache-miss: true
- name: Extract test data
run: |
tar -zxvf example.tar.gz
- name: Run metadetect pipeline
run: |
mkdir -p cache/workspaces
ceci examples/metadetect/pipeline.yml
test -f data/example/outputs_metadetect/shear_xi_plus.png
- name: Run randoms pipeline
run: |
ceci examples/randoms/pipeline_randomsonly.yml
test -f data/example/outputs_randoms/random_cats.hdf5
- name: Run notebook
run: |
jupyter nbconvert --to notebook --execute "notebooks/Welcome to TXPipe.ipynb"
- name: Show logs
if: ${{ always() }}
run: |
tail -n +1 data/example/logs_metadetect/*
Metadetect_Source_Only_Pipeline:
runs-on: ubuntu-latest
needs: Download_Data
container:
image: ghcr.io/lsstdesc/txpipe:v1.0
steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
submodules: true
- name: Restore example data
uses: actions/cache/restore@v3
with:
path: example.tar.gz
# update this when we change package contents and want
# to force an update
key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}
fail-on-cache-miss: true
- name: Extract test data
run: |
tar -zxvf example.tar.gz
- name: Run metadetect pipeline
run: |
mkdir -p cache/workspaces
ceci examples/metadetect_source_only/pipeline.yml
test -f data/example/metadetect_source_only/shear_xi_plus.png
- name: Show logs
if: ${{ always() }}
run: |
tail -n +1 data/example/metadetect_source_only/*
Lensfit_Pipeline:
runs-on: ubuntu-latest
needs: Download_Data
container:
image: ghcr.io/lsstdesc/txpipe:v1.0
steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
submodules: true
- name: Restore example data
uses: actions/cache/restore@v3
with:
path: example.tar.gz
# update this when we change package contents and want
# to force an update
key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}
fail-on-cache-miss: true
- name: Extract test data
run: |
tar -zxvf example.tar.gz
- name: Run lensfit pipeline
run: |
mkdir -p cache/workspaces
ceci examples/lensfit/pipeline.yml
test -f data/example/outputs_lensfit/shear_xi_plus.png
- name: Show logs
if: ${{ always() }}
run: |
tail -n +1 data/example/logs_lensfit/*
Redmagic_Pipeline:
runs-on: ubuntu-latest
needs: Download_Data
container:
image: ghcr.io/lsstdesc/txpipe:v1.0
steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
submodules: true
- name: Restore example data
uses: actions/cache/restore@v3
with:
path: example.tar.gz
# update this when we change package contents and want
# to force an update
key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}
fail-on-cache-miss: true
- name: Extract test data
run: |
tar -zxvf example.tar.gz
- name: Run redmagic pipeline
run: |
mkdir -p cache/workspaces
ceci examples/redmagic/pipeline.yml
test -f data/example/outputs_redmagic/shear_xi_plus.png
- name: Show logs
if: ${{ always() }}
run: |
tail -n +1 data/example/logs_redmagic/*
Other_Pipeline_Dry_Runs:
runs-on: ubuntu-latest
container:
image: ghcr.io/lsstdesc/txpipe:v1.0
steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
submodules: true
- name: Dry-run pipelines
run: |
ceci --dry-run examples/2.2i/pipeline.yml
ceci --dry-run examples/buzzard/pipeline.yml
ceci --dry-run examples/cosmodc2/pipeline.yml
ceci --dry-run examples/skysim/pipeline.yml
Test_Auto_Installer:
runs-on: ${{ matrix.os }}
needs: Download_Data
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-14]
include:
- os: ubuntu-latest
INSTALL_DEPS: sudo apt-get update && sudo apt-get -y install wget
- os: macos-14
INSTALL_DEPS: brew update-reset && brew install wget
steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
submodules: true
- name: Install wget
run: ${{ matrix.INSTALL_DEPS }}
- name: Run Auto-Install
run: ./bin/install.sh
- name: Restore example data
uses: actions/cache/restore@v3
with:
path: example.tar.gz
# update this when we change package contents and want
# to force an update
key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}
fail-on-cache-miss: true
- name: Extract test data
run: |
tar -zxvf example.tar.gz
- name: Test after auto-install
run: |
mkdir -p cache/workspaces
source ./conda/bin/activate
ceci examples/metadetect/pipeline.yml
- name: Show logs
if: ${{ always() }}
run: |
tail -n +1 data/example/logs_metadetect/*