diff --git a/beta-pipelines/skylab/ATAC/ATAC.wdl b/beta-pipelines/skylab/ATAC/ATAC.wdl index 6dfaedb83..2210f0772 100644 --- a/beta-pipelines/skylab/ATAC/ATAC.wdl +++ b/beta-pipelines/skylab/ATAC/ATAC.wdl @@ -241,7 +241,7 @@ task BWAPairedEndAlignment { read_group_sample_name: "the read group sample to be added upon alignment" cpu: "the number of cpu cores to use during alignment" output_base_name: "basename to be used for the output of the task" - docker_image: "the docker image using BWA to be used (default: us.gcr.io/broad-gotc-prod/pytools:1.0.0-1661263730)" + docker_image: "the docker image using BWA to be used (default: us.gcr.io/broad-gotc-prod/bwa:1.0.0-0.7.17-1660770463)" } # runtime requirements based upon input file size @@ -673,13 +673,13 @@ task MakeCompliantBAM { input { File bam_input String output_base_name - String docker_image = "us.gcr.io/broad-gotc-prod/pytools:1.0.0-1661263730" + String docker_image = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.1-1686932671" } parameter_meta { bam_input: "the bam with barcodes in the read ids that need to be converted to barcodes in bam tags" output_base_name: "base name to be used for the output of the task" - docker_image: "the docker image using the python script to convert the bam barcodes/read ids (default: us.gcr.io/broad-gotc-prod/pytools:1.0.0-1661263730)" + docker_image: "the docker image using the python script to convert the bam barcodes/read ids (default: us.gcr.io/broad-gotc-prod/warp-tools:1.0.1-1686932671)" } Int disk_size = ceil(2.5 * (if size(bam_input, "GiB") < 1 then 1 else size(bam_input, "GiB"))) @@ -687,7 +687,7 @@ task MakeCompliantBAM { String compliant_bam_output_name = output_base_name + ".compliant.bam" command { - /usr/gitc/makeCompliantBAM.py \ + /warptools/scripts/makeCompliantBAM.py \ --input-bam ~{bam_input} \ --output-bam ~{compliant_bam_output_name} } @@ -707,7 +707,7 @@ task MakeCompliantBAM { task BreakoutSnap { input { File snap_input - String docker_image = "us.gcr.io/broad-gotc-prod/pytools:1.0.0-1661263730" + String docker_image = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.1-1686932671" String bin_size_list } Int num_threads = 1 @@ -715,7 +715,7 @@ task BreakoutSnap { command { set -euo pipefail mkdir output - /usr/gitc/breakoutSnap.py --input ~{snap_input} \ + /warptools/scripts/breakoutSnap.py --input ~{snap_input} \ --output-prefix output/ } output { diff --git a/pipelines/skylab/multiome/Multiome.changelog.md b/pipelines/skylab/multiome/Multiome.changelog.md index a27ca6d3c..de9d8b623 100644 --- a/pipelines/skylab/multiome/Multiome.changelog.md +++ b/pipelines/skylab/multiome/Multiome.changelog.md @@ -1,8 +1,11 @@ # 1.0.1 -2023-07-11 (Date of Last Commit) +2023-07-07 (Date of Last Commit) + +* Changed Pytools container to Warp-tools in StarAlign and updated command paths accordingly * Updated the CountAlignments task in the FeatureCounts.wdl to use a new docker image. This change does not affect the Multiome pipeline + # 1.0.0 2023-06-22 (Date of Last Commit) diff --git a/pipelines/skylab/optimus/Optimus.changelog.md b/pipelines/skylab/optimus/Optimus.changelog.md index 216d14cfe..480504d21 100644 --- a/pipelines/skylab/optimus/Optimus.changelog.md +++ b/pipelines/skylab/optimus/Optimus.changelog.md @@ -1,3 +1,7 @@ +# 5.8.4 +2023-07-07 (Date of Last Commit) + +* Changed Pytools container to Warp-tools in StarAlign and updated command paths accordingly # 5.8.3 2023-06-23 (Date of Last Commit) diff --git a/pipelines/skylab/optimus/Optimus.wdl b/pipelines/skylab/optimus/Optimus.wdl index e6b41fe69..5801bd936 100644 --- a/pipelines/skylab/optimus/Optimus.wdl +++ b/pipelines/skylab/optimus/Optimus.wdl @@ -67,7 +67,7 @@ workflow Optimus { # version of this pipeline - String pipeline_version = "5.8.3" + String pipeline_version = "5.8.4" # this is used to scatter matched [r1_fastq, r2_fastq, i1_fastq] arrays Array[Int] indices = range(length(r1_fastq)) diff --git a/pipelines/skylab/scATAC/scATAC.changelog.md b/pipelines/skylab/scATAC/scATAC.changelog.md index c9ad71222..8b7e6ff91 100644 --- a/pipelines/skylab/scATAC/scATAC.changelog.md +++ b/pipelines/skylab/scATAC/scATAC.changelog.md @@ -1,3 +1,8 @@ +# 1.3.2 +2023-07-07 (Date of Last Commit) + +* Changed Pytools container to Warp-tools and updated command paths accordingly + # 1.3.1 2023-01-19 (Date of Last Commit) diff --git a/pipelines/skylab/scATAC/scATAC.wdl b/pipelines/skylab/scATAC/scATAC.wdl index 3399afabb..10f456fc3 100644 --- a/pipelines/skylab/scATAC/scATAC.wdl +++ b/pipelines/skylab/scATAC/scATAC.wdl @@ -15,7 +15,7 @@ workflow scATAC { String bin_size_list = "10000" } - String pipeline_version = "1.3.1" + String pipeline_version = "1.3.2" parameter_meta { input_fastq1: "read 1 input fastq, the read names must be tagged with the cellular barcodes" @@ -254,7 +254,7 @@ task MakeCompliantBAM { input { File input_bam String output_bam_filename - String docker_image = "us.gcr.io/broad-gotc-prod/pytools:1.0.0-1661263730" + String docker_image = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.1-1686932671" Int cpu = 1 Int disk = ceil(3 * (size(input_bam, "GiB"))) + 100 Int machine_mem_mb = 4000 @@ -271,7 +271,7 @@ task MakeCompliantBAM { command { set -euo pipefail - /usr/gitc/makeCompliantBAM.py --input-bam ~{input_bam} --output-bam ~{output_bam_filename} + /warptools/scripts/makeCompliantBAM.py --input-bam ~{input_bam} --output-bam ~{output_bam_filename} } output { @@ -291,7 +291,7 @@ task MakeCompliantBAM { task BreakoutSnap { input { File snap_input - String docker_image = "us.gcr.io/broad-gotc-prod/pytools:1.0.0-1661263730" + String docker_image = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.1-1686932671" String bin_size_list String input_id Int preemptible = 3 @@ -310,7 +310,7 @@ task BreakoutSnap { command { set -euo pipefail mkdir output - python3 /usr/gitc/breakoutSnap.py --input ~{snap_input} \ + python3 /warptools/scripts/breakoutSnap.py --input ~{snap_input} \ --output-prefix output/~{input_id}_ } diff --git a/pipelines/skylab/slideseq/SlideSeq.changelog.md b/pipelines/skylab/slideseq/SlideSeq.changelog.md index 2252c8f60..bcb9e3c85 100644 --- a/pipelines/skylab/slideseq/SlideSeq.changelog.md +++ b/pipelines/skylab/slideseq/SlideSeq.changelog.md @@ -1,3 +1,8 @@ +# 1.0.10 +2023-07-07 (Date of Last Commit) + +* Changed Pytools container to Warp-tools in StarAlign and LoomUtils, and updated command paths accordingly + # 1.0.9 2023-06-14 (Date of Last Commit) diff --git a/pipelines/skylab/slideseq/SlideSeq.wdl b/pipelines/skylab/slideseq/SlideSeq.wdl index ec631d8d2..5328906c2 100644 --- a/pipelines/skylab/slideseq/SlideSeq.wdl +++ b/pipelines/skylab/slideseq/SlideSeq.wdl @@ -23,7 +23,7 @@ import "../../../tasks/skylab/MergeSortBam.wdl" as Merge workflow SlideSeq { - String pipeline_version = "1.0.9" + String pipeline_version = "1.0.10" input { Array[File] r1_fastq diff --git a/pipelines/skylab/smartseq2_multisample/MultiSampleSmartSeq2.changelog.md b/pipelines/skylab/smartseq2_multisample/MultiSampleSmartSeq2.changelog.md index 5bc4212b9..64eb8bc7f 100644 --- a/pipelines/skylab/smartseq2_multisample/MultiSampleSmartSeq2.changelog.md +++ b/pipelines/skylab/smartseq2_multisample/MultiSampleSmartSeq2.changelog.md @@ -1,3 +1,8 @@ +# 2.2.22 +2023-07-07 (Date of Last Commit) + +* Changed Pytools container to Warp-tools in LoomUtils, and updated command paths accordingly + # 2.2.21 2023-04-19 (Date of Last Commit) diff --git a/pipelines/skylab/smartseq2_multisample/MultiSampleSmartSeq2.wdl b/pipelines/skylab/smartseq2_multisample/MultiSampleSmartSeq2.wdl index 91c9d4f88..0717f23b7 100644 --- a/pipelines/skylab/smartseq2_multisample/MultiSampleSmartSeq2.wdl +++ b/pipelines/skylab/smartseq2_multisample/MultiSampleSmartSeq2.wdl @@ -40,7 +40,7 @@ workflow MultiSampleSmartSeq2 { Boolean paired_end } # Version of this pipeline - String pipeline_version = "2.2.21" + String pipeline_version = "2.2.22" if (false) { String? none = "None" diff --git a/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md b/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md index e36c8b804..92724debf 100644 --- a/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md +++ b/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md @@ -1,8 +1,10 @@ # 1.2.25 2023-07-11 (Date of Last Commit) +* Changed Pytools container to Warp-tools in LoomUtils, and updated command paths accordingly * Updated the CountAlignments task in the FeatureCounts.wdl to use a new docker image. This change does not affect the MultiSampleSmartSeq2SingleNucleus pipeline + # 1.2.24 2023-06-23 (Date of Last Commit) diff --git a/pipelines/skylab/smartseq2_single_sample/SmartSeq2SingleSample.changelog.md b/pipelines/skylab/smartseq2_single_sample/SmartSeq2SingleSample.changelog.md index 421964d45..eb0432276 100644 --- a/pipelines/skylab/smartseq2_single_sample/SmartSeq2SingleSample.changelog.md +++ b/pipelines/skylab/smartseq2_single_sample/SmartSeq2SingleSample.changelog.md @@ -1,3 +1,8 @@ +# 5.1.21 +2023-07-07 (Date of Last Commit) + +* Changed Pytools container to Warp-tools in LoomUtils, and updated command paths accordingly + # 5.1.20 2023-04-19 (Date of Last Commit) diff --git a/pipelines/skylab/smartseq2_single_sample/SmartSeq2SingleSample.wdl b/pipelines/skylab/smartseq2_single_sample/SmartSeq2SingleSample.wdl index efec1c416..b9df38485 100644 --- a/pipelines/skylab/smartseq2_single_sample/SmartSeq2SingleSample.wdl +++ b/pipelines/skylab/smartseq2_single_sample/SmartSeq2SingleSample.wdl @@ -36,7 +36,7 @@ workflow SmartSeq2SingleSample { } # version of this pipeline - String pipeline_version = "5.1.20" + String pipeline_version = "5.1.21" parameter_meta { genome_ref_fasta: "Genome reference in fasta format" diff --git a/tasks/skylab/StarAlign.wdl b/tasks/skylab/StarAlign.wdl index c3a59df19..aa46396b6 100644 --- a/tasks/skylab/StarAlign.wdl +++ b/tasks/skylab/StarAlign.wdl @@ -401,7 +401,7 @@ task MergeStarOutput { String input_id #runtime values - String docker = "us.gcr.io/broad-gotc-prod/pytools:1.0.0-1661263730" + String docker = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.1-1686932671" Int machine_mem_mb = 8250 Int cpu = 1 Int disk = ceil(size(matrix, "Gi") * 2) + 10 @@ -426,7 +426,7 @@ task MergeStarOutput { declare -a matrix_files=(~{sep=' ' matrix}) # create the compressed raw count matrix with the counts, gene names and the barcodes - python3 /usr/gitc/create-merged-npz-output.py \ + python3 /warptools/scripts/create-merged-npz-output.py \ --barcodes ${barcodes_files[@]} \ --features ${features_files[@]} \ --matrix ${matrix_files[@]} \ diff --git a/tests/skylab/smartseq2_multisample/pr/ValidateMultiSampleSmartSeq2.wdl b/tests/skylab/smartseq2_multisample/pr/ValidateMultiSampleSmartSeq2.wdl index cdfbbeb4e..e66a54a29 100644 --- a/tests/skylab/smartseq2_multisample/pr/ValidateMultiSampleSmartSeq2.wdl +++ b/tests/skylab/smartseq2_multisample/pr/ValidateMultiSampleSmartSeq2.wdl @@ -13,12 +13,12 @@ task ValidateSmartSeq2Plate { # catch intermittent failures set -eo pipefail - python3 /usr/gitc/loomCompare.py --truth-loom ~{truth_loom} --check-loom ~{loom_output} --delta-cutoff 10 + python3 /warptools/scripts/loomCompare.py --truth-loom ~{truth_loom} --check-loom ~{loom_output} --delta-cutoff 10 >>> runtime { - docker: "us.gcr.io/broad-gotc-prod/pytools:1.0.0-1661263730" + docker: "us.gcr.io/broad-gotc-prod/warp-tools:1.0.1-1686932671" cpu: 1 memory: "8 GiB" disks: "local-disk 1${disk_size} HDD" diff --git a/tests/skylab/smartseq2_single_nucleus/pr/ValidateSmartSeq2SingleNucleus.wdl b/tests/skylab/smartseq2_single_nucleus/pr/ValidateSmartSeq2SingleNucleus.wdl index 80880c2b3..4f43b068d 100644 --- a/tests/skylab/smartseq2_single_nucleus/pr/ValidateSmartSeq2SingleNucleus.wdl +++ b/tests/skylab/smartseq2_single_nucleus/pr/ValidateSmartSeq2SingleNucleus.wdl @@ -16,7 +16,7 @@ task ValidateSnSmartSeq2 { set -eo pipefail #compare looms - python3 /usr/gitc/loomCompare.py --truth-loom ~{truth_loom} --check-loom ~{loom_output} --delta-cutoff 10 + python3 /warptools/scripts/loomCompare.py --truth-loom ~{truth_loom} --check-loom ~{loom_output} --delta-cutoff 10 # calculate hashes; awk is used to extract the hash from the md5sum output that contains both # a hash and the filename that was passed. We parse the first 7 columns because a bug in RSEM @@ -32,7 +32,7 @@ task ValidateSnSmartSeq2 { >>> runtime { - docker: "us.gcr.io/broad-gotc-prod/pytools:1.0.0-1661263730" + docker: "us.gcr.io/broad-gotc-prod/warp-tools:1.0.1-1686932671" cpu: 1 memory: "8 GB" disks: "local-disk 1${disk_size} HDD" diff --git a/website/docs/Pipelines/Single_Cell_ATAC_Seq_Pipeline/README.md b/website/docs/Pipelines/Single_Cell_ATAC_Seq_Pipeline/README.md index ed577397c..8af34e922 100644 --- a/website/docs/Pipelines/Single_Cell_ATAC_Seq_Pipeline/README.md +++ b/website/docs/Pipelines/Single_Cell_ATAC_Seq_Pipeline/README.md @@ -109,11 +109,11 @@ The SnapCellByBin task uses the Snap file to create cell-by-bin count matrices i #### MakeCompliantBAM -The MakeCompliantBAM task uses a [custom python script (here)](https://github.com/broadinstitute/warp/blob/develop/dockers/skylab/pytools/tools/makeCompliantBAM.py) to make a GA4GH compliant BAM by moving the cellular barcodes in the read names to the CB tag. +The MakeCompliantBAM task uses a [custom python script (here)](https://github.com/broadinstitute/warp-tools/blob/develop/tools/scripts/makeCompliantBAM.py) to make a GA4GH compliant BAM by moving the cellular barcodes in the read names to the CB tag. #### BreakoutSnap -The BreakoutSnap task extracts data from the Snap file and exports it to individual CSVs. These CSV outputs are listed in the table in the Outputs section below. The code is available [here](https://github.com/broadinstitute/warp/tree/master/dockers/skylab/snap-breakout/breakoutSnap.py). +The BreakoutSnap task extracts data from the Snap file and exports it to individual CSVs. These CSV outputs are listed in the table in the Outputs section below. The code is available [here](https://github.com/broadinstitute/warp-tools/blob/develop/tools/scripts/breakoutSnap.py). ## Outputs