Update docker images list, triggered by 6077c31

Add json template Update top-level workflows; concat variant lists Fix contig list read
broadinstitute · Jan 12, 2024 · 935dffc · 935dffc
1 parent ba672d2
commit 935dffc
Show file tree

Hide file tree

Showing 6 changed files with 61 additions and 18 deletions.
diff --git a/wdl/CleanVcf.wdl b/wdl/CleanVcf.wdl
@@ -11,8 +11,8 @@ workflow CleanVcf {
     String cohort_name
 
     Array[File] complex_genotype_vcfs
-    Array[File] complex_resolve_bothside_pass_lists
-    Array[File] complex_resolve_background_fail_lists
+    File complex_resolve_bothside_pass_list
+    File complex_resolve_background_fail_list
     File ped_file
 
     File contig_list
@@ -121,9 +121,9 @@ workflow CleanVcf {
       input:
         vcf=complex_genotype_vcfs[i],
         contig=contig,
-        background_list=complex_resolve_background_fail_lists[i],
+        background_list=complex_resolve_background_fail_list,
         ped_file=ped_file,
-        bothsides_pass_list=complex_resolve_bothside_pass_lists[i],
+        bothsides_pass_list=complex_resolve_bothside_pass_list,
         allosome_fai=allosome_fai,
         prefix="~{cohort_name}.~{contig}",
         max_shards_per_chrom_step1=max_shards_per_chrom_step1,

diff --git a/wdl/GATKSVPipelineBatch.wdl b/wdl/GATKSVPipelineBatch.wdl
@@ -467,8 +467,8 @@ workflow GATKSVPipelineBatch {
     # ResolveComplexVariants
     Array[File] complex_resolve_vcfs = MakeCohortVcf.complex_resolve_vcfs
     Array[File] complex_resolve_vcf_indexes = MakeCohortVcf.complex_resolve_vcf_indexes
-    Array[File] complex_resolve_bothside_pass_lists = MakeCohortVcf.complex_resolve_bothside_pass_lists
-    Array[File] complex_resolve_background_fail_lists = MakeCohortVcf.complex_resolve_background_fail_lists
+    File complex_resolve_bothside_pass_list = MakeCohortVcf.complex_resolve_bothside_pass_list
+    File complex_resolve_background_fail_list = MakeCohortVcf.complex_resolve_background_fail_list
     Array[File] breakpoint_overlap_dropped_record_vcfs = MakeCohortVcf.breakpoint_overlap_dropped_record_vcfs
     Array[File] breakpoint_overlap_dropped_record_vcf_indexes = MakeCohortVcf.breakpoint_overlap_dropped_record_vcf_indexes
 

diff --git a/wdl/GATKSVPipelineSingleSample.wdl b/wdl/GATKSVPipelineSingleSample.wdl
@@ -444,6 +444,8 @@ workflow GATKSVPipelineSingleSample {
     RuntimeAttr? runtime_override_update_sr_list_fail
     RuntimeAttr? runtime_override_breakpoint_overlap_filter
     RuntimeAttr? runtime_override_concat_resolve
+    RuntimeAttr? runtime_override_concat_bothside_pass
+    RuntimeAttr? runtime_override_concat_background_fail
 
     RuntimeAttr? runtime_override_get_se_cutoff
     RuntimeAttr? runtime_override_shard_vcf_cpx
@@ -1202,6 +1204,8 @@ workflow GATKSVPipelineSingleSample {
       runtime_override_update_sr_list_fail=runtime_override_update_sr_list_fail,
       runtime_override_subset_inversions=runtime_override_subset_inversions,
       runtime_override_concat_resolve=runtime_override_concat_resolve,
+      runtime_override_concat_bothside_pass=runtime_override_concat_bothside_pass,
+      runtime_override_concat_background_fail=runtime_override_concat_background_fail,
       runtime_override_get_se_cutoff=runtime_override_get_se_cutoff,
       runtime_override_shard_vcf_cpx=runtime_override_shard_vcf_cpx,
       runtime_override_shard_vids_resolve=runtime_override_shard_vids_resolve,

diff --git a/wdl/MakeCohortVcf.wdl b/wdl/MakeCohortVcf.wdl
@@ -126,6 +126,8 @@ workflow MakeCohortVcf {
     RuntimeAttr? runtime_override_update_sr_list_pass
     RuntimeAttr? runtime_override_update_sr_list_fail
     RuntimeAttr? runtime_override_concat_resolve
+    RuntimeAttr? runtime_override_concat_bothside_pass
+    RuntimeAttr? runtime_override_concat_background_fail
 
     RuntimeAttr? runtime_override_get_se_cutoff
     RuntimeAttr? runtime_override_shard_vcf_cpx
@@ -150,6 +152,9 @@ workflow MakeCohortVcf {
     RuntimeAttr? runtime_override_hail_merge_resolve_inv
     RuntimeAttr? runtime_override_fix_header_resolve_inv
 
+    # overrides for ReshardVcf
+    RuntimeAttr? runtime_override_reshard
+
     # overrides for GenotypeComplexContig
     RuntimeAttr? runtime_override_ids_from_median
     RuntimeAttr? runtime_override_split_vcf_to_genotype
@@ -306,6 +311,8 @@ workflow MakeCohortVcf {
       runtime_override_breakpoint_overlap_filter=runtime_override_breakpoint_overlap_filter,
       runtime_override_subset_inversions=runtime_override_subset_inversions,
       runtime_override_concat=runtime_override_concat_resolve,
+      runtime_override_concat_bothside_pass=runtime_override_concat_bothside_pass,
+      runtime_override_concat_background_fail=runtime_override_concat_background_fail,
 
       runtime_override_get_se_cutoff=runtime_override_get_se_cutoff,
       runtime_override_shard_vcf_cpx=runtime_override_shard_vcf_cpx,
@@ -373,8 +380,8 @@ workflow MakeCohortVcf {
     input:
       cohort_name=cohort_name,
       complex_genotype_vcfs=GenotypeComplexVariants.complex_genotype_vcfs,
-      complex_resolve_bothside_pass_lists=ResolveComplexVariants.complex_resolve_bothside_pass_lists,
-      complex_resolve_background_fail_lists=ResolveComplexVariants.complex_resolve_background_fail_lists,
+      complex_resolve_bothside_pass_list=ResolveComplexVariants.complex_resolve_bothside_pass_list,
+      complex_resolve_background_fail_list=ResolveComplexVariants.complex_resolve_background_fail_list,
       ped_file=ped_file,
       contig_list=contig_list,
       allosome_fai=allosome_fai,
@@ -507,8 +514,8 @@ workflow MakeCohortVcf {
     # ResolveComplexVariants
     Array[File] complex_resolve_vcfs = ResolveComplexVariants.complex_resolve_vcfs
     Array[File] complex_resolve_vcf_indexes = ResolveComplexVariants.complex_resolve_vcf_indexes
-    Array[File] complex_resolve_bothside_pass_lists = ResolveComplexVariants.complex_resolve_bothside_pass_lists
-    Array[File] complex_resolve_background_fail_lists = ResolveComplexVariants.complex_resolve_background_fail_lists
+    File complex_resolve_bothside_pass_list = ResolveComplexVariants.complex_resolve_bothside_pass_list
+    File complex_resolve_background_fail_list = ResolveComplexVariants.complex_resolve_background_fail_list
     Array[File] breakpoint_overlap_dropped_record_vcfs = ResolveComplexVariants.breakpoint_overlap_dropped_record_vcfs
     Array[File] breakpoint_overlap_dropped_record_vcf_indexes = ResolveComplexVariants.breakpoint_overlap_dropped_record_vcf_indexes
 

diff --git a/wdl/ReshardVcf.wdl b/wdl/ReshardVcf.wdl
@@ -3,20 +3,19 @@ version 1.0
 import "Structs.wdl"
 import "TasksMakeCohortVcf.wdl" as MiniTasks
 
-# Consumes a contig-sharded vcf where some shards may contain a small proportion of records from other
-#  contigs and moves them to their correct shards.
+# Consumes a set of vcfs and distributes them into contig shards
 
 workflow ReshardVcf {
   input {
-    Array[File] vcfs  # Must be sorted and indexed
+    Array[File] vcfs  # Order does not matter but must be sorted and indexed
     File contig_list
     String prefix
     Boolean? use_ssd
     String sv_base_mini_docker
     RuntimeAttr? runtime_override_reshard
   }
 
-  Array[String] contigs = read_lines(contig_list)
+  Array[String] contigs = transpose(read_tsv(contig_list))[0]
 
   scatter (i in range(length(vcfs))) {
     File vcf_indexes = vcfs[i] + ".tbi"

diff --git a/wdl/ResolveComplexVariants.wdl b/wdl/ResolveComplexVariants.wdl
@@ -1,5 +1,6 @@
 version 1.0
 
+import "ReshardVcf.wdl" as Reshard
 import "ResolveCpxSv.wdl" as ResolveComplexContig
 import "TasksMakeCohortVcf.wdl" as MiniTasks
 
@@ -38,6 +39,8 @@ workflow ResolveComplexVariants {
     RuntimeAttr? runtime_override_breakpoint_overlap_filter
     RuntimeAttr? runtime_override_subset_inversions
     RuntimeAttr? runtime_override_concat
+    RuntimeAttr? runtime_override_concat_bothside_pass
+    RuntimeAttr? runtime_override_concat_background_fail
 
     # overrides for ResolveComplexContig
     RuntimeAttr? runtime_override_get_se_cutoff
@@ -63,6 +66,9 @@ workflow ResolveComplexVariants {
     RuntimeAttr? runtime_override_preconcat_inv
     RuntimeAttr? runtime_override_hail_merge_inv
     RuntimeAttr? runtime_override_fix_header_inv
+
+    # overrides for ReshardVcf
+    RuntimeAttr? runtime_override_reshard
   }
 
   #Scatter per chromosome
@@ -206,6 +212,33 @@ workflow ResolveComplexVariants {
     }
   }
 
+  call Reshard.ReshardVcf {
+    input:
+      vcfs=RenameVariants.renamed_vcf,
+      contig_list=contig_list,
+      prefix="~{cohort_name}.reshard_vcf",
+      sv_base_mini_docker=sv_base_mini_docker,
+      runtime_override_reshard=runtime_override_reshard
+  }
+
+  # Due to vcf resharding, these contig-sharded lists are no longer valid
+  # Here we combine variant list arrays into single genome-wide lists
+  # This is faster than reshuffling and shouldn't have much performance impact in CleanVcf1a
+  call MiniTasks.CatUncompressedFiles as ConcatBothsidePass {
+    input:
+      shards=UpdateBothsidePass.updated_list,
+      outfile_name="~{cohort_name}.all.sr_bothside_pass.updated3.txt",
+      sv_base_mini_docker=sv_base_mini_docker,
+      runtime_attr_override=runtime_override_concat_bothside_pass
+  }
+  call MiniTasks.CatUncompressedFiles as ConcatBackgroundFail {
+    input:
+      shards=UpdateBackgroundFail.updated_list,
+      outfile_name="~{cohort_name}.all.sr_background_fail.updated3.txt",
+      sv_base_mini_docker=sv_base_mini_docker,
+      runtime_attr_override=runtime_override_concat_background_fail
+  }
+
   #Merge resolved vcfs for QC
   if (merge_vcfs) {
     call MiniTasks.ConcatVcfs {
@@ -221,10 +254,10 @@ workflow ResolveComplexVariants {
 
   #Final outputs
   output {
-    Array[File] complex_resolve_vcfs = RenameVariants.renamed_vcf
-    Array[File] complex_resolve_vcf_indexes = RenameVariants.renamed_vcf_index
-    Array[File] complex_resolve_bothside_pass_lists = UpdateBothsidePass.updated_list
-    Array[File] complex_resolve_background_fail_lists = UpdateBackgroundFail.updated_list
+    Array[File] complex_resolve_vcfs = ReshardVcf.resharded_vcfs
+    Array[File] complex_resolve_vcf_indexes = ReshardVcf.resharded_vcf_indexes
+    File complex_resolve_bothside_pass_list = ConcatBothsidePass.outfile
+    File complex_resolve_background_fail_list = ConcatBackgroundFail.outfile
     Array[File] breakpoint_overlap_dropped_record_vcfs = BreakpointOverlap.dropped_record_vcf
     Array[File] breakpoint_overlap_dropped_record_vcf_indexes = BreakpointOverlap.dropped_record_vcf_index
     File? complex_resolve_merged_vcf = ConcatVcfs.concat_vcf