diff --git a/deeprvat/preprocessing/preprocess.py b/deeprvat/preprocessing/preprocess.py index 71745c13..e979912b 100644 --- a/deeprvat/preprocessing/preprocess.py +++ b/deeprvat/preprocessing/preprocess.py @@ -255,6 +255,7 @@ def process_sparse_gt( if chromosomes is not None: chromosomes = [f"chr{chrom}" for chrom in chromosomes.split(",")] variants = variants[variants["chrom"].isin(chromosomes)] + total_variants = len(variants) if len(exclude_variants) > 0: variant_exclusion_files = [ @@ -267,17 +268,15 @@ def process_sparse_gt( ], ignore_index=True, ) - if chromosomes is not None: - variants_to_exclude = variants_to_exclude[ - variants_to_exclude["chrom"].isin(chromosomes) - ] + variants_to_exclude = variants_to_exclude.drop_duplicates(ignore_index=True) variant_ids_to_exclude = pd.merge( variants_to_exclude, variants, validate="1:1" )["id"] + variants = variants[~variants["id"].isin(variant_ids_to_exclude)] if not skip_sanity_checks: - assert total_variants - len(variants) == len(variants_to_exclude) + assert total_variants - len(variants) == len(variant_ids_to_exclude) logging.info(f"Dropped {total_variants - len(variants)} variants") logging.info(f"...done ({time.time() - start_time} s)") diff --git a/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/expected/expected_data.npz b/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/expected/expected_data.npz new file mode 100644 index 00000000..ac609b3d Binary files /dev/null and b/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/expected/expected_data.npz differ diff --git a/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/qc/input_c1_b1.tsv.gz b/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/qc/input_c1_b1.tsv.gz new file mode 100644 index 00000000..07b6dbc7 Binary files /dev/null and b/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/qc/input_c1_b1.tsv.gz differ diff --git a/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/qc/input_c1_b2.tsv.gz b/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/qc/input_c1_b2.tsv.gz new file mode 100644 index 00000000..95c29b15 Binary files /dev/null and b/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/qc/input_c1_b2.tsv.gz differ diff --git a/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/samples_chr.csv b/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/samples_chr.csv new file mode 100644 index 00000000..ea0a0082 --- /dev/null +++ b/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/samples_chr.csv @@ -0,0 +1,11 @@ +100096 +100097 +100099 +100100 +100101 +100102 +100103 +100104 +100105 +100106 +100107 diff --git a/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/sparse_gt/chr1/input_c1_b1.tsv.gz b/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/sparse_gt/chr1/input_c1_b1.tsv.gz new file mode 100644 index 00000000..de13378e Binary files /dev/null and b/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/sparse_gt/chr1/input_c1_b1.tsv.gz differ diff --git a/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/variants.parquet b/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/variants.parquet new file mode 100644 index 00000000..df779fb3 Binary files /dev/null and b/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/variants.parquet differ diff --git a/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/variants.tsv.gz b/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/variants.tsv.gz new file mode 100644 index 00000000..6da9e9cd Binary files /dev/null and b/tests/preprocessing/test_data/process_sparse_gt/filter_variants_multiple/input/variants.tsv.gz differ diff --git a/tests/preprocessing/test_preprocess.py b/tests/preprocessing/test_preprocess.py index db3a3084..a8fc0415 100644 --- a/tests/preprocessing/test_preprocess.py +++ b/tests/preprocessing/test_preprocess.py @@ -51,6 +51,16 @@ def load_h5_archive(h5_path): ], "genotypes_chr1.h5", ), + ( + "filter_variants_multiple", + [ + "--chromosomes", + "1", + "--exclude-variants", + f"{(tests_data_dir / 'process_sparse_gt/filter_variants_multiple/input/qc').as_posix()}", + ], + "genotypes_chr1.h5", + ), ( "filter_samples_minimal", [