Skip to content

Commit

Permalink
Update tests to use the merged weighted_tokens query, and calcuate …
Browse files Browse the repository at this point in the history 
…NDCG based on a smaller version of the queries used to test performance.
  • Loading branch information
@kderusso
kderusso committed Jan 4, 2024
1 parent 09a3031 commit ac79e54
Show file tree
Hide file tree
Showing 6 changed files with 9,609 additions and 228 deletions.
6 changes: 3 additions & 3 deletions msmarco-passage-ranking/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ It's important to highlight that the text-expansion and hybrid strategies are de

### Example Document

Documents adhere to the [JSON Lines format](https://jsonlines.org/).
Documents adhere to the [JSON Lines format](https://jsonlines.org/).
When a single document is pretty printed, it takes the following example format:

<details>
Expand Down Expand Up @@ -454,7 +454,7 @@ EnsembleDistil](https://huggingface.co/naver/splade-cocondenser-ensembledistil)

### Example Query

Queries are structured within a JSON array, where each individual object signifies a unique 'query' and its corresponding expansion achieved through ELSER v2, which is stored pre-computed in the 'query_expansion' field.:
Queries are structured within a JSON array, where each individual object signifies a unique 'query' and its corresponding expansion achieved through ELSER v2, which is stored pre-computed in the 'text_expansion_elser' field.:

<details>
<summary><i>Example query object</i></summary>
Expand Down Expand Up @@ -639,4 +639,4 @@ title = {From Distillation to Hard Negative Sampling: Making Sparse Neural IR Mo
publisher = {arXiv},
year = {2022},
copyright = {Creative Commons Attribution Non Commercial Share Alike 4.0 International}
}
}
37 changes: 22 additions & 15 deletions msmarco-passage-ranking/challenges/default.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
},
{
"operation": {
"include-in-reporting": false,
"operation-type": "bulk",
"bulk-size": {{bulk_size | default(500)}},
"ingest-percentage": {{ingest_percentage | default(100)}}
Expand Down Expand Up @@ -67,30 +66,44 @@
"iterations": 1000,
"clients": {{search_clients | default(1)}}
},
{
"operation": "text-expansion-splade-search-maxwand-enabled",
"warmup-iterations": 100,
"iterations": 1000,
"clients": {{search_clients | default(1)}}
},
{
"operation": "text-expansion-splade-search-maxwand-disabled",
"warmup-iterations": 100,
"iterations": 1000,
"clients": {{search_clients | default(1)}}
},
{
"operation": "pruned-text-expansion-search-maxwand-enabled",
"warmup-iterations": 100,
"iterations": 1000,
"clients": {{search_clients | default(1)}}
},
},
{
"operation": "pruned-text-expansion-search-maxwand-disabled",
"warmup-iterations": 100,
"iterations": 1000,
"clients": {{search_clients | default(1)}}
},
},
{
"operation": "pruned-all-score-text-expansion-search-maxwand-enabled",
"operation": "pruned-rescored-text-expansion-search-maxwand-enabled",
"warmup-iterations": 100,
"iterations": 1000,
"num_candidates": 100,
"clients": {{search_clients | default(1)}}
},
},
{
"operation": "pruned-all-score-text-expansion-search-maxwand-disabled",
"operation": "pruned-rescored-text-expansion-search-maxwand-disabled",
"warmup-iterations": 100,
"iterations": 1000,
"num_candidates": 100,
"clients": {{search_clients | default(1)}}
},
},
{
"operation": "bm25-search-maxwand-enabled",
"warmup-iterations": 100,
Expand All @@ -116,16 +129,10 @@
"clients": {{search_clients | default(1)}}
},
{
"operation": "weighted-terms-recall-10-10"
},
{
"operation": "weighted-terms-recall-10-100"
},
{
"operation": "weighted-terms-recall-100-100"
"operation": "pruned-weighted-terms-recall-10-10"
},
{
"operation": "weighted-terms-recall-100-1000"
"operation": "pruned-weighted-terms-recall-10-100"
}
]
}
290 changes: 152 additions & 138 deletions msmarco-passage-ranking/operations/default.json
View file
Original file line number Diff line number Diff line change
@@ -1,138 +1,152 @@
{
"name": "text-expansion-search-maxwand-disabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"text_expansion_field": "text_expansion_elser",
"track_total_hits": true
},
{
"name": "text-expansion-search-maxwand-enabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"text_expansion_field": "text_expansion_elser",
"track_total_hits": false
},
{
"name": "text-expansion-splade-search-maxwand-disabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"text_expansion_field": "text_expansion_splade",
"track_total_hits": true
},
{
"name": "text-expansion-splade-search-maxwand-enabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"text_expansion_field": "text_expansion_splade",
"track_total_hits": false
},
{
"name": "pruned-text-expansion-search-maxwand-disabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"pruning_strategy": "REMOVE_HIGH_FREQUENCY_TERMS",
"track_total_hits": true
},
{
"name": "pruned-text-expansion-search-maxwand-enabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"pruning_strategy": "REMOVE_HIGH_FREQUENCY_TERMS",
"track_total_hits": false
},
{
"name": "pruned-all-score-text-expansion-search-maxwand-disabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"pruning_strategy": "ONLY_SCORE_LOW_FREQUENCY_TERMS",
"track_total_hits": true
},
{
"name": "pruned-all-score-text-expansion-search-maxwand-enabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"pruning_strategy": "ONLY_SCORE_LOW_FREQUENCY_TERMS",
"track_total_hits": false
},
{
"name": "bm25-search-maxwand-disabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "bm25",
"track_total_hits": true
},
{
"name": "bm25-search-maxwand-enabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "bm25",
"track_total_hits": false
},
{
"name": "hybrid-search-maxwand-disabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "hybrid",
"text_expansion_field": "text_expansion_elser",
"track_total_hits": true
},
{
"name": "hybrid-search-maxwand-enabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "hybrid",
"text_expansion_field": "text_expansion_elser",
"track_total_hits": false
},
{
"name": "weighted-terms-recall-10-10",
"operation-type": "weighted_terms_recall",
"param-source": "weighted_terms_recall_param_source",
"top_k": 10,
"num_candidates": 10,
"include-in-reporting": false
},
{
"name": "weighted-terms-recall-10-100",
"operation-type": "weighted_terms_recall",
"param-source": "weighted_terms_recall_param_source",
"top_k": 10,
"num_candidates": 100,
"include-in-reporting": false
},
{
"name": "weighted-terms-recall-100-100",
"operation-type": "weighted_terms_recall",
"param-source": "weighted_terms_recall_param_source",
"top_k": 100,
"num_candidates": 100,
"include-in-reporting": false
},
{
"name": "weighted-terms-recall-100-1000",
"operation-type": "weighted_terms_recall",
"param-source": "weighted_terms_recall_param_source",
"top_k": 100,
"num_candidates": 1000,
"include-in-reporting": false
}
{
"name": "text-expansion-search-maxwand-disabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"text_expansion_field": "text_expansion_elser",
"track_total_hits": true
},
{
"name": "text-expansion-search-maxwand-enabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"text_expansion_field": "text_expansion_elser",
"track_total_hits": false
},
{
"name": "text-expansion-splade-search-maxwand-disabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"text_expansion_field": "text_expansion_splade",
"track_total_hits": true
},
{
"name": "text-expansion-splade-search-maxwand-enabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"text_expansion_field": "text_expansion_splade",
"track_total_hits": false
},
{
"name": "pruned-text-expansion-search-maxwand-disabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"track_total_hits": true,
"prune": true
},
{
"name": "pruned-text-expansion-search-maxwand-enabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"track_total_hits": false,
"prune": true
},
{
"name": "pruned-rescored-text-expansion-search-maxwand-disabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"track_total_hits": true,
"prune": true,
"rescore": true
},
{
"name": "pruned-rescored-text-expansion-search-maxwand-enabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "text_expansion",
"track_total_hits": false,
"prune": true,
"rescore": true
},
{
"name": "bm25-search-maxwand-disabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "bm25",
"track_total_hits": true
},
{
"name": "bm25-search-maxwand-enabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "bm25",
"track_total_hits": false
},
{
"name": "hybrid-search-maxwand-disabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "hybrid",
"text_expansion_field": "text_expansion_elser",
"track_total_hits": true
},
{
"name": "hybrid-search-maxwand-enabled",
"operation-type": "search",
"param-source": "query_param_source",
"query_source": "queries.json",
"query_strategy": "hybrid",
"text_expansion_field": "text_expansion_elser",
"track_total_hits": false
},
{
"name": "pruned-weighted-terms-recall-10-10",
"operation-type": "weighted_terms_recall",
"param-source": "weighted_terms_recall_param_source",
"top_k": 10,
"num_candidates": 10,
"query_source": "queries-small.json",
"qrels_source": "qrels-small.tsv",
"text_expansion_field": "text_expansion_elser",
"include-in-reporting": true
},
{
"name": "pruned-weighted-terms-recall-10-100",
"operation-type": "weighted_terms_recall",
"param-source": "weighted_terms_recall_param_source",
"top_k": 10,
"num_candidates": 100,
"query_source": "queries-small.json",
"qrels_source": "qrels-small.tsv",
"text_expansion_field": "text_expansion_elser",
"include-in-reporting": true
},
{
"name": "pruned-weighted-terms-recall-100-100",
"operation-type": "weighted_terms_recall",
"param-source": "weighted_terms_recall_param_source",
"top_k": 100,
"num_candidates": 100,
"query_source": "queries-small.json",
"qrels_source": "qrels-small.tsv",
"text_expansion_field": "text_expansion_elser",
"include-in-reporting": true
},
{
"name": "pruned-weighted-terms-recall-100-1000",
"operation-type": "weighted_terms_recall",
"param-source": "weighted_terms_recall_param_source",
"top_k": 100,
"num_candidates": 1000,
"query_source": "queries-small.json",
"qrels_source": "qrels-small.tsv",
"text_expansion_field": "text_expansion_elser",
"include-in-reporting": true
}
Loading

0 comments on commit ac79e54

Please sign in to comment.