diff --git a/FlagEmbedding/evaluation/beir/data_loader.py b/FlagEmbedding/evaluation/beir/data_loader.py
index cdba4c91..29c36ff9 100644
--- a/FlagEmbedding/evaluation/beir/data_loader.py
+++ b/FlagEmbedding/evaluation/beir/data_loader.py
@@ -145,7 +145,7 @@ def _load_remote_qrels(
         if dataset_name != 'cqadupstack':
             qrels = datasets.load_dataset(
                 'BeIR/{d}-qrels'.format(d=dataset_name),
-                split=split, 
+                split=split if split != 'dev' else 'validation', 
                 trust_remote_code=True,
                 cache_dir=self.cache_dir,
                 download_mode=self.hf_download_mode