Skip to content

Commit

Permalink
Allow skip of parquet file, fix bug if no pq file is returned
Browse files Browse the repository at this point in the history
  • Loading branch information
PGijsbers committed Jan 25, 2025
1 parent 2a1af77 commit 70e9901
Showing 1 changed file with 5 additions and 2 deletions.
7 changes: 5 additions & 2 deletions openml/datasets/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import gzip
import logging
import os
import pickle
import re
import warnings
Expand Down Expand Up @@ -358,8 +359,10 @@ def _download_data(self) -> None:
# import required here to avoid circular import.
from .functions import _get_dataset_arff, _get_dataset_parquet

if self._parquet_url is not None:
self.parquet_file = str(_get_dataset_parquet(self))
skip_parquet = os.environ.get("OPENML_SKIP_PQ", "false").casefold() == "true"
if self._parquet_url is not None and not skip_parquet:
parquet_file = _get_dataset_parquet(self)
self.parquet_file = None if parquet_file is None else str(parquet_file)
if self.parquet_file is None:
self.data_file = str(_get_dataset_arff(self))

Expand Down

0 comments on commit 70e9901

Please sign in to comment.