Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove from_config and only keep from_configs #659

Merged
merged 1 commit into from
Dec 4, 2024
Merged

Remove from_config and only keep from_configs #659

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
226 changes: 147 additions & 79 deletions amlb/resources.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
**resources** modules exposes a singleton ``Resources`` instance providing easy access to app configuration properties,
as well as handy methods to access other resources like *automl frameworks* and *benchmark definitions*
"""

from __future__ import annotations

import copy
Expand All @@ -13,7 +14,15 @@

from amlb.benchmarks.parser import benchmark_load
from amlb.frameworks import default_tag, load_framework_definitions
from .utils import Namespace, lazy_property, memoize, normalize_path, run_cmd, str_sanitize, touch
from .utils import (
Namespace,
lazy_property,
memoize,
normalize_path,
run_cmd,
str_sanitize,
touch,
)
from .utils.config import TransformRule, config_load, transform_config
from .__version__ import __version__, _dev_version as dev

Expand All @@ -22,7 +31,6 @@


class Resources:

@staticmethod
def _normalize(config: Namespace, replace=None):
def nz_path(path):
Expand All @@ -34,8 +42,10 @@ def nz_path(path):
for k, v in config:
if isinstance(v, Namespace):
normalized[k] = Resources._normalize(v, replace=replace)
elif re.search(r'_(dir|file|cmd)s?$', k):
normalized[k] = [nz_path(p) for p in v] if isinstance(v, list) else nz_path(v)
elif re.search(r"_(dir|file|cmd)s?$", k):
normalized[k] = (
[nz_path(p) for p in v] if isinstance(v, list) else nz_path(v)
)
return normalized

def __init__(self, config: Namespace):
Expand All @@ -51,20 +61,16 @@ def __init__(self, config: Namespace):
log.debug("Using config:\n%s", self.config)

# allowing to load custom modules from user directory
sys.path.append(common_dirs['user'])
sys.path.append(common_dirs["user"])
log.debug("Extended Python sys.path to user directory: %s.", sys.path)

@lazy_property
def project_info(self):
split_url = self.config.project_repository.split('#', 1)
split_url = self.config.project_repository.split("#", 1)
repo = split_url[0]
tag = None if len(split_url) == 1 else split_url[1]
branch = tag or 'master'
return Namespace(
repo=repo,
tag=tag,
branch=branch
)
branch = tag or "master"
return Namespace(repo=repo, tag=tag, branch=branch)

@lazy_property
def git_info(self):
Expand All @@ -88,11 +94,7 @@ def git(cmd, defval=None):
repo = branch = commit = na
tags = status = []
return Namespace(
repo=repo,
branch=branch,
commit=commit,
tags=tags,
status=status
repo=repo, branch=branch, commit=commit, tags=tags, status=status
)

@lazy_property
Expand All @@ -109,17 +111,19 @@ def app_version(self):
return "{v} [{details}]".format(v=v, details=", ".join(tokens))

def seed(self, fold=None):
if isinstance(fold, int) and str(self.config.seed).lower() in ['auto']:
return fold+self._seed
if isinstance(fold, int) and str(self.config.seed).lower() in ["auto"]:
return fold + self._seed
else:
return self._seed

@lazy_property
def _seed(self):
if str(self.config.seed).lower() in ['none', '']:
if str(self.config.seed).lower() in ["none", ""]:
return None
elif str(self.config.seed).lower() in ['auto']:
return random.randint(1, (1 << 31) - 1) # limiting seed to signed int32 for R frameworks
elif str(self.config.seed).lower() in ["auto"]:
return random.randint(
1, (1 << 31) - 1
) # limiting seed to signed int32 for R frameworks
else:
return self.config.seed

Expand All @@ -132,20 +136,34 @@ def framework_definition(self, name, tag=None):
if tag is None:
tag = default_tag
if tag not in self._frameworks:
raise ValueError("Incorrect tag `{}`: only those among {} are allowed.".format(tag, self.config.frameworks.tags))
raise ValueError(
"Incorrect tag `{}`: only those among {} are allowed.".format(
tag, self.config.frameworks.tags
)
)
frameworks = self._frameworks[tag]
log.debug("Available framework definitions:\n%s", frameworks)
framework = next((f for n, f in frameworks if n.lower() == lname), None)
# TODO: Clean up this workflow and error messaging as part of #518
base_framework = next((f for n, f in self._frameworks[default_tag] if n.lower() == lname), None)
if framework and framework['removed']:
raise ValueError(f"Framework definition `{name}` has been removed from the benchmark: {framework['removed']}")
if not framework and (base_framework and base_framework['removed']):
raise ValueError(f"Framework definition `{name}` has been removed from the benchmark: {base_framework['removed']}")
base_framework = next(
(f for n, f in self._frameworks[default_tag] if n.lower() == lname), None
)
if framework and framework["removed"]:
raise ValueError(
f"Framework definition `{name}` has been removed from the benchmark: {framework['removed']}"
)
if not framework and (base_framework and base_framework["removed"]):
raise ValueError(
f"Framework definition `{name}` has been removed from the benchmark: {base_framework['removed']}"
)
if not framework:
raise ValueError(f"Incorrect framework `{name}`: not listed in {self.config.frameworks.definition_file}.")
if framework['abstract']:
raise ValueError(f"Framework definition `{name}` is abstract and cannot be run directly.")
raise ValueError(
f"Incorrect framework `{name}`: not listed in {self.config.frameworks.definition_file}."
)
if framework["abstract"]:
raise ValueError(
f"Framework definition `{name}` is abstract and cannot be run directly."
)
return framework, framework.name

@lazy_property
Expand All @@ -161,7 +179,11 @@ def constraint_definition(self, name):
"""
constraint = self._constraints[name.lower()]
if not constraint:
raise ValueError("Incorrect constraint definition `{}`: not listed in {}.".format(name, self.config.benchmarks.constraints_file))
raise ValueError(
"Incorrect constraint definition `{}`: not listed in {}.".format(
name, self.config.benchmarks.constraints_file
)
)
return constraint, constraint.name

@lazy_property
Expand Down Expand Up @@ -191,11 +213,15 @@ def benchmark_definition(self, name, defaults=None):
:param defaults: defaults used as a base config for each task in the benchmark definition
:return:
"""
hard_defaults, tasks, benchmark_path, benchmark_name = benchmark_load(name, self.config.benchmarks.definition_dir)
hard_defaults, tasks, benchmark_path, benchmark_name = benchmark_load(
name, self.config.benchmarks.definition_dir
)

defaults = Namespace.merge(defaults, hard_defaults, Namespace(name='__defaults__'))
defaults = Namespace.merge(
defaults, hard_defaults, Namespace(name="__defaults__")
)
for task in tasks:
task |= defaults # add missing keys from hard defaults + defaults
task |= defaults # add missing keys from hard defaults + defaults
self._validate_task(task)

self._validate_task(defaults, lenient=True)
Expand All @@ -206,66 +232,98 @@ def benchmark_definition(self, name, defaults=None):

def _validate_task(self, task, lenient=False):
missing = []
for conf in ['name']:
for conf in ["name"]:
if task[conf] is None:
missing.append(conf)
if not lenient and len(missing) > 0:
raise ValueError("{missing} mandatory properties as missing in task definition {taskdef}.".format(missing=missing, taskdef=task))

for conf in ['max_runtime_seconds', 'cores', 'folds', 'max_mem_size_mb', 'min_vol_size_mb', 'quantile_levels']:
raise ValueError(
"{missing} mandatory properties as missing in task definition {taskdef}.".format(
missing=missing, taskdef=task
)
)

for conf in [
"max_runtime_seconds",
"cores",
"folds",
"max_mem_size_mb",
"min_vol_size_mb",
"quantile_levels",
]:
if task[conf] is None:
task[conf] = self.config.benchmarks.defaults[conf]
log.debug("Config `{config}` not set for task {name}, using default `{value}`.".format(config=conf, name=task.name, value=task[conf]))
log.debug(
"Config `{config}` not set for task {name}, using default `{value}`.".format(
config=conf, name=task.name, value=task[conf]
)
)

conf = 'id'
conf = "id"
if task[conf] is None:
task[conf] = ("openml.org/t/{}".format(task.openml_task_id) if task['openml_task_id'] is not None
else "openml.org/d/{}".format(task.openml_dataset_id) if task['openml_dataset_id'] is not None
else ((task.dataset['id'] if isinstance(task.dataset, (dict, Namespace))
else task.dataset if isinstance(task.dataset, str)
else None) or task.name) if task['dataset'] is not None
else None)
task[conf] = (
"openml.org/t/{}".format(task.openml_task_id)
if task["openml_task_id"] is not None
else "openml.org/d/{}".format(task.openml_dataset_id)
if task["openml_dataset_id"] is not None
else (
(
task.dataset["id"]
if isinstance(task.dataset, (dict, Namespace))
else task.dataset
if isinstance(task.dataset, str)
else None
)
or task.name
)
if task["dataset"] is not None
else None
)
if not lenient and task[conf] is None:
raise ValueError("task definition must contain an ID or one property "
"among ['openml_task_id', 'dataset'] to create an ID, "
"but task definition is {task}".format(task=str(task)))
raise ValueError(
"task definition must contain an ID or one property "
"among ['openml_task_id', 'dataset'] to create an ID, "
"but task definition is {task}".format(task=str(task))
)

conf = 'metric'
conf = "metric"
if task[conf] is None:
task[conf] = None

conf = 'ec2_instance_type'
conf = "ec2_instance_type"
if task[conf] is None:
i_series = self.config.aws.ec2.instance_type.series
i_map = self.config.aws.ec2.instance_type.map
if str(task.cores) in i_map:
i_size = i_map[str(task.cores)]
elif task.cores > 0:
supported_cores = list(map(int, Namespace.dict(i_map).keys() - {'default'}))
supported_cores = list(
map(int, Namespace.dict(i_map).keys() - {"default"})
)
supported_cores.sort()
cores = next((c for c in supported_cores if c >= task.cores), 'default')
cores = next((c for c in supported_cores if c >= task.cores), "default")
i_size = i_map[str(cores)]
else:
i_size = i_map.default
task[conf] = '.'.join([i_series, i_size])
log.debug("Config `{config}` not set for task {name}, using default selection `{value}`.".format(config=conf, name=task.name, value=task[conf]))

conf = 'ec2_volume_type'
task[conf] = ".".join([i_series, i_size])
log.debug(
"Config `{config}` not set for task {name}, using default selection `{value}`.".format(
config=conf, name=task.name, value=task[conf]
)
)

conf = "ec2_volume_type"
if task[conf] is None:
task[conf] = self.config.aws.ec2.volume_type
log.debug("Config `{config}` not set for task {name}, using default `{value}`.".format(config=conf, name=task.name, value=task[conf]))
log.debug(
"Config `{config}` not set for task {name}, using default `{value}`.".format(
config=conf, name=task.name, value=task[conf]
)
)


__INSTANCE__: Resources | None = None


def from_config(config: Namespace):
global __INSTANCE__
transform_config(config, _backward_compatibility_config_rules_)
__INSTANCE__ = Resources(config)
return __INSTANCE__
Comment on lines -262 to -266
Copy link
Collaborator Author

@PGijsbers PGijsbers Nov 29, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actual change, the replacing function is directly below (unchanged).



def from_configs(*configs: Namespace):
global __INSTANCE__
for c in configs:
Expand All @@ -286,18 +344,17 @@ def config():


def output_dirs(root, session=None, subdirs=None, create=False):
root = root if root is not None else '.'
root = root if root is not None else "."
if create and not os.path.exists(root):
touch(root, as_dir=True)

dirs = Namespace(
root=root,
session=os.path.join(root, session) if session is not None else root
root=root, session=os.path.join(root, session) if session is not None else root
)

subdirs = ([] if subdirs is None
else [subdirs] if isinstance(subdirs, str)
else subdirs)
subdirs = (
[] if subdirs is None else [subdirs] if isinstance(subdirs, str) else subdirs
)

for d in subdirs:
dirs[d] = os.path.join(dirs.session, d)
Expand All @@ -307,11 +364,22 @@ def output_dirs(root, session=None, subdirs=None, create=False):


_backward_compatibility_config_rules_ = [
TransformRule(from_key='exit_on_error', to_key='job_scheduler.exit_on_job_failure'),
TransformRule(from_key='parallel_jobs', to_key='job_scheduler.parallel_jobs'),
TransformRule(from_key='max_parallel_jobs', to_key='job_scheduler.max_parallel_jobs'),
TransformRule(from_key='delay_between_jobs', to_key='job_scheduler.delay_between_jobs'),
TransformRule(from_key='monitoring.frequency_seconds', to_key='monitoring.interval_seconds'),
TransformRule(from_key='aws.query_frequency_seconds', to_key='aws.query_interval_seconds'),
TransformRule(from_key='aws.ec2.monitoring.cpu.query_frequency_seconds', to_key='aws.ec2.monitoring.cpu.query_interval_seconds'),
TransformRule(from_key="exit_on_error", to_key="job_scheduler.exit_on_job_failure"),
TransformRule(from_key="parallel_jobs", to_key="job_scheduler.parallel_jobs"),
TransformRule(
from_key="max_parallel_jobs", to_key="job_scheduler.max_parallel_jobs"
),
TransformRule(
from_key="delay_between_jobs", to_key="job_scheduler.delay_between_jobs"
),
TransformRule(
from_key="monitoring.frequency_seconds", to_key="monitoring.interval_seconds"
),
TransformRule(
from_key="aws.query_frequency_seconds", to_key="aws.query_interval_seconds"
),
TransformRule(
from_key="aws.ec2.monitoring.cpu.query_frequency_seconds",
to_key="aws.ec2.monitoring.cpu.query_interval_seconds",
),
]
Loading
Loading