From dc7d736688f9f0f4ec5e1814a86a7a17a5bfe72a Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Thu, 6 Feb 2025 16:09:49 +0100 Subject: [PATCH 01/17] update prediction scripts --- models/grace/join_grace_preds.py | 114 +++++---- models/grace/run_eval_parallel.sh | 15 +- models/grace/test_grace_discovery.py | 357 ++++++++++++++------------- 3 files changed, 268 insertions(+), 218 deletions(-) diff --git a/models/grace/join_grace_preds.py b/models/grace/join_grace_preds.py index f33cd4d4..f9f4ecba 100644 --- a/models/grace/join_grace_preds.py +++ b/models/grace/join_grace_preds.py @@ -1,5 +1,6 @@ import glob import os +import argparse # Import the argparse module import pandas as pd from pymatviz.enums import Key @@ -10,70 +11,91 @@ from matbench_discovery.enums import MbdKey, Task __author__ = "Yury Lysogorskiy" -__date__ = "2024-11-22" +__date__ = "2025-02-06" -energy_column = "grace2l_r6_energy" -e_form_grace_col = "e_form_per_atom_grace" -struct_col = "grace2l_r6_structure" +energy_column = "grace_energy" # or your actual column name. +e_form_grace_col = "e_form_per_atom_grace" # or your desired name +struct_col = "grace_structure" -module_dir = os.path.dirname(__file__) -task_type = Task.IS2RE -date = "2024-11-21" -glob_pattern = "2024-11-21-MP_GRACE_2L_r6_11Nov2024-wbm-IS2RE-FIRE/production-*.json.gz" -file_paths = glob.glob(glob_pattern) -print(f"Found {len(file_paths):,} files for {glob_pattern = }") +module_dir = os.path.dirname(__file__) +task_type = Task.IS2RE # or Task.RS2RE, depending on what you processed -dfs: list[pd.DataFrame] = [] -for fn in file_paths: - print(fn) - dfs.append(pd.read_json(fn)) +def process_results(path: str): + """ + Processes relaxation results from a given path. + Args: + path (str): The path to the directory containing the .json.gz files. + """ + glob_pattern = os.path.join(path, "production-*.json.gz") + file_paths = glob.glob(glob_pattern) -tot_df = pd.concat(dfs) -tot_df["id_tuple"] = ( - tot_df["material_id"].str.split("-").map(lambda x: (int(x[1]), int(x[2]))) -) -tot_df = ( - tot_df.sort_values("id_tuple") - .reset_index(drop=True) - .drop(columns=["id_tuple", struct_col]) -) + print(f"Found {len(file_paths):,} files for {glob_pattern = }") -df_grace = tot_df.set_index("material_id") -df_grace[Key.formula] = df_wbm[Key.formula] + if not file_paths: + print(f"No files found matching {glob_pattern}. Exiting.") + return # Exit if no files are found + dfs: list[pd.DataFrame] = [] + for fn in file_paths: + print(fn) + try: + dfs.append(pd.read_json(fn)) + except Exception as e: + print(f"Error reading {fn}: {e}") # Print any errors during file reading. + continue # Continue to the next file -print("Calculating formation energies") -e_form_list = [] -for _, row in tqdm(df_grace.iterrows(), total=len(df_grace)): - e_form = calc_energy_from_e_refs( - row["formula"], - ref_energies=mp_elemental_ref_energies, - total_energy=row[energy_column], + tot_df = pd.concat(dfs) + tot_df["id_tuple"] = ( + tot_df["material_id"].str.split("-").map(lambda x: (int(x[1]), int(x[2]))) ) - e_form_list.append(e_form) + tot_df = ( + tot_df.sort_values("id_tuple") + .reset_index(drop=True) + .drop(columns=["id_tuple", struct_col]) + ) + + df_grace = tot_df.set_index("material_id") + df_grace[Key.formula] = df_wbm[Key.formula] + + + print("Calculating formation energies") + e_form_list = [] + for _, row in tqdm(df_grace.iterrows(), total=len(df_grace)): + e_form = calc_energy_from_e_refs( + row["formula"], + ref_energies=mp_elemental_ref_energies, + total_energy=row[energy_column], + ) + e_form_list.append(e_form) + + df_grace[e_form_grace_col] = e_form_list -df_grace[e_form_grace_col] = e_form_list + df_wbm[[*df_grace]] = df_grace -df_wbm[[*df_grace]] = df_grace + # %% + bad_mask = abs(df_wbm[e_form_grace_col] - df_wbm[MbdKey.e_form_dft]) > 5 + n_preds = len(df_wbm[e_form_grace_col].dropna()) + print(f"{sum(bad_mask)=} is {sum(bad_mask) / len(df_wbm):.2%} of {n_preds:,}") + out_path = file_paths[0].rsplit("/", 1)[0] # Get directory from first file path. -# %% -bad_mask = abs(df_wbm[e_form_grace_col] - df_wbm[MbdKey.e_form_dft]) > 5 -n_preds = len(df_wbm[e_form_grace_col].dropna()) -print(f"{sum(bad_mask)=} is {sum(bad_mask) / len(df_wbm):.2%} of {n_preds:,}") -out_path = file_paths[0].rsplit("/", 1)[0] + df_grace = df_grace.round(4) + df_grace.select_dtypes("number").to_csv(f"{out_path}/{model_name}_{date}.csv.gz") #added model and date + df_grace.reset_index().to_json(f"{out_path}/{model_name}_{date}.json.gz", default_handler=as_dict_handler) #added model and date + df_bad = df_grace[bad_mask].copy() + df_bad[MbdKey.e_form_dft] = df_wbm[MbdKey.e_form_dft] + df_bad.to_csv(f"{out_path}/{model_name}_{date}_bad.csv") #added model and date -df_grace = df_grace.round(4) -df_grace.select_dtypes("number").to_csv(f"{out_path}.csv.gz") -df_grace.reset_index().to_json(f"{out_path}.json.gz", default_handler=as_dict_handler) +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Process relaxation results.") + parser.add_argument("path", type=str, help="Path to the directory with relaxation results.") -df_bad = df_grace[bad_mask].copy() -df_bad[MbdKey.e_form_dft] = df_wbm[MbdKey.e_form_dft] -df_bad.to_csv(f"{out_path}-bad.csv") + args = parser.parse_args() + process_results(args.path) diff --git a/models/grace/run_eval_parallel.sh b/models/grace/run_eval_parallel.sh index 571d840b..fd62156a 100755 --- a/models/grace/run_eval_parallel.sh +++ b/models/grace/run_eval_parallel.sh @@ -5,9 +5,13 @@ export TF_CPP_MIN_LOG_LEVEL=3 THREADS=4 # number of threads for OMP, MKL, NUMEXPR etc. To share resources on single machine -export NGPU=4 # Set the total number of GPUs here +# Define the list of available GPUs +GPUS=(0 1 2 3) # Set the specific GPU IDs you want to use here +NGPU=${#GPUS[@]} # Calculate the number of GPUs based on the array length -model_name="MP_GRACE_2L_r6_11Nov2024" # just for information, model name is hardcoded in 1_test_srme.py +#model_name="MP_GRACE_2L_r6_11Nov2024" # just for information, model name is hardcoded in 1_test_srme.py +model_name="GRACE-1L-OAM_2Feb25" +#model_name="GRACE_2L_OAM_28Jan25" export MODEL_NAME="${model_name}" echo "MODEL_NAME=${MODEL_NAME}" @@ -40,6 +44,11 @@ echo "SLURM_ARRAY_TASK_COUNT=${SLURM_ARRAY_TASK_COUNT}" echo "Running test_grace.py" for task_id in $(seq 0 $((SLURM_ARRAY_TASK_COUNT-1))) do - CUDA_VISIBLE_DEVICES=$((task_id % NGPU)) SLURM_ARRAY_TASK_ID=${task_id} python test_grace.py > "out-${task_id}.txt" 2>&1 & + # Calculate the GPU index using the modulo operator (%) + gpu_index=$((task_id % NGPU)) + # Get the actual GPU ID from the GPUS array + gpu_id=${GPUS[$gpu_index]} + + CUDA_VISIBLE_DEVICES=$gpu_id SLURM_ARRAY_TASK_ID=${task_id} python test_grace_discovery.py "${MODEL_NAME}" > "out-${task_id}.txt" 2>&1 & done wait diff --git a/models/grace/test_grace_discovery.py b/models/grace/test_grace_discovery.py index 7e4f2176..e8cf9b2f 100644 --- a/models/grace/test_grace_discovery.py +++ b/models/grace/test_grace_discovery.py @@ -1,4 +1,10 @@ # %% +import warnings + +# Ignore all RuntimeWarnings +warnings.filterwarnings("ignore", category=RuntimeWarning) + +import argparse import json import os from collections.abc import Callable @@ -23,175 +29,188 @@ from matbench_discovery.enums import Task __author__ = "Yury Lysogorskiy" -__date__ = "2024-11-21" - - -# %% -model_name = "MP_GRACE_2L_r6_11Nov2024" - -smoke_test = False -task_type = Task.IS2RE -module_dir = os.path.dirname(__file__) -# set large job array size for smaller data splits and faster testing/debugging -slurm_array_task_count = int( - os.getenv("SLURM_ARRAY_TASK_COUNT", "1") -) # will be set to the number of tasks in the job array. -ase_optimizer = "FIRE" -job_name = f"{model_name}-wbm-{task_type}-{ase_optimizer}" -out_dir = os.getenv("SBATCH_OUTPUT", f"{module_dir}/{today}-{job_name}") -device = "gpu" -# whether to record intermediate structures into pymatgen Trajectory -record_traj = False # has no effect if relax_cell is False - -ase_filter: Literal["frechet", "exp"] = "frechet" -os.makedirs(out_dir, exist_ok=True) - -# slurm_vars = slurm_submit( -# job_name=job_name, -# out_dir=out_dir, -# array=f"1-{slurm_array_task_count}", -# # slurm_flags="--qos shared --constraint gpu --gpus 1", -# slurm_flags="--ntasks=1 --cpus-per-task=1 --partition high-priority", -# ) - -slurm_vars = {k: v for k, v in os.environ.items() if k.startswith("SLURM_")} - - -# %% -slurm_array_task_id = int( - os.getenv("SLURM_ARRAY_TASK_ID", "0") -) # will be set to the job array index value. -slurm_array_job_id = os.getenv( - "SLURM_ARRAY_JOB_ID", "debug" -) # will be set to the first job ID of the array. - -out_path = f"{out_dir}/{slurm_array_job_id}-{slurm_array_task_id:>03}.json.gz" - -if os.path.isfile(out_path): - raise SystemExit(f"{out_path=} already exists, exciting early") - -print(f"{slurm_array_task_id=}") -print(f"{slurm_array_job_id=}") -print(f"{slurm_array_task_count=}") -print(f"{slurm_vars=}") -print(f"{out_dir=}") - - -# %% -data_path = { - Task.RS2RE: DataFiles.wbm_relaxed_atoms.path, - Task.IS2RE: DataFiles.wbm_initial_atoms.path, -}[task_type] -print(f"\nJob {job_name} started {timestamp}") -e_pred_col = "grace2l_r6_energy" -max_steps = 500 -force_max = 0.05 # Run until the forces are smaller than this in eV/A -checkpoint = "11Nov2024" -dtype = "float64" -calc = grace_fm( - model=model_name, pad_neighbors_fraction=0.05, pad_atoms_number=2, min_dist=0.5 -) - -print(f"Read data from {data_path}") -atoms_list: list[Atoms] = ase_atoms_from_zip(data_path) -atoms_list = np.array(atoms_list, dtype="object") - -if slurm_array_job_id == "debug": - if smoke_test: - atoms_list = atoms_list[:128] - else: - pass -elif slurm_array_task_count > 1: - atoms_list = np.array_split(atoms_list, slurm_array_task_count)[ - slurm_array_task_id - 1 - ] - - -# %% -run_params = { - "data_path": data_path, - "versions": { - dep: version(dep) for dep in ("tensorpotential", "numpy", "tensorflow", "ase") - }, - "checkpoint": checkpoint, - Key.task_type: task_type, - "n_structures": len(atoms_list), - "slurm_vars": slurm_vars, - "max_steps": max_steps, - "record_traj": record_traj, - "force_max": force_max, - "ase_optimizer": ase_optimizer, - "device": device, - # Key.model_params: count_parameters(calc.models[0]), - "model_name": model_name, - "dtype": dtype, - "ase_filter": ase_filter, -} - -run_name = f"{job_name}-{slurm_array_task_id}" - -with open( - os.path.join( - out_dir, f"run_data_{slurm_array_task_id}-{slurm_array_task_count}.json" - ), - "w", -) as f: - json.dump(run_params, f) - -# wandb.init(project="matbench-discovery", name=run_name, config=run_params) - - -# %% time -relax_results: dict[str, dict[str, Any]] = {} -filter_cls: Callable[[Atoms], Atoms] = { - "frechet": FrechetCellFilter, - "exp": ExpCellFilter, -}[ase_filter] -optim_cls: Optimizer = {"FIRE": FIRE, "LBFGS": LBFGS}[ase_optimizer] -atoms_list = sorted(atoms_list, key=lambda at: len(at)) -# print(atoms_list) -for atoms in tqdm(deepcopy(atoms_list), desc="Relaxing", mininterval=5): - mat_id = atoms.info[Key.mat_id] - if mat_id in relax_results: - continue - try: - atoms.calc = calc - if max_steps > 0: - filtered_atoms = filter_cls(atoms) - optimizer = optim_cls(filtered_atoms, logfile="/dev/null") - - if record_traj: - coords, lattices, energies = [], [], [] - # attach observer functions to the optimizer - optimizer.attach(lambda: coords.append(atoms.get_positions())) # noqa: B023 - optimizer.attach(lambda: lattices.append(atoms.get_cell())) # noqa: B023 - optimizer.attach(lambda: energies.append(atoms.get_potential_energy())) # noqa: B023 - - optimizer.run(fmax=force_max, steps=max_steps) - energy = atoms.get_potential_energy() # relaxed energy - # if max_steps > 0, atoms is wrapped by filter_cls, so extract with getattr - relaxed_struct = AseAtomsAdaptor.get_structure(atoms) - relax_results[mat_id] = {"structure": relaxed_struct, "energy": energy} - - coords = locals().get("coords", []) - lattices = locals().get("lattices", []) - energies = locals().get("energies", []) - if record_traj and coords and lattices and energies: - traj = Trajectory( - species=atoms.get_chemical_symbols(), - coords=coords, - lattice=lattices, - constant_lattice=False, - frame_properties=[{"energy": energy} for energy in energies], - ) - relax_results[mat_id]["trajectory"] = traj - except Exception as exc: - print(f"Failed to relax {mat_id}: {exc!r}") - continue +__date__ = "2025-02-06" # %% -df_out = pd.DataFrame(relax_results).T.add_prefix("grace2l_r6_") -df_out.index.name = Key.mat_id -if not smoke_test: - df_out.reset_index().to_json(out_path, default_handler=as_dict_handler) +# model_name = "MP_GRACE_2L_r6_11Nov2024" # REMOVE this hardcoded line + +def main(model_name: str): # Added function and argument + """ + Main function to run the relaxation with the given model name. + """ + smoke_test = False + task_type = Task.IS2RE + module_dir = os.path.dirname(__file__) + # set large job array size for smaller data splits and faster testing/debugging + slurm_array_task_count = int( + os.getenv("SLURM_ARRAY_TASK_COUNT", "1") + ) # will be set to the number of tasks in the job array. + ase_optimizer = "FIRE" + job_name = f"{model_name}-wbm-{task_type}-{ase_optimizer}" # Use passed model_name + out_dir = os.getenv("SBATCH_OUTPUT", f"{module_dir}/{today}-{job_name}") + device = "gpu" + # whether to record intermediate structures into pymatgen Trajectory + record_traj = False # has no effect if relax_cell is False + + ase_filter: Literal["frechet", "exp"] = "frechet" + os.makedirs(out_dir, exist_ok=True) + + # slurm_vars = slurm_submit( + # job_name=job_name, + # out_dir=out_dir, + # array=f"1-{slurm_array_task_count}", + # # slurm_flags="--qos shared --constraint gpu --gpus 1", + # slurm_flags="--ntasks=1 --cpus-per-task=1 --partition high-priority", + # ) + + slurm_vars = {k: v for k, v in os.environ.items() if k.startswith("SLURM_")} + + + # %% + slurm_array_task_id = int( + os.getenv("SLURM_ARRAY_TASK_ID", "0") + ) # will be set to the job array index value. + slurm_array_job_id = os.getenv( + "SLURM_ARRAY_JOB_ID", "debug" + ) # will be set to the first job ID of the array. + + out_path = f"{out_dir}/{slurm_array_job_id}-{slurm_array_task_id:>03}.json.gz" + + if os.path.isfile(out_path): + raise SystemExit(f"{out_path=} already exists, exciting early") + + print(f"{slurm_array_task_id=}") + print(f"{slurm_array_job_id=}") + print(f"{slurm_array_task_count=}") + print(f"{slurm_vars=}") + print(f"{out_dir=}") + + + # %% + data_path = { + Task.RS2RE: DataFiles.wbm_relaxed_atoms.path, + Task.IS2RE: DataFiles.wbm_initial_atoms.path, + }[task_type] + print(f"\nJob {job_name} started {timestamp}") + e_pred_col = "grace_energy" + max_steps = 500 + force_max = 0.05 # Run until the forces are smaller than this in eV/A + checkpoint = "" + dtype = "float64" + calc = grace_fm( + model=model_name, pad_neighbors_fraction=0.05, pad_atoms_number=2, min_dist=0.5 + ) # Use passed model_name + + print(f"Read data from {data_path}") + atoms_list: list[Atoms] = ase_atoms_from_zip(data_path) + atoms_list = np.array(atoms_list, dtype="object") + + if slurm_array_job_id == "debug": + if smoke_test: + atoms_list = atoms_list[:128] + else: + pass + elif slurm_array_task_count > 1: + atoms_list = np.array_split(atoms_list, slurm_array_task_count)[ + slurm_array_task_id - 1 + ] + + + # %% + run_params = { + "data_path": data_path, + "versions": { + dep: version(dep) for dep in ("tensorpotential", "numpy", "tensorflow", "ase") + }, + "checkpoint": checkpoint, + Key.task_type: task_type, + "n_structures": len(atoms_list), + "slurm_vars": slurm_vars, + "max_steps": max_steps, + "record_traj": record_traj, + "force_max": force_max, + "ase_optimizer": ase_optimizer, + "device": device, + # Key.model_params: count_parameters(calc.models[0]), + "model_name": model_name, # Use passed model_name + "dtype": dtype, + "ase_filter": ase_filter, + } + + run_name = f"{job_name}-{slurm_array_task_id}" + + with open( + os.path.join( + out_dir, f"run_data_{slurm_array_task_id}-{slurm_array_task_count}.json" + ), + "w", + ) as f: + json.dump(run_params, f) + + # wandb.init(project="matbench-discovery", name=run_name, config=run_params) + + + # %% time + relax_results: dict[str, dict[str, Any]] = {} + filter_cls: Callable[[Atoms], Atoms] = { + "frechet": FrechetCellFilter, + "exp": ExpCellFilter, + }[ase_filter] + optim_cls: Optimizer = {"FIRE": FIRE, "LBFGS": LBFGS}[ase_optimizer] + atoms_list = sorted(atoms_list, key=lambda at: len(at)) + # print(atoms_list) + for atoms in tqdm(deepcopy(atoms_list), desc="Relaxing", mininterval=5): + mat_id = atoms.info[Key.mat_id] + if mat_id in relax_results: + continue + try: + atoms.calc = calc + if max_steps > 0: + filtered_atoms = filter_cls(atoms) + optimizer = optim_cls(filtered_atoms, logfile="/dev/null") + + if record_traj: + coords, lattices, energies = [], [], [] + # attach observer functions to the optimizer + optimizer.attach(lambda: coords.append(atoms.get_positions())) # noqa: B023 + optimizer.attach(lambda: lattices.append(atoms.get_cell())) # noqa: B023 + optimizer.attach(lambda: energies.append(atoms.get_potential_energy())) # noqa: B023 + + optimizer.run(fmax=force_max, steps=max_steps) + energy = atoms.get_potential_energy() # relaxed energy + # if max_steps > 0, atoms is wrapped by filter_cls, so extract with getattr + relaxed_struct = AseAtomsAdaptor.get_structure(atoms) + relax_results[mat_id] = {"structure": relaxed_struct, "energy": energy} + + coords = locals().get("coords", []) + lattices = locals().get("lattices", []) + energies = locals().get("energies", []) + if record_traj and coords and lattices and energies: + traj = Trajectory( + species=atoms.get_chemical_symbols(), + coords=coords, + lattice=lattices, + constant_lattice=False, + frame_properties=[{"energy": energy} for energy in energies], + ) + relax_results[mat_id]["trajectory"] = traj + except Exception as exc: + print(f"Failed to relax {mat_id}: {exc!r}") + continue + + + # %% + df_out = pd.DataFrame(relax_results).T.add_prefix("grace_") + df_out.index.name = Key.mat_id + if not smoke_test: + df_out.reset_index().to_json(out_path, default_handler=as_dict_handler) + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run relaxation with a specified model.") + parser.add_argument("model_name", type=str, help="The name of the model to use.") + args = parser.parse_args() + + main(args.model_name) From d86695c1745a92112abd642a3fa3d47720481568 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Thu, 6 Feb 2025 16:34:20 +0100 Subject: [PATCH 02/17] bugfix join_grace_preds.py --- models/grace/join_grace_preds.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/models/grace/join_grace_preds.py b/models/grace/join_grace_preds.py index f9f4ecba..416de653 100644 --- a/models/grace/join_grace_preds.py +++ b/models/grace/join_grace_preds.py @@ -1,6 +1,7 @@ import glob import os import argparse # Import the argparse module +import re import pandas as pd from pymatviz.enums import Key @@ -23,6 +24,25 @@ task_type = Task.IS2RE # or Task.RS2RE, depending on what you processed +def split_string(input_string): + """ + Splits the input string into date and model name. + + Args: + input_string: The string to split. + + Returns: + A tuple containing the date string and the model name string, + or (None, None) if the string doesn't match the expected pattern. + """ + match = re.match(r"^(\d{4}-\d{2}-\d{2})-(.*)-wbm.*", input_string) + if match: + date_str = match.group(1) + model_name = match.group(2) + return date_str, model_name + else: + return None, None + def process_results(path: str): """ Processes relaxation results from a given path. @@ -30,6 +50,10 @@ def process_results(path: str): Args: path (str): The path to the directory containing the .json.gz files. """ + date, model_name = split_string(path) + print("date:",date) + print("model_name:", model_name) + glob_pattern = os.path.join(path, "production-*.json.gz") file_paths = glob.glob(glob_pattern) From 4c60335d0e2024731b2fc7ccbab0850c7a56b1a1 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Thu, 6 Feb 2025 17:19:05 +0100 Subject: [PATCH 03/17] join_grace_preds.py save relaxed structures --- models/grace/join_grace_preds.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/models/grace/join_grace_preds.py b/models/grace/join_grace_preds.py index 416de653..f69be05a 100644 --- a/models/grace/join_grace_preds.py +++ b/models/grace/join_grace_preds.py @@ -57,6 +57,9 @@ def process_results(path: str): glob_pattern = os.path.join(path, "production-*.json.gz") file_paths = glob.glob(glob_pattern) + out_path = file_paths[0].rsplit("/", 1)[0] # Get directory from first file path. + + print(f"Found {len(file_paths):,} files for {glob_pattern = }") if not file_paths: @@ -79,10 +82,10 @@ def process_results(path: str): tot_df = ( tot_df.sort_values("id_tuple") .reset_index(drop=True) - .drop(columns=["id_tuple", struct_col]) + .drop(columns=["id_tuple"]) ) - df_grace = tot_df.set_index("material_id") + df_grace = tot_df.set_index("material_id")#.drop(columns=[struct_col]) df_grace[Key.formula] = df_wbm[Key.formula] @@ -99,6 +102,13 @@ def process_results(path: str): df_grace[e_form_grace_col] = e_form_list + # save relaxed structures + print("df_grace.columns=",df_grace.columns) + df_grace.to_json(f"{out_path}/{model_name}_{date}-wbm-IS2RE-FIRE.json.gz", default_handler=as_dict_handler) #added model and date + df_grace=df_grace.drop(columns=[struct_col]) + + + df_wbm[[*df_grace]] = df_grace @@ -106,7 +116,6 @@ def process_results(path: str): bad_mask = abs(df_wbm[e_form_grace_col] - df_wbm[MbdKey.e_form_dft]) > 5 n_preds = len(df_wbm[e_form_grace_col].dropna()) print(f"{sum(bad_mask)=} is {sum(bad_mask) / len(df_wbm):.2%} of {n_preds:,}") - out_path = file_paths[0].rsplit("/", 1)[0] # Get directory from first file path. df_grace = df_grace.round(4) df_grace.select_dtypes("number").to_csv(f"{out_path}/{model_name}_{date}.csv.gz") #added model and date From 4e92e97ecfe3d7d708dd8b5681869e9811d322f3 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Thu, 6 Feb 2025 17:49:05 +0100 Subject: [PATCH 04/17] add GRACE-OAM's yaml files --- models/grace/grace-1L-OAM.yml | 143 +++++++++++++++++++++++++++ models/grace/grace-2L-OAM.yml | 143 +++++++++++++++++++++++++++ models/grace/run_eval_parallel.sh | 23 ++++- models/grace/test_grace_discovery.py | 1 - 4 files changed, 306 insertions(+), 4 deletions(-) create mode 100644 models/grace/grace-1L-OAM.yml create mode 100644 models/grace/grace-2L-OAM.yml diff --git a/models/grace/grace-1L-OAM.yml b/models/grace/grace-1L-OAM.yml new file mode 100644 index 00000000..d6eb9d44 --- /dev/null +++ b/models/grace/grace-1L-OAM.yml @@ -0,0 +1,143 @@ +model_name: GRACE-1L-OAM +model_key: grace +model_version: GRACE-1L-OAM_2Feb25 +matbench_discovery_version: 1.2.0 +date_added: "2025-02-06" +date_published: "2025-02-06" +authors: + - name: Anton Bochkarev + affiliation: ICAMS, Ruhr University Bochum + email: anton.bochkarev@rub.de + - name: Yury Lysogorskiy + affiliation: ICAMS, Ruhr University Bochum + email: yury.lysogorskiy@rub.de + - name: Ralf Drautz + affiliation: ICAMS, Ruhr University Bochum + email: ralf.drautz@rub.de +trained_by: + - name: Yury Lysogorskiy + affiliation: ICAMS, Ruhr University Bochum + email: yury.lysogorskiy@rub.de + - name: Anton Bochkarev + affiliation: ICAMS, Ruhr University Bochum + email: anton.bochkarev@rub.de + +repo: https://github.com/ICAMS/grace-tensorpotential +doi: https://doi.org/10.1103/PhysRevX.14.021036 +paper: https://journals.aps.org/prx/abstract/10.1103/PhysRevX.14.021036 +url: https://gracemaker.readthedocs.io/en/latest/gracemaker/foundation +#pr_url: https://github.com/janosh/matbench-discovery/pull/160 # TODO + +requirements: + tensorpotential: 0.4.5 + tensorflow: 2.16.2 + ase: 3.23.0 + pymatgen: 2023.7.14 + numpy: 1.26.4 + +openness: OSOD +trained_for_benchmark: true +train_task: S2EF +test_task: IS2RE-SR +targets: EFS_G +model_type: UIP +model_params: 12_597_516 +n_estimators: 1 + +training_set: [OMat24, sAlex, MPtrj] + +hyperparams: + max_force: 0.03 + max_steps: 500 + ase_optimizer: FIRE + radial_cutoff: 6.0 + +metrics: + phonons: + kappa_103: + κ_SRME: 0.516 # https://github.com/MPA2suite/k_SRME/pull/20 + pred_file: TODO + pred_file_url: TODO + geo_opt: + pred_file: TODO + pred_file_url: TODO + pred_col: grace_structure + # symprec=1e-5: + # rmsd: 0.0186 # Å + # n_sym_ops_mae: 1.8703 # unitless + # symmetry_decrease: 0.0355 # fraction + # symmetry_match: 0.7315 # fraction + # symmetry_increase: 0.2285 # fraction + # n_structures: 256862 # count + # analysis_file: models/grace/grace2l_r6/2024-11-11-relaxed-structures-symprec=1e-5.csv.gz + # analysis_file_url: https://figshare.com/ndownloader/files/52062641 + # symprec=1e-2: + # rmsd: 0.0186 # Å + # n_sym_ops_mae: 1.8982 # unitless + # symmetry_decrease: 0.0592 # fraction + # symmetry_match: 0.7976 # fraction + # symmetry_increase: 0.1363 # fraction + # n_structures: 256513 # count + # analysis_file: models/grace/grace2l_r6/2024-11-11-relaxed-structures-symprec=1e-2.csv.gz + # analysis_file_url: https://figshare.com/ndownloader/files/52062650 + # discovery: + # pred_file: TODO + # pred_file_url: TODO + # pred_col: e_form_per_atom_grace + # full_test_set: + # F1: 0.687 # fraction + # DAF: 3.714 # dimensionless + # Precision: 0.637 # fraction + # Recall: 0.746 # fraction + # Accuracy: 0.884 # fraction + # TPR: 0.746 # fraction + # FPR: 0.088 # fraction + # TNR: 0.912 # fraction + # FNR: 0.254 # fraction + # TP: 32880.0 # count + # FP: 18721.0 # count + # TN: 194150.0 # count + # FN: 11212.0 # count + # MAE: 0.05 # eV/atom + # RMSE: 0.092 # eV/atom + # R2: 0.74 # dimensionless + # missing_preds: 128 # count + # missing_percent: 0.05% # fraction + # most_stable_10k: + # F1: 0.914 # fraction + # DAF: 5.503 # dimensionless + # Precision: 0.841 # fraction + # Recall: 1.0 # fraction + # Accuracy: 0.841 # fraction + # TPR: 1.0 # fraction + # FPR: 1.0 # fraction + # TNR: 0.0 # fraction + # FNR: 0.0 # fraction + # TP: 8413.0 # count + # FP: 1587.0 # count + # TN: 0.0 # count + # FN: 0.0 # count + # MAE: 0.065 # eV/atom + # RMSE: 0.133 # eV/atom + # R2: 0.641 # dimensionless + # missing_preds: 0 # count + # missing_percent: 0.00% # fraction + # unique_prototypes: + # F1: 0.691 # fraction + # DAF: 4.163 # dimensionless + # Precision: 0.636 # fraction + # Recall: 0.757 # fraction + # Accuracy: 0.896 # fraction + # TPR: 0.757 # fraction + # FPR: 0.079 # fraction + # TNR: 0.921 # fraction + # FNR: 0.243 # fraction + # TP: 25255.0 # count + # FP: 14425.0 # count + # TN: 167689.0 # count + # FN: 8119.0 # count + # MAE: 0.052 # eV/atom + # RMSE: 0.094 # eV/atom + # R2: 0.741 # dimensionless + # missing_preds: 110 # count + # missing_percent: 0.05% # fraction diff --git a/models/grace/grace-2L-OAM.yml b/models/grace/grace-2L-OAM.yml new file mode 100644 index 00000000..f15247f5 --- /dev/null +++ b/models/grace/grace-2L-OAM.yml @@ -0,0 +1,143 @@ +model_name: GRACE-2L-OAM +model_key: grace +model_version: GRACE_2L_OAM_28Jan25 +matbench_discovery_version: 1.2.0 +date_added: "2025-02-06" +date_published: "2025-02-06" +authors: + - name: Anton Bochkarev + affiliation: ICAMS, Ruhr University Bochum + email: anton.bochkarev@rub.de + - name: Yury Lysogorskiy + affiliation: ICAMS, Ruhr University Bochum + email: yury.lysogorskiy@rub.de + - name: Ralf Drautz + affiliation: ICAMS, Ruhr University Bochum + email: ralf.drautz@rub.de +trained_by: + - name: Yury Lysogorskiy + affiliation: ICAMS, Ruhr University Bochum + email: yury.lysogorskiy@rub.de + - name: Anton Bochkarev + affiliation: ICAMS, Ruhr University Bochum + email: anton.bochkarev@rub.de + +repo: https://github.com/ICAMS/grace-tensorpotential +doi: https://doi.org/10.1103/PhysRevX.14.021036 +paper: https://journals.aps.org/prx/abstract/10.1103/PhysRevX.14.021036 +url: https://gracemaker.readthedocs.io/en/latest/gracemaker/foundation +#pr_url: https://github.com/janosh/matbench-discovery/pull/160 # TODO + +requirements: + tensorpotential: 0.4.5 + tensorflow: 2.16.2 + ase: 3.23.0 + pymatgen: 2023.7.14 + numpy: 1.26.4 + +openness: OSOD +trained_for_benchmark: true +train_task: S2EF +test_task: IS2RE-SR +targets: EFS_G +model_type: UIP +model_params: 12_597_516 +n_estimators: 1 + +training_set: [OMat24, sAlex,MPtrj] + +hyperparams: + max_force: 0.03 + max_steps: 500 + ase_optimizer: FIRE + radial_cutoff: 6.0 + +metrics: + phonons: + kappa_103: + κ_SRME: 0.294 # https://github.com/MPA2suite/k_SRME/pull/20 + pred_file: TODO + pred_file_url: TODO + geo_opt: + pred_file: TODO + pred_file_url: TODO + pred_col: grace_structure + # symprec=1e-5: + # rmsd: 0.0186 # Å + # n_sym_ops_mae: 1.8703 # unitless + # symmetry_decrease: 0.0355 # fraction + # symmetry_match: 0.7315 # fraction + # symmetry_increase: 0.2285 # fraction + # n_structures: 256862 # count + # analysis_file: models/grace/grace2l_r6/2024-11-11-relaxed-structures-symprec=1e-5.csv.gz + # analysis_file_url: https://figshare.com/ndownloader/files/52062641 + # symprec=1e-2: + # rmsd: 0.0186 # Å + # n_sym_ops_mae: 1.8982 # unitless + # symmetry_decrease: 0.0592 # fraction + # symmetry_match: 0.7976 # fraction + # symmetry_increase: 0.1363 # fraction + # n_structures: 256513 # count + # analysis_file: models/grace/grace2l_r6/2024-11-11-relaxed-structures-symprec=1e-2.csv.gz + # analysis_file_url: https://figshare.com/ndownloader/files/52062650 + discovery: + pred_file: TODO + pred_file_url: TODO + pred_col: e_form_per_atom_grace + full_test_set: + F1: 0.687 # fraction + DAF: 3.714 # dimensionless + Precision: 0.637 # fraction + Recall: 0.746 # fraction + Accuracy: 0.884 # fraction + TPR: 0.746 # fraction + FPR: 0.088 # fraction + TNR: 0.912 # fraction + FNR: 0.254 # fraction + TP: 32880.0 # count + FP: 18721.0 # count + TN: 194150.0 # count + FN: 11212.0 # count + MAE: 0.05 # eV/atom + RMSE: 0.092 # eV/atom + R2: 0.74 # dimensionless + missing_preds: 128 # count + missing_percent: 0.05% # fraction + most_stable_10k: + F1: 0.914 # fraction + DAF: 5.503 # dimensionless + Precision: 0.841 # fraction + Recall: 1.0 # fraction + Accuracy: 0.841 # fraction + TPR: 1.0 # fraction + FPR: 1.0 # fraction + TNR: 0.0 # fraction + FNR: 0.0 # fraction + TP: 8413.0 # count + FP: 1587.0 # count + TN: 0.0 # count + FN: 0.0 # count + MAE: 0.065 # eV/atom + RMSE: 0.133 # eV/atom + R2: 0.641 # dimensionless + missing_preds: 0 # count + missing_percent: 0.00% # fraction + unique_prototypes: + F1: 0.691 # fraction + DAF: 4.163 # dimensionless + Precision: 0.636 # fraction + Recall: 0.757 # fraction + Accuracy: 0.896 # fraction + TPR: 0.757 # fraction + FPR: 0.079 # fraction + TNR: 0.921 # fraction + FNR: 0.243 # fraction + TP: 25255.0 # count + FP: 14425.0 # count + TN: 167689.0 # count + FN: 8119.0 # count + MAE: 0.052 # eV/atom + RMSE: 0.094 # eV/atom + R2: 0.741 # dimensionless + missing_preds: 110 # count + missing_percent: 0.05% # fraction diff --git a/models/grace/run_eval_parallel.sh b/models/grace/run_eval_parallel.sh index fd62156a..55c8628e 100755 --- a/models/grace/run_eval_parallel.sh +++ b/models/grace/run_eval_parallel.sh @@ -1,5 +1,22 @@ #!/bin/bash +########## HOW-TO ############ +# +# STAGE 1 +# 1. set GPUS to list of avilable GPUs +# 2. set model_name=.... +# 3. adjust SLURM_ARRAY_TASK_COUNT= if needed +# 4. run locally or submit to the queue +# +# STAGE 2 +# python join_grace_preds.py 2025-02-06-GRACE-1L-OAM_2Feb25-wbm-IS2RE-FIRE # or other path +# +# STAGE 3 +# upload large files to fileshare, commit models' YAML files +################################ + + + ## this script is for imitation of SLURM tasks array on local machine with multiple GPUs export TF_CPP_MIN_LOG_LEVEL=3 @@ -9,9 +26,9 @@ THREADS=4 # number of threads for OMP, MKL, NUMEXPR etc. To share resources on GPUS=(0 1 2 3) # Set the specific GPU IDs you want to use here NGPU=${#GPUS[@]} # Calculate the number of GPUs based on the array length -#model_name="MP_GRACE_2L_r6_11Nov2024" # just for information, model name is hardcoded in 1_test_srme.py -model_name="GRACE-1L-OAM_2Feb25" -#model_name="GRACE_2L_OAM_28Jan25" +#model_name="MP_GRACE_2L_r6_11Nov2024" +#model_name="GRACE-1L-OAM_2Feb25" +model_name="GRACE_2L_OAM_28Jan25" export MODEL_NAME="${model_name}" echo "MODEL_NAME=${MODEL_NAME}" diff --git a/models/grace/test_grace_discovery.py b/models/grace/test_grace_discovery.py index e8cf9b2f..4b99c463 100644 --- a/models/grace/test_grace_discovery.py +++ b/models/grace/test_grace_discovery.py @@ -33,7 +33,6 @@ # %% -# model_name = "MP_GRACE_2L_r6_11Nov2024" # REMOVE this hardcoded line def main(model_name: str): # Added function and argument """ From e9bf247e145b7076529e6d6b15796b304d480f1e Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Thu, 6 Feb 2025 17:54:28 +0100 Subject: [PATCH 05/17] upd grace-1L/2L-OAM.yml --- models/grace/grace-1L-OAM.yml | 4 +- models/grace/grace-2L-OAM.yml | 120 +++++++++++++++++----------------- 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/models/grace/grace-1L-OAM.yml b/models/grace/grace-1L-OAM.yml index d6eb9d44..6c7a82e6 100644 --- a/models/grace/grace-1L-OAM.yml +++ b/models/grace/grace-1L-OAM.yml @@ -41,7 +41,7 @@ train_task: S2EF test_task: IS2RE-SR targets: EFS_G model_type: UIP -model_params: 12_597_516 +model_params: 3_447_148 n_estimators: 1 training_set: [OMat24, sAlex, MPtrj] @@ -80,7 +80,7 @@ metrics: # n_structures: 256513 # count # analysis_file: models/grace/grace2l_r6/2024-11-11-relaxed-structures-symprec=1e-2.csv.gz # analysis_file_url: https://figshare.com/ndownloader/files/52062650 - # discovery: + discovery: # pred_file: TODO # pred_file_url: TODO # pred_col: e_form_per_atom_grace diff --git a/models/grace/grace-2L-OAM.yml b/models/grace/grace-2L-OAM.yml index f15247f5..5411c101 100644 --- a/models/grace/grace-2L-OAM.yml +++ b/models/grace/grace-2L-OAM.yml @@ -81,63 +81,63 @@ metrics: # analysis_file: models/grace/grace2l_r6/2024-11-11-relaxed-structures-symprec=1e-2.csv.gz # analysis_file_url: https://figshare.com/ndownloader/files/52062650 discovery: - pred_file: TODO - pred_file_url: TODO - pred_col: e_form_per_atom_grace - full_test_set: - F1: 0.687 # fraction - DAF: 3.714 # dimensionless - Precision: 0.637 # fraction - Recall: 0.746 # fraction - Accuracy: 0.884 # fraction - TPR: 0.746 # fraction - FPR: 0.088 # fraction - TNR: 0.912 # fraction - FNR: 0.254 # fraction - TP: 32880.0 # count - FP: 18721.0 # count - TN: 194150.0 # count - FN: 11212.0 # count - MAE: 0.05 # eV/atom - RMSE: 0.092 # eV/atom - R2: 0.74 # dimensionless - missing_preds: 128 # count - missing_percent: 0.05% # fraction - most_stable_10k: - F1: 0.914 # fraction - DAF: 5.503 # dimensionless - Precision: 0.841 # fraction - Recall: 1.0 # fraction - Accuracy: 0.841 # fraction - TPR: 1.0 # fraction - FPR: 1.0 # fraction - TNR: 0.0 # fraction - FNR: 0.0 # fraction - TP: 8413.0 # count - FP: 1587.0 # count - TN: 0.0 # count - FN: 0.0 # count - MAE: 0.065 # eV/atom - RMSE: 0.133 # eV/atom - R2: 0.641 # dimensionless - missing_preds: 0 # count - missing_percent: 0.00% # fraction - unique_prototypes: - F1: 0.691 # fraction - DAF: 4.163 # dimensionless - Precision: 0.636 # fraction - Recall: 0.757 # fraction - Accuracy: 0.896 # fraction - TPR: 0.757 # fraction - FPR: 0.079 # fraction - TNR: 0.921 # fraction - FNR: 0.243 # fraction - TP: 25255.0 # count - FP: 14425.0 # count - TN: 167689.0 # count - FN: 8119.0 # count - MAE: 0.052 # eV/atom - RMSE: 0.094 # eV/atom - R2: 0.741 # dimensionless - missing_preds: 110 # count - missing_percent: 0.05% # fraction + # pred_file: TODO + # pred_file_url: TODO + # pred_col: e_form_per_atom_grace + # full_test_set: + # F1: 0.687 # fraction + # DAF: 3.714 # dimensionless + # Precision: 0.637 # fraction + # Recall: 0.746 # fraction + # Accuracy: 0.884 # fraction + # TPR: 0.746 # fraction + # FPR: 0.088 # fraction + # TNR: 0.912 # fraction + # FNR: 0.254 # fraction + # TP: 32880.0 # count + # FP: 18721.0 # count + # TN: 194150.0 # count + # FN: 11212.0 # count + # MAE: 0.05 # eV/atom + # RMSE: 0.092 # eV/atom + # R2: 0.74 # dimensionless + # missing_preds: 128 # count + # missing_percent: 0.05% # fraction + # most_stable_10k: + # F1: 0.914 # fraction + # DAF: 5.503 # dimensionless + # Precision: 0.841 # fraction + # Recall: 1.0 # fraction + # Accuracy: 0.841 # fraction + # TPR: 1.0 # fraction + # FPR: 1.0 # fraction + # TNR: 0.0 # fraction + # FNR: 0.0 # fraction + # TP: 8413.0 # count + # FP: 1587.0 # count + # TN: 0.0 # count + # FN: 0.0 # count + # MAE: 0.065 # eV/atom + # RMSE: 0.133 # eV/atom + # R2: 0.641 # dimensionless + # missing_preds: 0 # count + # missing_percent: 0.00% # fraction + # unique_prototypes: + # F1: 0.691 # fraction + # DAF: 4.163 # dimensionless + # Precision: 0.636 # fraction + # Recall: 0.757 # fraction + # Accuracy: 0.896 # fraction + # TPR: 0.757 # fraction + # FPR: 0.079 # fraction + # TNR: 0.921 # fraction + # FNR: 0.243 # fraction + # TP: 25255.0 # count + # FP: 14425.0 # count + # TN: 167689.0 # count + # FN: 8119.0 # count + # MAE: 0.052 # eV/atom + # RMSE: 0.094 # eV/atom + # R2: 0.741 # dimensionless + # missing_preds: 110 # count + # missing_percent: 0.05% # fraction From 69f35bc4f5f983e9094517e7ef80f91b72061063 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 6 Feb 2025 16:55:58 +0000 Subject: [PATCH 06/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- models/grace/grace-2L-OAM.yml | 2 +- models/grace/join_grace_preds.py | 48 ++++++++++++++-------------- models/grace/run_eval_parallel.sh | 4 +-- models/grace/test_grace_discovery.py | 18 +++++------ 4 files changed, 35 insertions(+), 37 deletions(-) diff --git a/models/grace/grace-2L-OAM.yml b/models/grace/grace-2L-OAM.yml index 5411c101..7a933f14 100644 --- a/models/grace/grace-2L-OAM.yml +++ b/models/grace/grace-2L-OAM.yml @@ -44,7 +44,7 @@ model_type: UIP model_params: 12_597_516 n_estimators: 1 -training_set: [OMat24, sAlex,MPtrj] +training_set: [OMat24, sAlex, MPtrj] hyperparams: max_force: 0.03 diff --git a/models/grace/join_grace_preds.py b/models/grace/join_grace_preds.py index f69be05a..2cbcb961 100644 --- a/models/grace/join_grace_preds.py +++ b/models/grace/join_grace_preds.py @@ -1,6 +1,6 @@ +import argparse # Import the argparse module import glob import os -import argparse # Import the argparse module import re import pandas as pd @@ -40,8 +40,8 @@ def split_string(input_string): date_str = match.group(1) model_name = match.group(2) return date_str, model_name - else: - return None, None + return None, None + def process_results(path: str): """ @@ -51,7 +51,7 @@ def process_results(path: str): path (str): The path to the directory containing the .json.gz files. """ date, model_name = split_string(path) - print("date:",date) + print("date:", date) print("model_name:", model_name) glob_pattern = os.path.join(path, "production-*.json.gz") @@ -59,7 +59,6 @@ def process_results(path: str): out_path = file_paths[0].rsplit("/", 1)[0] # Get directory from first file path. - print(f"Found {len(file_paths):,} files for {glob_pattern = }") if not file_paths: @@ -72,23 +71,20 @@ def process_results(path: str): try: dfs.append(pd.read_json(fn)) except Exception as e: - print(f"Error reading {fn}: {e}") # Print any errors during file reading. - continue # Continue to the next file + print(f"Error reading {fn}: {e}") # Print any errors during file reading. + continue # Continue to the next file tot_df = pd.concat(dfs) tot_df["id_tuple"] = ( tot_df["material_id"].str.split("-").map(lambda x: (int(x[1]), int(x[2]))) ) tot_df = ( - tot_df.sort_values("id_tuple") - .reset_index(drop=True) - .drop(columns=["id_tuple"]) + tot_df.sort_values("id_tuple").reset_index(drop=True).drop(columns=["id_tuple"]) ) - df_grace = tot_df.set_index("material_id")#.drop(columns=[struct_col]) + df_grace = tot_df.set_index("material_id") # .drop(columns=[struct_col]) df_grace[Key.formula] = df_wbm[Key.formula] - print("Calculating formation energies") e_form_list = [] for _, row in tqdm(df_grace.iterrows(), total=len(df_grace)): @@ -99,36 +95,40 @@ def process_results(path: str): ) e_form_list.append(e_form) - df_grace[e_form_grace_col] = e_form_list # save relaxed structures - print("df_grace.columns=",df_grace.columns) - df_grace.to_json(f"{out_path}/{model_name}_{date}-wbm-IS2RE-FIRE.json.gz", default_handler=as_dict_handler) #added model and date - df_grace=df_grace.drop(columns=[struct_col]) - - + print("df_grace.columns=", df_grace.columns) + df_grace.to_json( + f"{out_path}/{model_name}_{date}-wbm-IS2RE-FIRE.json.gz", + default_handler=as_dict_handler, + ) # added model and date + df_grace = df_grace.drop(columns=[struct_col]) df_wbm[[*df_grace]] = df_grace - # %% bad_mask = abs(df_wbm[e_form_grace_col] - df_wbm[MbdKey.e_form_dft]) > 5 n_preds = len(df_wbm[e_form_grace_col].dropna()) print(f"{sum(bad_mask)=} is {sum(bad_mask) / len(df_wbm):.2%} of {n_preds:,}") df_grace = df_grace.round(4) - df_grace.select_dtypes("number").to_csv(f"{out_path}/{model_name}_{date}.csv.gz") #added model and date - df_grace.reset_index().to_json(f"{out_path}/{model_name}_{date}.json.gz", default_handler=as_dict_handler) #added model and date + df_grace.select_dtypes("number").to_csv( + f"{out_path}/{model_name}_{date}.csv.gz" + ) # added model and date + df_grace.reset_index().to_json( + f"{out_path}/{model_name}_{date}.json.gz", default_handler=as_dict_handler + ) # added model and date df_bad = df_grace[bad_mask].copy() df_bad[MbdKey.e_form_dft] = df_wbm[MbdKey.e_form_dft] - df_bad.to_csv(f"{out_path}/{model_name}_{date}_bad.csv") #added model and date - + df_bad.to_csv(f"{out_path}/{model_name}_{date}_bad.csv") # added model and date if __name__ == "__main__": parser = argparse.ArgumentParser(description="Process relaxation results.") - parser.add_argument("path", type=str, help="Path to the directory with relaxation results.") + parser.add_argument( + "path", type=str, help="Path to the directory with relaxation results." + ) args = parser.parse_args() process_results(args.path) diff --git a/models/grace/run_eval_parallel.sh b/models/grace/run_eval_parallel.sh index 55c8628e..6876cc8c 100755 --- a/models/grace/run_eval_parallel.sh +++ b/models/grace/run_eval_parallel.sh @@ -26,7 +26,7 @@ THREADS=4 # number of threads for OMP, MKL, NUMEXPR etc. To share resources on GPUS=(0 1 2 3) # Set the specific GPU IDs you want to use here NGPU=${#GPUS[@]} # Calculate the number of GPUs based on the array length -#model_name="MP_GRACE_2L_r6_11Nov2024" +#model_name="MP_GRACE_2L_r6_11Nov2024" #model_name="GRACE-1L-OAM_2Feb25" model_name="GRACE_2L_OAM_28Jan25" @@ -65,7 +65,7 @@ do gpu_index=$((task_id % NGPU)) # Get the actual GPU ID from the GPUS array gpu_id=${GPUS[$gpu_index]} - + CUDA_VISIBLE_DEVICES=$gpu_id SLURM_ARRAY_TASK_ID=${task_id} python test_grace_discovery.py "${MODEL_NAME}" > "out-${task_id}.txt" 2>&1 & done wait diff --git a/models/grace/test_grace_discovery.py b/models/grace/test_grace_discovery.py index 4b99c463..8585734d 100644 --- a/models/grace/test_grace_discovery.py +++ b/models/grace/test_grace_discovery.py @@ -33,7 +33,6 @@ # %% - def main(model_name: str): # Added function and argument """ Main function to run the relaxation with the given model name. @@ -65,7 +64,6 @@ def main(model_name: str): # Added function and argument slurm_vars = {k: v for k, v in os.environ.items() if k.startswith("SLURM_")} - # %% slurm_array_task_id = int( os.getenv("SLURM_ARRAY_TASK_ID", "0") @@ -85,7 +83,6 @@ def main(model_name: str): # Added function and argument print(f"{slurm_vars=}") print(f"{out_dir=}") - # %% data_path = { Task.RS2RE: DataFiles.wbm_relaxed_atoms.path, @@ -115,12 +112,12 @@ def main(model_name: str): # Added function and argument slurm_array_task_id - 1 ] - # %% run_params = { "data_path": data_path, "versions": { - dep: version(dep) for dep in ("tensorpotential", "numpy", "tensorflow", "ase") + dep: version(dep) + for dep in ("tensorpotential", "numpy", "tensorflow", "ase") }, "checkpoint": checkpoint, Key.task_type: task_type, @@ -149,7 +146,6 @@ def main(model_name: str): # Added function and argument # wandb.init(project="matbench-discovery", name=run_name, config=run_params) - # %% time relax_results: dict[str, dict[str, Any]] = {} filter_cls: Callable[[Atoms], Atoms] = { @@ -174,7 +170,9 @@ def main(model_name: str): # Added function and argument # attach observer functions to the optimizer optimizer.attach(lambda: coords.append(atoms.get_positions())) # noqa: B023 optimizer.attach(lambda: lattices.append(atoms.get_cell())) # noqa: B023 - optimizer.attach(lambda: energies.append(atoms.get_potential_energy())) # noqa: B023 + optimizer.attach( + lambda: energies.append(atoms.get_potential_energy()) + ) # noqa: B023 optimizer.run(fmax=force_max, steps=max_steps) energy = atoms.get_potential_energy() # relaxed energy @@ -198,7 +196,6 @@ def main(model_name: str): # Added function and argument print(f"Failed to relax {mat_id}: {exc!r}") continue - # %% df_out = pd.DataFrame(relax_results).T.add_prefix("grace_") df_out.index.name = Key.mat_id @@ -206,9 +203,10 @@ def main(model_name: str): # Added function and argument df_out.reset_index().to_json(out_path, default_handler=as_dict_handler) - if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Run relaxation with a specified model.") + parser = argparse.ArgumentParser( + description="Run relaxation with a specified model." + ) parser.add_argument("model_name", type=str, help="The name of the model to use.") args = parser.parse_args() From d19ecd12713a338241dcbb2a9098d85088b3084f Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Fri, 7 Feb 2025 06:01:37 -0500 Subject: [PATCH 07/17] tweak test_grace_discovery.py + join_grace_preds.py - fix outdated phonondb_pbe_103_structures URL in data-files.yml - simplify file path handling in hpc.py and test_hpc.py --- matbench_discovery/data-files.yml | 2 +- matbench_discovery/hpc.py | 2 +- models/grace/join_grace_preds.py | 39 +--- models/grace/run_eval_parallel.sh | 2 +- models/grace/test_grace_discovery.py | 338 ++++++++++++--------------- models/orb/join_orb_preds.py | 4 +- tests/test_hpc.py | 4 +- 7 files changed, 174 insertions(+), 217 deletions(-) diff --git a/matbench_discovery/data-files.yml b/matbench_discovery/data-files.yml index dba2167e..326c2cad 100644 --- a/matbench_discovery/data-files.yml +++ b/matbench_discovery/data-files.yml @@ -95,7 +95,7 @@ wbm_dft_geo_opt_symprec_1e_5: md5: phonondb_pbe_103_structures: - url: https://figshare.com/ndownloader/files/51680888 + url: https://figshare.com/ndownloader/files/52179965 path: phonons/2024-11-09-phononDB-PBE-103-structures.extxyz description: 103 phononDB structures run by Togo with PBE settings received in private communication. See https://github.com/atztogo/phonondb/blob/bba206/README.md#url-links-to-phono3py-finite-displacement-method-inputs-of-103-compounds-on-mdr-at-nims-pbe for details. md5: a396d4c517fa6d57defeffc6c83f0118 diff --git a/matbench_discovery/hpc.py b/matbench_discovery/hpc.py index 3e12895b..e727cb63 100644 --- a/matbench_discovery/hpc.py +++ b/matbench_discovery/hpc.py @@ -88,7 +88,7 @@ def slurm_submit( # Copy the file to a temporary directory if submit_as_temp_file is True if submit_as_temp_file and SLURM_SUBMIT_KEY in sys.argv: temp_dir = tempfile.mkdtemp(prefix="slurm_job_") - temp_file_path = os.path.join(temp_dir, os.path.basename(py_file_path)) + temp_file_path = f"{temp_dir}/{os.path.basename(py_file_path)}" shutil.copy2(py_file_path, temp_file_path) py_file_path = temp_file_path diff --git a/models/grace/join_grace_preds.py b/models/grace/join_grace_preds.py index 2cbcb961..896d940a 100644 --- a/models/grace/join_grace_preds.py +++ b/models/grace/join_grace_preds.py @@ -1,7 +1,7 @@ import argparse # Import the argparse module -import glob import os import re +from glob import glob import pandas as pd from pymatviz.enums import Key @@ -24,38 +24,23 @@ task_type = Task.IS2RE # or Task.RS2RE, depending on what you processed -def split_string(input_string): - """ - Splits the input string into date and model name. - - Args: - input_string: The string to split. - - Returns: - A tuple containing the date string and the model name string, - or (None, None) if the string doesn't match the expected pattern. - """ - match = re.match(r"^(\d{4}-\d{2}-\d{2})-(.*)-wbm.*", input_string) - if match: - date_str = match.group(1) - model_name = match.group(2) - return date_str, model_name - return None, None - - -def process_results(path: str): +def process_results(path: str) -> None: """ Processes relaxation results from a given path. Args: - path (str): The path to the directory containing the .json.gz files. + path (str): The path to the directory containing the .json.gz files from each + job in the job array. """ - date, model_name = split_string(path) - print("date:", date) - print("model_name:", model_name) - glob_pattern = os.path.join(path, "production-*.json.gz") - file_paths = glob.glob(glob_pattern) + if match := re.match(r"^(\d{4}-\d{2}-\d{2})-(.*)-wbm.*", path): + date, model_name = match[0], match[1] + print(f"{date=} {model_name=}") + else: + raise ValueError(f"{path=} failed to match regex") + + glob_pattern = f"{path}/production-*.json.gz" + file_paths = glob(glob_pattern) out_path = file_paths[0].rsplit("/", 1)[0] # Get directory from first file path. diff --git a/models/grace/run_eval_parallel.sh b/models/grace/run_eval_parallel.sh index 6876cc8c..f09ed338 100755 --- a/models/grace/run_eval_parallel.sh +++ b/models/grace/run_eval_parallel.sh @@ -3,7 +3,7 @@ ########## HOW-TO ############ # # STAGE 1 -# 1. set GPUS to list of avilable GPUs +# 1. set GPUS to list of available GPUs # 2. set model_name=.... # 3. adjust SLURM_ARRAY_TASK_COUNT= if needed # 4. run locally or submit to the queue diff --git a/models/grace/test_grace_discovery.py b/models/grace/test_grace_discovery.py index 8585734d..06187c34 100644 --- a/models/grace/test_grace_discovery.py +++ b/models/grace/test_grace_discovery.py @@ -1,12 +1,7 @@ # %% -import warnings - -# Ignore all RuntimeWarnings -warnings.filterwarnings("ignore", category=RuntimeWarning) - -import argparse import json import os +import warnings from collections.abc import Callable from copy import deepcopy from importlib.metadata import version @@ -32,182 +27,159 @@ __date__ = "2025-02-06" +warnings.filterwarnings("ignore", category=RuntimeWarning) + + +# %% +model_name = "GRACE-2L-OAM" +smoke_test = False +task_type = Task.IS2RE +module_dir = os.path.dirname(__file__) +# set large job array size for smaller data splits and faster testing/debugging +slurm_array_task_count = int( + os.getenv("SLURM_ARRAY_TASK_COUNT", "1") +) # will be set to the number of tasks in the job array. +ase_optimizer = "FIRE" +job_name = f"{model_name}-wbm-{task_type}-{ase_optimizer}" +out_dir = os.getenv("SBATCH_OUTPUT", f"{module_dir}/{today}-{job_name}") +device = "gpu" +# whether to record intermediate structures into pymatgen Trajectory +record_traj = False # has no effect if relax_cell is False + +ase_filter: Literal["frechet", "exp"] = "frechet" +os.makedirs(out_dir, exist_ok=True) + + +# %% +# will be set to the job array index value. +slurm_array_task_id = int(os.getenv("SLURM_ARRAY_TASK_ID", "0")) +# will be set to the first job ID of the array. +slurm_array_job_id = os.getenv("SLURM_ARRAY_JOB_ID", "debug") + +out_path = f"{out_dir}/{slurm_array_job_id}-{slurm_array_task_id:>03}.json.gz" + +if os.path.isfile(out_path): + raise SystemExit(f"{out_path=} already exists, exciting early") + +print(f"{slurm_array_task_id=}") +print(f"{slurm_array_job_id=}") +print(f"{slurm_array_task_count=}") +print(f"{out_dir=}") + + +# %% +data_path = { + Task.RS2RE: DataFiles.wbm_relaxed_atoms.path, + Task.IS2RE: DataFiles.wbm_initial_atoms.path, +}[task_type] +print(f"\nJob {job_name} started {timestamp}") +e_pred_col = "grace_energy" +max_steps = 500 +force_max = 0.05 # Run until the forces are smaller than this in eV/A +checkpoint = "" +dtype = "float64" +calc = grace_fm( + model=model_name, pad_neighbors_fraction=0.05, pad_atoms_number=2, min_dist=0.5 +) # Use passed model_name + +print(f"Read data from {data_path}") +atoms_list: list[Atoms] = ase_atoms_from_zip(data_path) +atoms_list = np.array(atoms_list, dtype="object") + +if slurm_array_job_id == "debug": + if smoke_test: + atoms_list = atoms_list[:128] + else: + pass +elif slurm_array_task_count > 1: + atoms_list = np.array_split(atoms_list, slurm_array_task_count)[ + slurm_array_task_id - 1 + ] + + # %% -def main(model_name: str): # Added function and argument - """ - Main function to run the relaxation with the given model name. - """ - smoke_test = False - task_type = Task.IS2RE - module_dir = os.path.dirname(__file__) - # set large job array size for smaller data splits and faster testing/debugging - slurm_array_task_count = int( - os.getenv("SLURM_ARRAY_TASK_COUNT", "1") - ) # will be set to the number of tasks in the job array. - ase_optimizer = "FIRE" - job_name = f"{model_name}-wbm-{task_type}-{ase_optimizer}" # Use passed model_name - out_dir = os.getenv("SBATCH_OUTPUT", f"{module_dir}/{today}-{job_name}") - device = "gpu" - # whether to record intermediate structures into pymatgen Trajectory - record_traj = False # has no effect if relax_cell is False - - ase_filter: Literal["frechet", "exp"] = "frechet" - os.makedirs(out_dir, exist_ok=True) - - # slurm_vars = slurm_submit( - # job_name=job_name, - # out_dir=out_dir, - # array=f"1-{slurm_array_task_count}", - # # slurm_flags="--qos shared --constraint gpu --gpus 1", - # slurm_flags="--ntasks=1 --cpus-per-task=1 --partition high-priority", - # ) - - slurm_vars = {k: v for k, v in os.environ.items() if k.startswith("SLURM_")} - - # %% - slurm_array_task_id = int( - os.getenv("SLURM_ARRAY_TASK_ID", "0") - ) # will be set to the job array index value. - slurm_array_job_id = os.getenv( - "SLURM_ARRAY_JOB_ID", "debug" - ) # will be set to the first job ID of the array. - - out_path = f"{out_dir}/{slurm_array_job_id}-{slurm_array_task_id:>03}.json.gz" - - if os.path.isfile(out_path): - raise SystemExit(f"{out_path=} already exists, exciting early") - - print(f"{slurm_array_task_id=}") - print(f"{slurm_array_job_id=}") - print(f"{slurm_array_task_count=}") - print(f"{slurm_vars=}") - print(f"{out_dir=}") - - # %% - data_path = { - Task.RS2RE: DataFiles.wbm_relaxed_atoms.path, - Task.IS2RE: DataFiles.wbm_initial_atoms.path, - }[task_type] - print(f"\nJob {job_name} started {timestamp}") - e_pred_col = "grace_energy" - max_steps = 500 - force_max = 0.05 # Run until the forces are smaller than this in eV/A - checkpoint = "" - dtype = "float64" - calc = grace_fm( - model=model_name, pad_neighbors_fraction=0.05, pad_atoms_number=2, min_dist=0.5 - ) # Use passed model_name - - print(f"Read data from {data_path}") - atoms_list: list[Atoms] = ase_atoms_from_zip(data_path) - atoms_list = np.array(atoms_list, dtype="object") - - if slurm_array_job_id == "debug": - if smoke_test: - atoms_list = atoms_list[:128] - else: - pass - elif slurm_array_task_count > 1: - atoms_list = np.array_split(atoms_list, slurm_array_task_count)[ - slurm_array_task_id - 1 - ] - - # %% - run_params = { - "data_path": data_path, - "versions": { - dep: version(dep) - for dep in ("tensorpotential", "numpy", "tensorflow", "ase") - }, - "checkpoint": checkpoint, - Key.task_type: task_type, - "n_structures": len(atoms_list), - "slurm_vars": slurm_vars, - "max_steps": max_steps, - "record_traj": record_traj, - "force_max": force_max, - "ase_optimizer": ase_optimizer, - "device": device, - # Key.model_params: count_parameters(calc.models[0]), - "model_name": model_name, # Use passed model_name - "dtype": dtype, - "ase_filter": ase_filter, - } - - run_name = f"{job_name}-{slurm_array_task_id}" - - with open( - os.path.join( - out_dir, f"run_data_{slurm_array_task_id}-{slurm_array_task_count}.json" - ), - "w", - ) as f: - json.dump(run_params, f) - - # wandb.init(project="matbench-discovery", name=run_name, config=run_params) - - # %% time - relax_results: dict[str, dict[str, Any]] = {} - filter_cls: Callable[[Atoms], Atoms] = { - "frechet": FrechetCellFilter, - "exp": ExpCellFilter, - }[ase_filter] - optim_cls: Optimizer = {"FIRE": FIRE, "LBFGS": LBFGS}[ase_optimizer] - atoms_list = sorted(atoms_list, key=lambda at: len(at)) - # print(atoms_list) - for atoms in tqdm(deepcopy(atoms_list), desc="Relaxing", mininterval=5): - mat_id = atoms.info[Key.mat_id] - if mat_id in relax_results: - continue - try: - atoms.calc = calc - if max_steps > 0: - filtered_atoms = filter_cls(atoms) - optimizer = optim_cls(filtered_atoms, logfile="/dev/null") - - if record_traj: - coords, lattices, energies = [], [], [] - # attach observer functions to the optimizer - optimizer.attach(lambda: coords.append(atoms.get_positions())) # noqa: B023 - optimizer.attach(lambda: lattices.append(atoms.get_cell())) # noqa: B023 - optimizer.attach( - lambda: energies.append(atoms.get_potential_energy()) - ) # noqa: B023 - - optimizer.run(fmax=force_max, steps=max_steps) - energy = atoms.get_potential_energy() # relaxed energy - # if max_steps > 0, atoms is wrapped by filter_cls, so extract with getattr - relaxed_struct = AseAtomsAdaptor.get_structure(atoms) - relax_results[mat_id] = {"structure": relaxed_struct, "energy": energy} - - coords = locals().get("coords", []) - lattices = locals().get("lattices", []) - energies = locals().get("energies", []) - if record_traj and coords and lattices and energies: - traj = Trajectory( - species=atoms.get_chemical_symbols(), - coords=coords, - lattice=lattices, - constant_lattice=False, - frame_properties=[{"energy": energy} for energy in energies], +run_params = { + "data_path": data_path, + "versions": { + dep: version(dep) for dep in ("tensorpotential", "numpy", "tensorflow", "ase") + }, + "checkpoint": checkpoint, + Key.task_type: task_type, + "n_structures": len(atoms_list), + "max_steps": max_steps, + "record_traj": record_traj, + "force_max": force_max, + "ase_optimizer": ase_optimizer, + "device": device, + # Key.model_params: count_parameters(calc.models[0]), + "model_name": model_name, # Use passed model_name + "dtype": dtype, + "ase_filter": ase_filter, +} + +run_name = f"{job_name}-{slurm_array_task_id}" + +with open( + f"{out_dir}/run_data_{slurm_array_task_id}-{slurm_array_task_count}.json", mode="w" +) as file: + json.dump(run_params, file) + +# wandb.init(project="matbench-discovery", name=run_name, config=run_params) + + +# %% time +relax_results: dict[str, dict[str, Any]] = {} +filter_cls: Callable[[Atoms], Atoms] = { + "frechet": FrechetCellFilter, + "exp": ExpCellFilter, +}[ase_filter] +optim_cls: Optimizer = {"FIRE": FIRE, "LBFGS": LBFGS}[ase_optimizer] +atoms_list = sorted(atoms_list, key=lambda at: len(at)) +# print(atoms_list) +for atoms in tqdm(deepcopy(atoms_list), desc="Relaxing", mininterval=5): + mat_id = atoms.info[Key.mat_id] + if mat_id in relax_results: + continue + try: + atoms.calc = calc + if max_steps > 0: + filtered_atoms = filter_cls(atoms) + optimizer = optim_cls(filtered_atoms, logfile="/dev/null") + + if record_traj: + coords, lattices, energies = [], [], [] + # attach observer functions to the optimizer + optimizer.attach(lambda: coords.append(atoms.get_positions())) # noqa: B023 + optimizer.attach(lambda: lattices.append(atoms.get_cell())) # noqa: B023 + optimizer.attach( + lambda: energies.append(atoms.get_potential_energy()) # noqa: B023 ) - relax_results[mat_id]["trajectory"] = traj - except Exception as exc: - print(f"Failed to relax {mat_id}: {exc!r}") - continue - - # %% - df_out = pd.DataFrame(relax_results).T.add_prefix("grace_") - df_out.index.name = Key.mat_id - if not smoke_test: - df_out.reset_index().to_json(out_path, default_handler=as_dict_handler) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Run relaxation with a specified model." - ) - parser.add_argument("model_name", type=str, help="The name of the model to use.") - args = parser.parse_args() - - main(args.model_name) + + optimizer.run(fmax=force_max, steps=max_steps) + energy = atoms.get_potential_energy() # relaxed energy + # if max_steps > 0, atoms is wrapped by filter_cls, so extract with getattr + relaxed_struct = AseAtomsAdaptor.get_structure(atoms) + relax_results[mat_id] = {"structure": relaxed_struct, "energy": energy} + + coords = locals().get("coords", []) + lattices = locals().get("lattices", []) + energies = locals().get("energies", []) + if record_traj and coords and lattices and energies: + traj = Trajectory( + species=atoms.get_chemical_symbols(), + coords=coords, + lattice=lattices, + constant_lattice=False, + frame_properties=[{"energy": energy} for energy in energies], + ) + relax_results[mat_id]["trajectory"] = traj + except Exception as exc: + print(f"Failed to relax {mat_id}: {exc!r}") + continue + + +# %% +df_out = pd.DataFrame(relax_results).T.add_prefix("grace_") +df_out.index.name = Key.mat_id +if not smoke_test: + df_out.reset_index().to_json(out_path, default_handler=as_dict_handler) diff --git a/models/orb/join_orb_preds.py b/models/orb/join_orb_preds.py index 3b6cfc1a..4d5ac353 100644 --- a/models/orb/join_orb_preds.py +++ b/models/orb/join_orb_preds.py @@ -1,4 +1,4 @@ -import glob +from glob import glob import pandas as pd import typer @@ -35,7 +35,7 @@ def main( - {predictions_dir}/{prefix}.json.gz (all predictions as JSON) - {predictions_dir}/{prefix}-bad.csv (predictions with large errors) """ - file_paths = sorted(glob.glob(f"{predictions_dir}/{glob_pattern}")) + file_paths = sorted(glob(f"{predictions_dir}/{glob_pattern}")) print(f"Found {len(file_paths):,} files for {glob_pattern = }") dfs: dict[str, pd.DataFrame] = {} diff --git a/tests/test_hpc.py b/tests/test_hpc.py index 1b889077..bd197baa 100644 --- a/tests/test_hpc.py +++ b/tests/test_hpc.py @@ -319,8 +319,8 @@ def test_slurm_submit( expected_py_file_path = py_file_path or __file__ if submit_as_temp_file: - expected_py_file_path = os.path.join( - "/tmp/slurm_job_123", os.path.basename(expected_py_file_path) + expected_py_file_path = ( + f"/tmp/slurm_job_123/{os.path.basename(expected_py_file_path)}" ) assert mock_copy2.called assert mock_copy2.call_args[0][0] == (py_file_path or __file__) From 852d0f5ee9b440d5292e59da0405248c22bd29a0 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Fri, 7 Feb 2025 06:43:09 -0500 Subject: [PATCH 08/17] update GRACE model YAML files with pred_file paths and pr_url - rename grace2l-r6.yml to grace-2L-mptrj.yml - rename GRACE-(1|2)L-OAM model YAML files to grace-(1|2)l-oam.yml - update Model enum in data.py to include new GRACE model paths --- matbench_discovery/data.py | 4 +- models/grace/grace-1L-OAM.yml | 143 ------------------ models/grace/grace-1L-oam.yml | 68 +++++++++ models/grace/grace-2L-OAM.yml | 143 ------------------ .../{grace2l-r6.yml => grace-2L-mptrj.yml} | 14 +- models/grace/grace-2L-oam.yml | 68 +++++++++ tests/test_data.py | 2 +- 7 files changed, 147 insertions(+), 295 deletions(-) delete mode 100644 models/grace/grace-1L-OAM.yml create mode 100644 models/grace/grace-1L-oam.yml delete mode 100644 models/grace/grace-2L-OAM.yml rename models/grace/{grace2l-r6.yml => grace-2L-mptrj.yml} (88%) create mode 100644 models/grace/grace-2L-oam.yml diff --git a/matbench_discovery/data.py b/matbench_discovery/data.py index 6cf42f3e..1058ad1e 100644 --- a/matbench_discovery/data.py +++ b/matbench_discovery/data.py @@ -472,7 +472,9 @@ class Model(Files, base_dir=f"{ROOT}/models"): eqv2_s_dens = "eqV2/eqV2-s-dens-mp.yml" eqv2_m = "eqV2/eqV2-m-omat-mp-salex.yml" - grace2l_r6 = "grace/grace2l-r6.yml" + grace_2l_mptrj = "grace/grace-2L-mptrj.yml" + grace_2l_oam = "grace/grace-2L-oam.yml" + grace_1l_oam = "grace/grace-1L-oam.yml" # --- Model Combos # # CHGNet-relaxed structures fed into MEGNet for formation energy prediction diff --git a/models/grace/grace-1L-OAM.yml b/models/grace/grace-1L-OAM.yml deleted file mode 100644 index 6c7a82e6..00000000 --- a/models/grace/grace-1L-OAM.yml +++ /dev/null @@ -1,143 +0,0 @@ -model_name: GRACE-1L-OAM -model_key: grace -model_version: GRACE-1L-OAM_2Feb25 -matbench_discovery_version: 1.2.0 -date_added: "2025-02-06" -date_published: "2025-02-06" -authors: - - name: Anton Bochkarev - affiliation: ICAMS, Ruhr University Bochum - email: anton.bochkarev@rub.de - - name: Yury Lysogorskiy - affiliation: ICAMS, Ruhr University Bochum - email: yury.lysogorskiy@rub.de - - name: Ralf Drautz - affiliation: ICAMS, Ruhr University Bochum - email: ralf.drautz@rub.de -trained_by: - - name: Yury Lysogorskiy - affiliation: ICAMS, Ruhr University Bochum - email: yury.lysogorskiy@rub.de - - name: Anton Bochkarev - affiliation: ICAMS, Ruhr University Bochum - email: anton.bochkarev@rub.de - -repo: https://github.com/ICAMS/grace-tensorpotential -doi: https://doi.org/10.1103/PhysRevX.14.021036 -paper: https://journals.aps.org/prx/abstract/10.1103/PhysRevX.14.021036 -url: https://gracemaker.readthedocs.io/en/latest/gracemaker/foundation -#pr_url: https://github.com/janosh/matbench-discovery/pull/160 # TODO - -requirements: - tensorpotential: 0.4.5 - tensorflow: 2.16.2 - ase: 3.23.0 - pymatgen: 2023.7.14 - numpy: 1.26.4 - -openness: OSOD -trained_for_benchmark: true -train_task: S2EF -test_task: IS2RE-SR -targets: EFS_G -model_type: UIP -model_params: 3_447_148 -n_estimators: 1 - -training_set: [OMat24, sAlex, MPtrj] - -hyperparams: - max_force: 0.03 - max_steps: 500 - ase_optimizer: FIRE - radial_cutoff: 6.0 - -metrics: - phonons: - kappa_103: - κ_SRME: 0.516 # https://github.com/MPA2suite/k_SRME/pull/20 - pred_file: TODO - pred_file_url: TODO - geo_opt: - pred_file: TODO - pred_file_url: TODO - pred_col: grace_structure - # symprec=1e-5: - # rmsd: 0.0186 # Å - # n_sym_ops_mae: 1.8703 # unitless - # symmetry_decrease: 0.0355 # fraction - # symmetry_match: 0.7315 # fraction - # symmetry_increase: 0.2285 # fraction - # n_structures: 256862 # count - # analysis_file: models/grace/grace2l_r6/2024-11-11-relaxed-structures-symprec=1e-5.csv.gz - # analysis_file_url: https://figshare.com/ndownloader/files/52062641 - # symprec=1e-2: - # rmsd: 0.0186 # Å - # n_sym_ops_mae: 1.8982 # unitless - # symmetry_decrease: 0.0592 # fraction - # symmetry_match: 0.7976 # fraction - # symmetry_increase: 0.1363 # fraction - # n_structures: 256513 # count - # analysis_file: models/grace/grace2l_r6/2024-11-11-relaxed-structures-symprec=1e-2.csv.gz - # analysis_file_url: https://figshare.com/ndownloader/files/52062650 - discovery: - # pred_file: TODO - # pred_file_url: TODO - # pred_col: e_form_per_atom_grace - # full_test_set: - # F1: 0.687 # fraction - # DAF: 3.714 # dimensionless - # Precision: 0.637 # fraction - # Recall: 0.746 # fraction - # Accuracy: 0.884 # fraction - # TPR: 0.746 # fraction - # FPR: 0.088 # fraction - # TNR: 0.912 # fraction - # FNR: 0.254 # fraction - # TP: 32880.0 # count - # FP: 18721.0 # count - # TN: 194150.0 # count - # FN: 11212.0 # count - # MAE: 0.05 # eV/atom - # RMSE: 0.092 # eV/atom - # R2: 0.74 # dimensionless - # missing_preds: 128 # count - # missing_percent: 0.05% # fraction - # most_stable_10k: - # F1: 0.914 # fraction - # DAF: 5.503 # dimensionless - # Precision: 0.841 # fraction - # Recall: 1.0 # fraction - # Accuracy: 0.841 # fraction - # TPR: 1.0 # fraction - # FPR: 1.0 # fraction - # TNR: 0.0 # fraction - # FNR: 0.0 # fraction - # TP: 8413.0 # count - # FP: 1587.0 # count - # TN: 0.0 # count - # FN: 0.0 # count - # MAE: 0.065 # eV/atom - # RMSE: 0.133 # eV/atom - # R2: 0.641 # dimensionless - # missing_preds: 0 # count - # missing_percent: 0.00% # fraction - # unique_prototypes: - # F1: 0.691 # fraction - # DAF: 4.163 # dimensionless - # Precision: 0.636 # fraction - # Recall: 0.757 # fraction - # Accuracy: 0.896 # fraction - # TPR: 0.757 # fraction - # FPR: 0.079 # fraction - # TNR: 0.921 # fraction - # FNR: 0.243 # fraction - # TP: 25255.0 # count - # FP: 14425.0 # count - # TN: 167689.0 # count - # FN: 8119.0 # count - # MAE: 0.052 # eV/atom - # RMSE: 0.094 # eV/atom - # R2: 0.741 # dimensionless - # missing_preds: 110 # count - # missing_percent: 0.05% # fraction diff --git a/models/grace/grace-1L-oam.yml b/models/grace/grace-1L-oam.yml new file mode 100644 index 00000000..51674759 --- /dev/null +++ b/models/grace/grace-1L-oam.yml @@ -0,0 +1,68 @@ +model_name: GRACE-1L-OAM +model_key: grace-1L-oam +model_version: GRACE-1L-OAM_2Feb25 +matbench_discovery_version: 1.2.0 +date_added: "2025-02-06" +date_published: "2025-02-06" +authors: + - name: Anton Bochkarev + affiliation: ICAMS, Ruhr University Bochum + email: anton.bochkarev@rub.de + - name: Yury Lysogorskiy + affiliation: ICAMS, Ruhr University Bochum + email: yury.lysogorskiy@rub.de + - name: Ralf Drautz + affiliation: ICAMS, Ruhr University Bochum + email: ralf.drautz@rub.de +trained_by: + - name: Yury Lysogorskiy + affiliation: ICAMS, Ruhr University Bochum + email: yury.lysogorskiy@rub.de + - name: Anton Bochkarev + affiliation: ICAMS, Ruhr University Bochum + email: anton.bochkarev@rub.de + +repo: https://github.com/ICAMS/grace-tensorpotential +doi: https://doi.org/10.1103/PhysRevX.14.021036 +paper: https://journals.aps.org/prx/abstract/10.1103/PhysRevX.14.021036 +url: https://gracemaker.readthedocs.io/en/latest/gracemaker/foundation +pr_url: https://github.com/janosh/matbench-discovery/pull/202 + +requirements: + tensorpotential: 0.4.5 + tensorflow: 2.16.2 + ase: 3.23.0 + pymatgen: 2023.7.14 + numpy: 1.26.4 + +openness: OSOD +trained_for_benchmark: true +train_task: S2EF +test_task: IS2RE-SR +targets: EFS_G +model_type: UIP +model_params: 3_447_148 +n_estimators: 1 + +training_set: [OMat24, sAlex, MPtrj] + +hyperparams: + max_force: 0.03 + max_steps: 500 + ase_optimizer: FIRE + radial_cutoff: 6.0 + +metrics: + phonons: + kappa_103: + κ_SRME: 0.516 # https://github.com/MPA2suite/k_SRME/pull/20 + pred_file: models/grace/grace-1L-oam/2025-02-02-kappa-103-FIRE-dist=0.01-fmax=1e-4-symprec=1e-5.json.gz + pred_file_url: + geo_opt: + pred_file: models/grace/grace-1L-oam/2025-02-02-wbm-geo-opt.json.gz + pred_file_url: + struct_col: grace_structure + discovery: + pred_file: models/grace/grace-1L-oam/2025-02-02-wbm-IS2RE.csv.gz + pred_file_url: + pred_col: e_form_per_atom_grace diff --git a/models/grace/grace-2L-OAM.yml b/models/grace/grace-2L-OAM.yml deleted file mode 100644 index 7a933f14..00000000 --- a/models/grace/grace-2L-OAM.yml +++ /dev/null @@ -1,143 +0,0 @@ -model_name: GRACE-2L-OAM -model_key: grace -model_version: GRACE_2L_OAM_28Jan25 -matbench_discovery_version: 1.2.0 -date_added: "2025-02-06" -date_published: "2025-02-06" -authors: - - name: Anton Bochkarev - affiliation: ICAMS, Ruhr University Bochum - email: anton.bochkarev@rub.de - - name: Yury Lysogorskiy - affiliation: ICAMS, Ruhr University Bochum - email: yury.lysogorskiy@rub.de - - name: Ralf Drautz - affiliation: ICAMS, Ruhr University Bochum - email: ralf.drautz@rub.de -trained_by: - - name: Yury Lysogorskiy - affiliation: ICAMS, Ruhr University Bochum - email: yury.lysogorskiy@rub.de - - name: Anton Bochkarev - affiliation: ICAMS, Ruhr University Bochum - email: anton.bochkarev@rub.de - -repo: https://github.com/ICAMS/grace-tensorpotential -doi: https://doi.org/10.1103/PhysRevX.14.021036 -paper: https://journals.aps.org/prx/abstract/10.1103/PhysRevX.14.021036 -url: https://gracemaker.readthedocs.io/en/latest/gracemaker/foundation -#pr_url: https://github.com/janosh/matbench-discovery/pull/160 # TODO - -requirements: - tensorpotential: 0.4.5 - tensorflow: 2.16.2 - ase: 3.23.0 - pymatgen: 2023.7.14 - numpy: 1.26.4 - -openness: OSOD -trained_for_benchmark: true -train_task: S2EF -test_task: IS2RE-SR -targets: EFS_G -model_type: UIP -model_params: 12_597_516 -n_estimators: 1 - -training_set: [OMat24, sAlex, MPtrj] - -hyperparams: - max_force: 0.03 - max_steps: 500 - ase_optimizer: FIRE - radial_cutoff: 6.0 - -metrics: - phonons: - kappa_103: - κ_SRME: 0.294 # https://github.com/MPA2suite/k_SRME/pull/20 - pred_file: TODO - pred_file_url: TODO - geo_opt: - pred_file: TODO - pred_file_url: TODO - pred_col: grace_structure - # symprec=1e-5: - # rmsd: 0.0186 # Å - # n_sym_ops_mae: 1.8703 # unitless - # symmetry_decrease: 0.0355 # fraction - # symmetry_match: 0.7315 # fraction - # symmetry_increase: 0.2285 # fraction - # n_structures: 256862 # count - # analysis_file: models/grace/grace2l_r6/2024-11-11-relaxed-structures-symprec=1e-5.csv.gz - # analysis_file_url: https://figshare.com/ndownloader/files/52062641 - # symprec=1e-2: - # rmsd: 0.0186 # Å - # n_sym_ops_mae: 1.8982 # unitless - # symmetry_decrease: 0.0592 # fraction - # symmetry_match: 0.7976 # fraction - # symmetry_increase: 0.1363 # fraction - # n_structures: 256513 # count - # analysis_file: models/grace/grace2l_r6/2024-11-11-relaxed-structures-symprec=1e-2.csv.gz - # analysis_file_url: https://figshare.com/ndownloader/files/52062650 - discovery: - # pred_file: TODO - # pred_file_url: TODO - # pred_col: e_form_per_atom_grace - # full_test_set: - # F1: 0.687 # fraction - # DAF: 3.714 # dimensionless - # Precision: 0.637 # fraction - # Recall: 0.746 # fraction - # Accuracy: 0.884 # fraction - # TPR: 0.746 # fraction - # FPR: 0.088 # fraction - # TNR: 0.912 # fraction - # FNR: 0.254 # fraction - # TP: 32880.0 # count - # FP: 18721.0 # count - # TN: 194150.0 # count - # FN: 11212.0 # count - # MAE: 0.05 # eV/atom - # RMSE: 0.092 # eV/atom - # R2: 0.74 # dimensionless - # missing_preds: 128 # count - # missing_percent: 0.05% # fraction - # most_stable_10k: - # F1: 0.914 # fraction - # DAF: 5.503 # dimensionless - # Precision: 0.841 # fraction - # Recall: 1.0 # fraction - # Accuracy: 0.841 # fraction - # TPR: 1.0 # fraction - # FPR: 1.0 # fraction - # TNR: 0.0 # fraction - # FNR: 0.0 # fraction - # TP: 8413.0 # count - # FP: 1587.0 # count - # TN: 0.0 # count - # FN: 0.0 # count - # MAE: 0.065 # eV/atom - # RMSE: 0.133 # eV/atom - # R2: 0.641 # dimensionless - # missing_preds: 0 # count - # missing_percent: 0.00% # fraction - # unique_prototypes: - # F1: 0.691 # fraction - # DAF: 4.163 # dimensionless - # Precision: 0.636 # fraction - # Recall: 0.757 # fraction - # Accuracy: 0.896 # fraction - # TPR: 0.757 # fraction - # FPR: 0.079 # fraction - # TNR: 0.921 # fraction - # FNR: 0.243 # fraction - # TP: 25255.0 # count - # FP: 14425.0 # count - # TN: 167689.0 # count - # FN: 8119.0 # count - # MAE: 0.052 # eV/atom - # RMSE: 0.094 # eV/atom - # R2: 0.741 # dimensionless - # missing_preds: 110 # count - # missing_percent: 0.05% # fraction diff --git a/models/grace/grace2l-r6.yml b/models/grace/grace-2L-mptrj.yml similarity index 88% rename from models/grace/grace2l-r6.yml rename to models/grace/grace-2L-mptrj.yml index 8bbd8a8e..a863c7d3 100644 --- a/models/grace/grace2l-r6.yml +++ b/models/grace/grace-2L-mptrj.yml @@ -1,5 +1,5 @@ model_name: GRACE-2L (r6) -model_key: grace2l-r6 +model_key: grace-2L-mptrj model_version: MP_GRACE_2L_r6_11Nov2024 matbench_discovery_version: 1.0.0 date_added: "2024-11-21" @@ -53,12 +53,12 @@ metrics: phonons: kappa_103: κ_SRME: 0.525 # https://github.com/MPA2suite/k_SRME/pull/11/files - pred_file: models/grace/grace2l_r6/2024-11-20-kappa-103-FIRE-fmax=1e-4-symprec=1e-5.json.gz + pred_file: models/grace/grace-2L-mptrj/2024-11-20-kappa-103-FIRE-fmax=1e-4-symprec=1e-5.json.gz pred_file_url: https://figshare.com/ndownloader/files/52134896 geo_opt: - pred_file: models/grace/grace2l_r6/2024-11-11-relaxed-structures.json.gz + pred_file: models/grace/grace-2L-mptrj/2024-11-11-relaxed-structures.json.gz pred_file_url: https://figshare.com/ndownloader/files/52062590 - pred_col: grace2l_r6_structure + pred_col: grace-2L-mp_structure symprec=1e-5: rmsd: 0.0186 # Å n_sym_ops_mae: 1.8703 # unitless @@ -66,7 +66,7 @@ metrics: symmetry_match: 0.7315 # fraction symmetry_increase: 0.2285 # fraction n_structures: 256862 # count - analysis_file: models/grace/grace2l_r6/2024-11-11-relaxed-structures-symprec=1e-5.csv.gz + analysis_file: models/grace/grace-2L-mptrj/2024-11-11-relaxed-structures-symprec=1e-5.csv.gz analysis_file_url: https://figshare.com/ndownloader/files/52062641 symprec=1e-2: rmsd: 0.0186 # Å @@ -75,10 +75,10 @@ metrics: symmetry_match: 0.7976 # fraction symmetry_increase: 0.1363 # fraction n_structures: 256513 # count - analysis_file: models/grace/grace2l_r6/2024-11-11-relaxed-structures-symprec=1e-2.csv.gz + analysis_file: models/grace/grace-2L-mptrj/2024-11-11-relaxed-structures-symprec=1e-2.csv.gz analysis_file_url: https://figshare.com/ndownloader/files/52062650 discovery: - pred_file: models/grace/grace2l_r6/2024-11-11-wbm-IS2RE-FIRE.csv.gz + pred_file: models/grace/grace-2L-mptrj/2024-11-11-wbm-IS2RE-FIRE.csv.gz pred_file_url: https://figshare.com/ndownloader/files/52057577 pred_col: e_form_per_atom_grace full_test_set: diff --git a/models/grace/grace-2L-oam.yml b/models/grace/grace-2L-oam.yml new file mode 100644 index 00000000..87174ab1 --- /dev/null +++ b/models/grace/grace-2L-oam.yml @@ -0,0 +1,68 @@ +model_name: GRACE-2L-OAM +model_key: grace-2L-oam +model_version: GRACE_2L_OAM_28Jan25 +matbench_discovery_version: 1.2.0 +date_added: "2025-02-06" +date_published: "2025-02-06" +authors: + - name: Anton Bochkarev + affiliation: ICAMS, Ruhr University Bochum + email: anton.bochkarev@rub.de + - name: Yury Lysogorskiy + affiliation: ICAMS, Ruhr University Bochum + email: yury.lysogorskiy@rub.de + - name: Ralf Drautz + affiliation: ICAMS, Ruhr University Bochum + email: ralf.drautz@rub.de +trained_by: + - name: Yury Lysogorskiy + affiliation: ICAMS, Ruhr University Bochum + email: yury.lysogorskiy@rub.de + - name: Anton Bochkarev + affiliation: ICAMS, Ruhr University Bochum + email: anton.bochkarev@rub.de + +repo: https://github.com/ICAMS/grace-tensorpotential +doi: https://doi.org/10.1103/PhysRevX.14.021036 +paper: https://journals.aps.org/prx/abstract/10.1103/PhysRevX.14.021036 +url: https://gracemaker.readthedocs.io/en/latest/gracemaker/foundation +pr_url: https://github.com/janosh/matbench-discovery/pull/202 + +requirements: + tensorpotential: 0.4.5 + tensorflow: 2.16.2 + ase: 3.23.0 + pymatgen: 2023.7.14 + numpy: 1.26.4 + +openness: OSOD +trained_for_benchmark: true +train_task: S2EF +test_task: IS2RE-SR +targets: EFS_G +model_type: UIP +model_params: 12_597_516 +n_estimators: 1 + +training_set: [OMat24, sAlex, MPtrj] + +hyperparams: + max_force: 0.03 + max_steps: 500 + ase_optimizer: FIRE + radial_cutoff: 6.0 + +metrics: + phonons: + kappa_103: + κ_SRME: 0.294 # https://github.com/MPA2suite/k_SRME/pull/20 + pred_file: models/grace/grace-2L-oam/2025-01-28-kappa-103-FIRE-dist=0.01-fmax=1e-4-symprec=1e-5.json.gz + pred_file_url: + geo_opt: + pred_file: models/grace/grace-2L-oam/2025-01-28-wbm-geo-opt.json.gz + pred_file_url: + struct_col: grace_structure + discovery: + pred_file: models/grace/grace-2L-oam/2025-01-28-wbm-IS2RE.csv.gz + pred_file_url: + pred_col: e_form_per_atom_grace diff --git a/tests/test_data.py b/tests/test_data.py index 19873969..efde552a 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -246,7 +246,7 @@ def test_model() -> None: # Test yaml_path property assert Model.alignn.yaml_path.endswith("alignn/alignn.yml") - assert (Model.grace2l_r6.kappa_103_path or "").endswith( + assert (Model.grace_2l_mptrj.kappa_103_path or "").endswith( "2024-11-20-kappa-103-FIRE-fmax=1e-4-symprec=1e-5.json.gz" ) From bcd18db059bf96376b2955881c7139d6e917d83f Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Fri, 7 Feb 2025 07:15:47 -0500 Subject: [PATCH 09/17] rename metrics.geo_opt.pred_col to struct_col in model YAML files --- models/bowsr/bowsr.yml | 2 +- models/chgnet/chgnet.yml | 2 +- models/deepmd/dpa3-v1-mptrj.yml | 2 +- models/deepmd/dpa3-v1-openlam.yml | 2 +- models/eqV2/eqV2-m-omat-mp-salex.yml | 2 +- models/eqV2/eqV2-s-dens-mp.yml | 2 +- models/grace/grace-2L-mptrj.yml | 2 +- models/m3gnet/m3gnet.yml | 2 +- models/mace/mace-mp-0.yml | 2 +- models/mace/mace-mpa-0.yml | 2 +- models/mattersim/mattersim-v1-5m.yml | 2 +- models/orb/orb-mptrj.yml | 2 +- models/orb/orb.yml | 2 +- models/sevennet/sevennet-0.yml | 2 +- models/sevennet/sevennet-l3i5.yml | 2 +- tests/model-schema.yml | 4 ++-- 16 files changed, 17 insertions(+), 17 deletions(-) diff --git a/models/bowsr/bowsr.yml b/models/bowsr/bowsr.yml index 8844ef39..90fd9858 100644 --- a/models/bowsr/bowsr.yml +++ b/models/bowsr/bowsr.yml @@ -62,7 +62,7 @@ metrics: geo_opt: pred_file: models/bowsr/bowsr-megnet/2023-01-23-wbm-geo-opt.json.gz pred_file_url: https://figshare.com/ndownloader/files/52061984 - pred_col: structure_bowsr_megnet + struct_col: structure_bowsr_megnet symprec=1e-5: rmsd: 0.043 # Å symmetry_decrease: 0.0037 # fraction diff --git a/models/chgnet/chgnet.yml b/models/chgnet/chgnet.yml index 0d4a0807..894c9464 100644 --- a/models/chgnet/chgnet.yml +++ b/models/chgnet/chgnet.yml @@ -72,7 +72,7 @@ metrics: geo_opt: pred_file: models/chgnet/chgnet-0.3.0/2023-12-21-wbm-geo-opt.json.gz pred_file_url: https://figshare.com/ndownloader/files/52061999 - pred_col: chgnet_structure + struct_col: chgnet_structure symprec=1e-5: rmsd: 0.0216 # Å symmetry_decrease: 0.2526 # fraction diff --git a/models/deepmd/dpa3-v1-mptrj.yml b/models/deepmd/dpa3-v1-mptrj.yml index ed61ccb0..bd481ca3 100644 --- a/models/deepmd/dpa3-v1-mptrj.yml +++ b/models/deepmd/dpa3-v1-mptrj.yml @@ -97,7 +97,7 @@ metrics: pred_file_url: https://figshare.com/ndownloader/files/52134860 geo_opt: pred_file: models/deepmd/dpa3-v1-mptrj/2025-01-10-wbm-geo-opt.json.gz - pred_col: dp_structure + struct_col: dp_structure pred_file_url: https://figshare.com/ndownloader/files/52134974 symprec=1e-5: analysis_file: models/deepmd/dpa3-v1-mptrj/2025-01-10-wbm-geo-opt-symprec=1e-5.csv.gz diff --git a/models/deepmd/dpa3-v1-openlam.yml b/models/deepmd/dpa3-v1-openlam.yml index deb08be7..aed00dd4 100644 --- a/models/deepmd/dpa3-v1-openlam.yml +++ b/models/deepmd/dpa3-v1-openlam.yml @@ -109,7 +109,7 @@ metrics: pred_file_url: https://figshare.com/ndownloader/files/52134863 geo_opt: pred_file: models/deepmd/dpa3-v1-openlam/2025-01-10-wbm-geo-opt.json.gz - pred_col: dp_structure + struct_col: dp_structure pred_file_url: https://figshare.com/ndownloader/files/52135358 symprec=1e-5: analysis_file: models/deepmd/dpa3-v1-openlam/2025-01-10-wbm-geo-opt-symprec=1e-5.csv.gz diff --git a/models/eqV2/eqV2-m-omat-mp-salex.yml b/models/eqV2/eqV2-m-omat-mp-salex.yml index b8980d98..c3764123 100644 --- a/models/eqV2/eqV2-m-omat-mp-salex.yml +++ b/models/eqV2/eqV2-m-omat-mp-salex.yml @@ -92,7 +92,7 @@ metrics: geo_opt: pred_file: models/eqV2/eqV2-m-omat-mp-salex/2024-10-18-wbm-geo-opt.json.gz pred_file_url: https://figshare.com/ndownloader/files/51607436 - pred_col: eqV2-86M-omat-mp-salex_structure + struct_col: eqV2-86M-omat-mp-salex_structure symprec=1e-5: rmsd: 0.0138 # Å n_sym_ops_mae: 10.0558 # unitless diff --git a/models/eqV2/eqV2-s-dens-mp.yml b/models/eqV2/eqV2-s-dens-mp.yml index 098c34bd..9008ce1b 100644 --- a/models/eqV2/eqV2-s-dens-mp.yml +++ b/models/eqV2/eqV2-s-dens-mp.yml @@ -94,7 +94,7 @@ metrics: geo_opt: pred_file: models/eqV2/eqV2-s-dens-mp/2024-10-18-wbm-geo-opt.json.gz pred_file_url: https://figshare.com/ndownloader/files/52062392 - pred_col: eqV2-31M-dens-MP-p5_structure + struct_col: eqV2-31M-dens-MP-p5_structure symprec=1e-5: rmsd: 0.0138 # Å n_sym_ops_mae: 10.0558 # unitless diff --git a/models/grace/grace-2L-mptrj.yml b/models/grace/grace-2L-mptrj.yml index a863c7d3..749aaae8 100644 --- a/models/grace/grace-2L-mptrj.yml +++ b/models/grace/grace-2L-mptrj.yml @@ -58,7 +58,7 @@ metrics: geo_opt: pred_file: models/grace/grace-2L-mptrj/2024-11-11-relaxed-structures.json.gz pred_file_url: https://figshare.com/ndownloader/files/52062590 - pred_col: grace-2L-mp_structure + struct_col: grace-2L-mp_structure symprec=1e-5: rmsd: 0.0186 # Å n_sym_ops_mae: 1.8703 # unitless diff --git a/models/m3gnet/m3gnet.yml b/models/m3gnet/m3gnet.yml index 48d98022..2de74e45 100644 --- a/models/m3gnet/m3gnet.yml +++ b/models/m3gnet/m3gnet.yml @@ -62,7 +62,7 @@ metrics: geo_opt: pred_file: models/m3gnet/m3gnet-tf-manual-sampling/2023-06-01-wbm-geo-opt.json.gz pred_file_url: https://figshare.com/ndownloader/files/52062011 - pred_col: m3gnet_structure + struct_col: m3gnet_structure symprec=1e-5: rmsd: 0.0217 # Å n_sym_ops_mae: 1.7751 # unitless diff --git a/models/mace/mace-mp-0.yml b/models/mace/mace-mp-0.yml index 2c1ec232..b4475b8a 100644 --- a/models/mace/mace-mp-0.yml +++ b/models/mace/mace-mp-0.yml @@ -77,7 +77,7 @@ metrics: geo_opt: pred_file: models/mace/mace-mp-0/2023-12-11-wbm-IS2RE-FIRE.json.gz pred_file_url: https://figshare.com/ndownloader/files/52062020 - pred_col: mace_structure + struct_col: mace_structure symprec=1e-5: rmsd: 0.0194 # Å n_sym_ops_mae: 1.8584 # unitless diff --git a/models/mace/mace-mpa-0.yml b/models/mace/mace-mpa-0.yml index 7dc79d6d..d9f85217 100644 --- a/models/mace/mace-mpa-0.yml +++ b/models/mace/mace-mpa-0.yml @@ -76,7 +76,7 @@ metrics: geo_opt: pred_file: models/mace/mace-mpa-0/2024-12-09-wbm-IS2RE-FIRE.json.gz pred_file_url: https://figshare.com/TODO add pred file URL - pred_col: mace_structure + struct_col: mace_structure symprec=1e-2: rmsd: 0.0142 # Å n_sym_ops_mae: 1.807 # unitless diff --git a/models/mattersim/mattersim-v1-5m.yml b/models/mattersim/mattersim-v1-5m.yml index ff2fc8bf..3feade24 100644 --- a/models/mattersim/mattersim-v1-5m.yml +++ b/models/mattersim/mattersim-v1-5m.yml @@ -128,7 +128,7 @@ metrics: geo_opt: pred_file: models/mattersim/mattersim-v1-5M/2024-12-19-wbm-geo-opt.json.gz pred_file_url: https://figshare.com/ndownloader/files/52062176 - pred_col: mattersim_structure + struct_col: mattersim_structure discovery: pred_file: models/mattersim/mattersim-v1-5M/2024-12-16-wbm-IS2RE.csv.gz # the original Graphormer-based replaced the M3GNet-based MatterSim on the leaderboard diff --git a/models/orb/orb-mptrj.yml b/models/orb/orb-mptrj.yml index d33906da..43e35c2b 100644 --- a/models/orb/orb-mptrj.yml +++ b/models/orb/orb-mptrj.yml @@ -81,7 +81,7 @@ metrics: geo_opt: pred_file: models/orb/orbff-mptrj-v2/2024-10-14-wbm-geo-opt.json.gz pred_file_url: https://figshare.com/ndownloader/files/52062317 - pred_col: orb_structure + struct_col: orb_structure symprec=1e-5: rmsd: 0.0185 # Å n_sym_ops_mae: 10.036 # unitless diff --git a/models/orb/orb.yml b/models/orb/orb.yml index 808f85e0..251787bc 100644 --- a/models/orb/orb.yml +++ b/models/orb/orb.yml @@ -81,7 +81,7 @@ metrics: geo_opt: pred_file: models/orb/orbff-v2/2024-10-11-wbm-geo-opt.json.gz pred_file_url: https://figshare.com/ndownloader/files/52062194 - pred_col: orb_structure + struct_col: orb_structure symprec=1e-5: rmsd: 0.016 # Å n_sym_ops_mae: 9.8834 # unitless diff --git a/models/sevennet/sevennet-0.yml b/models/sevennet/sevennet-0.yml index bacbf7a4..85c4ff0e 100644 --- a/models/sevennet/sevennet-0.yml +++ b/models/sevennet/sevennet-0.yml @@ -85,7 +85,7 @@ metrics: geo_opt: pred_file: models/sevennet/sevennet-0/2024-07-11-wbm-geo-opt.json.gz pred_file_url: https://figshare.com/ndownloader/files/52062035 - pred_col: sevennet_structure + struct_col: sevennet_structure symprec=1e-5: rmsd: 0.0193 # Å n_sym_ops_mae: 2.5921 # unitless diff --git a/models/sevennet/sevennet-l3i5.yml b/models/sevennet/sevennet-l3i5.yml index ff9b7785..4995d9dc 100644 --- a/models/sevennet/sevennet-l3i5.yml +++ b/models/sevennet/sevennet-l3i5.yml @@ -84,7 +84,7 @@ metrics: geo_opt: pred_file: models/sevennet/sevennet-l3i5/2024-12-10-wbm-geo-opt.json.gz pred_file_url: https://figshare.com/ndownloader/files/52062056 - pred_col: sevennet_structures + struct_col: sevennet_structures symprec=1e-5: rmsd: 0.0182 # Å n_sym_ops_mae: 2.7288 # unitless diff --git a/tests/model-schema.yml b/tests/model-schema.yml index 50d4c664..f651d99e 100644 --- a/tests/model-schema.yml +++ b/tests/model-schema.yml @@ -178,7 +178,7 @@ properties: oneOf: - type: object additionalProperties: false - required: [pred_col] + required: [struct_col] properties: pred_file: type: [string, "null"] @@ -186,7 +186,7 @@ properties: type: [string, "null"] format: uri pattern: "^https?://" - pred_col: + struct_col: type: string symprec=1e-5: &symprec_metrics type: object From d12070ac88be881d9189c8337868e60def6822e6 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Fri, 7 Feb 2025 07:50:08 -0500 Subject: [PATCH 10/17] analyze_geo_opt.py add CLI - with argparse for passing models, symprec values to analyze and debug mode - Add example usage in script docstring --- scripts/analyze_geo_opt.py | 167 +++++++++++++++++++++++++------------ 1 file changed, 114 insertions(+), 53 deletions(-) diff --git a/scripts/analyze_geo_opt.py b/scripts/analyze_geo_opt.py index 10e5baa4..dd18a0d1 100644 --- a/scripts/analyze_geo_opt.py +++ b/scripts/analyze_geo_opt.py @@ -3,12 +3,18 @@ Output files will have the same name as the input file containing the model's relaxed structures, but with the symprec value appended to the filename. + +Example usage: + python scripts/analyze_geo_opt.py --models mace_mp_0 m3gnet --symprec 1e-2 1e-5 + python scripts/analyze_geo_opt.py --debug 10 # only analyze first 10 structures """ # %% +import argparse import importlib import importlib.metadata import os +from collections.abc import Sequence from typing import Any, Final import pandas as pd @@ -21,17 +27,52 @@ from matbench_discovery.models import MODEL_METADATA from matbench_discovery.structure import analyze_symmetry, pred_vs_ref_struct_symmetry +parser = argparse.ArgumentParser() +parser.add_argument( + "--models", + nargs="+", + default=["all"], + help="Model names to analyze. Use 'all' to analyze all models.", +) +parser.add_argument( + "--symprec", + nargs="+", + type=float, + default=[1e-2, 1e-5], + help="Symmetry precision values to analyze.", +) +parser.add_argument( + "--debug", + type=int, + default=0, + help="If > 0, only analyze this many structures.", +) +args = parser.parse_args() + # activate debug mode by setting to any number > 0, only that many structures will be # analyzed -debug_mode: Final[int] = 0 -symprec: Final[float] = 1e-2 -symprec_str = f"symprec={symprec:.0e}".replace("e-0", "e-") +debug_mode: Final[int] = args.debug +# List of symprec values to analyze +symprec_values: Final[Sequence[float]] = args.symprec -moyo_version = f"moyo={importlib.metadata.version('moyopy')}" -dft_csv_path = f"{ROOT}/data/wbm/dft-geo-opt-{symprec_str}-{moyo_version}.csv.gz" +# Get list of models to analyze +if "all" in args.models: + model_names = [model.name for model in Model] +else: + model_names = args.models + # Validate model names + valid_models = {model.name for model in Model} + for model_name in model_names: + if model_name not in valid_models: + raise ValueError( + f"Invalid model name: {model_name}. Valid models are: {valid_models}" + ) +moyo_version = f"moyo={importlib.metadata.version('moyopy')}" +moyo_version = "moyo=0.3.1" # %% Load WBM reference structures (this takes a while) +print("Loading WBM reference structures...") wbm_cse_path = DataFiles.wbm_computed_structure_entries.path df_wbm_structs: pd.DataFrame = pd.read_json(wbm_cse_path).set_index(Key.mat_id) dft_structs: dict[str, Structure] = { @@ -42,23 +83,27 @@ if debug_mode: df_wbm_structs = df_wbm_structs.head(debug_mode) - -# %% -if os.path.isfile(dft_csv_path): - dft_analysis = pd.read_csv(dft_csv_path) -else: - dft_analysis = analyze_symmetry( - dft_structs, - pbar=dict(desc=f"Getting DFT symmetries {symprec=}"), - symprec=symprec, - ) - dft_analysis.to_csv(dft_csv_path) - +# %% Process DFT structures for each symprec value +dft_analysis_dict = {} +for symprec in symprec_values: + symprec_str = f"symprec={symprec:.0e}".replace("e-0", "e-") + dft_csv_path = f"{ROOT}/data/wbm/dft-geo-opt-{symprec_str}-{moyo_version}.csv.gz" + + if os.path.isfile(dft_csv_path): + dft_analysis_dict[symprec] = pd.read_csv(dft_csv_path) + else: + dft_analysis_dict[symprec] = analyze_symmetry( + dft_structs, + pbar=dict(desc=f"Getting DFT symmetries {symprec=}"), + symprec=symprec, + ) + dft_analysis_dict[symprec].to_csv(dft_csv_path) # %% Process each model sequentially -for idx, (model_label, model_metadata) in enumerate(MODEL_METADATA.items()): - prog_str = f"{idx + 1}/{len(MODEL_METADATA)}:" - model = Model.from_label(model_label) +for idx, model_name in enumerate(model_names): + prog_str = f"{idx + 1}/{len(model_names)}:" + model = Model[model_name] + model_metadata = MODEL_METADATA[model.label] geo_opt_metrics: dict[str, Any] = model_metadata.get("metrics", {}).get( "geo_opt", {} @@ -69,37 +114,38 @@ continue if not model.geo_opt_path: - print(f"⚠️ {model_label} has no relaxed structures file") + print(f"⚠️ {model.label} has no relaxed structures file") continue if not os.path.isfile(ml_relaxed_structs_path := model.geo_opt_path): print( - f"⚠️ {model_label}-relaxed structures not found, expected " + f"⚠️ {model.label}-relaxed structures not found, expected " f"at {ml_relaxed_structs_path}" ) continue - # Create analysis file path by appending symprec to pred_file basename - geo_opt_filename = model.geo_opt_path.removesuffix(".json.gz") - geo_opt_csv_path = f"{geo_opt_filename}-{symprec_str}-{moyo_version}.csv.gz" - - if os.path.isfile(geo_opt_csv_path): - print(f"{prog_str} {model_label} already analyzed at {geo_opt_csv_path}") - continue - # Load model structures try: if ml_relaxed_structs_path.endswith((".json", ".json.gz", ".json.xz")): - df_ml_structs = pd.read_json(ml_relaxed_structs_path).set_index(Key.mat_id) + df_ml_structs = pd.read_json(ml_relaxed_structs_path) else: raise ValueError( "Relaxed structure analysis currently only supports pymatgen JSON, " f"got {ml_relaxed_structs_path}" ) except Exception as exc: - exc.add_note(f"{model_label=} {ml_relaxed_structs_path=}") + exc.add_note(f"{model.label=} {ml_relaxed_structs_path=}") raise + # try normalize material ID column or raise + if Key.mat_id in df_ml_structs: + df_ml_structs = df_ml_structs.set_index(Key.mat_id) + elif df_ml_structs.index[0].startswith("wbm-"): + df_ml_structs.index.name = Key.mat_id + df_ml_structs.reset_index().to_json(ml_relaxed_structs_path) + else: + raise ValueError(f"Could not infer ID column from {df_ml_structs.columns}") + if debug_mode: df_ml_structs = df_ml_structs.head(debug_mode) @@ -107,7 +153,7 @@ if struct_col not in df_ml_structs: struct_cols = [col for col in df_ml_structs if Key.structure in col] print( - f"⚠️ {struct_col=} not found in {model_label}-relaxed structures loaded " + f"⚠️ {struct_col=} not found in {model.label}-relaxed structures loaded " f"from {ml_relaxed_structs_path}. Did you mean one of {struct_cols}?" ) continue @@ -118,28 +164,43 @@ for mat_id, struct_dict in df_ml_structs[struct_col].items() } - # Analyze symmetry - model_analysis = analyze_symmetry( - model_structs, - pbar=dict(desc=f"{prog_str} Analyzing {model_label} symmetries"), - symprec=symprec, - ) + # Process each symprec value + for symprec in symprec_values: + symprec_str = f"symprec={symprec:.0e}".replace("e-0", "e-") + geo_opt_filename = model.geo_opt_path.removesuffix(".json.gz") + geo_opt_csv_path = f"{geo_opt_filename}-{symprec_str}-{moyo_version}.csv.gz" + + if os.path.isfile(geo_opt_csv_path): + print(f"{prog_str} {model.label} already analyzed at {geo_opt_csv_path}") + continue + + # Analyze symmetry for current symprec + model_analysis = analyze_symmetry( + model_structs, + pbar=dict(desc=f"{prog_str} {model.label} with {symprec=}"), + symprec=symprec, + ) - # Compare with DFT reference - df_ml_geo_analysis = pred_vs_ref_struct_symmetry( - model_analysis, - dft_analysis, - model_structs, - dft_structs, - pbar=dict(desc=f"{prog_str} Comparing DFT vs {model_label} symmetries"), - ) + # Compare with DFT reference + df_ml_geo_analysis = pred_vs_ref_struct_symmetry( + model_analysis, + dft_analysis_dict[symprec], + model_structs, + dft_structs, + pbar=dict( + desc=f"{prog_str} Comparing DFT vs {model.label} with {symprec=}" + ), + ) - # Save model results - df_ml_geo_analysis.to_csv(geo_opt_csv_path) - print(f"{prog_str} Completed {model_label} and saved results to {geo_opt_csv_path}") + # Save model results + df_ml_geo_analysis.to_csv(geo_opt_csv_path) + print( + f"{prog_str} Completed {model.label} {symprec=} and saved results " + f"to {geo_opt_csv_path}" + ) - # Calculate metrics and write to YAML - df_metrics = geo_opt.calc_geo_opt_metrics(df_ml_geo_analysis) - geo_opt.write_geo_opt_metrics_to_yaml(df_metrics, model, symprec) + # Calculate metrics and write to YAML + df_metrics = geo_opt.calc_geo_opt_metrics(df_ml_geo_analysis) + geo_opt.write_geo_opt_metrics_to_yaml(df_metrics, model, symprec) print("All models processed!") From 8074b3b1ae5ee846f217adc7e96036fed7762d28 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Feb 2025 13:53:39 +0000 Subject: [PATCH 11/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/analyze_geo_opt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/analyze_geo_opt.py b/scripts/analyze_geo_opt.py index dd18a0d1..afd13547 100644 --- a/scripts/analyze_geo_opt.py +++ b/scripts/analyze_geo_opt.py @@ -71,6 +71,7 @@ moyo_version = f"moyo={importlib.metadata.version('moyopy')}" moyo_version = "moyo=0.3.1" + # %% Load WBM reference structures (this takes a while) print("Loading WBM reference structures...") wbm_cse_path = DataFiles.wbm_computed_structure_entries.path @@ -83,6 +84,7 @@ if debug_mode: df_wbm_structs = df_wbm_structs.head(debug_mode) + # %% Process DFT structures for each symprec value dft_analysis_dict = {} for symprec in symprec_values: @@ -99,6 +101,7 @@ ) dft_analysis_dict[symprec].to_csv(dft_csv_path) + # %% Process each model sequentially for idx, model_name in enumerate(model_names): prog_str = f"{idx + 1}/{len(model_names)}:" From 4878f5677cfc4221d2cea5328badd15516b59a59 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Fri, 7 Feb 2025 08:59:12 -0500 Subject: [PATCH 12/17] add discovery metrics to grace-1L-oam.yml and grace-2L-oam.yml --- models/grace/grace-1L-oam.yml | 57 +++++++++++++++++++++++++++++++++++ models/grace/grace-2L-oam.yml | 57 +++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/models/grace/grace-1L-oam.yml b/models/grace/grace-1L-oam.yml index 51674759..109b3144 100644 --- a/models/grace/grace-1L-oam.yml +++ b/models/grace/grace-1L-oam.yml @@ -66,3 +66,60 @@ metrics: pred_file: models/grace/grace-1L-oam/2025-02-02-wbm-IS2RE.csv.gz pred_file_url: pred_col: e_form_per_atom_grace + full_test_set: + F1: 0.808 # fraction + DAF: 4.617 # dimensionless + Precision: 0.792 # fraction + Recall: 0.824 # fraction + Accuracy: 0.933 # fraction + TPR: 0.824 # fraction + FPR: 0.045 # fraction + TNR: 0.955 # fraction + FNR: 0.176 # fraction + TP: 36331.0 # count + FP: 9524.0 # count + TN: 203347.0 # count + FN: 7761.0 # count + MAE: 0.03 # eV/atom + RMSE: 0.073 # eV/atom + R2: 0.836 # dimensionless + missing_preds: 2 # count + missing_percent: 0.00% # fraction + most_stable_10k: + F1: 0.962 # fraction + DAF: 6.063 # dimensionless + Precision: 0.927 # fraction + Recall: 1.0 # fraction + Accuracy: 0.927 # fraction + TPR: 1.0 # fraction + FPR: 1.0 # fraction + TNR: 0.0 # fraction + FNR: 0.0 # fraction + TP: 9269.0 # count + FP: 731.0 # count + TN: 0.0 # count + FN: 0.0 # count + MAE: 0.035 # eV/atom + RMSE: 0.087 # eV/atom + R2: 0.843 # dimensionless + missing_preds: 0 # count + missing_percent: 0.00% # fraction + unique_prototypes: + F1: 0.824 # fraction + DAF: 5.255 # dimensionless + Precision: 0.803 # fraction + Recall: 0.846 # fraction + Accuracy: 0.944 # fraction + TPR: 0.846 # fraction + FPR: 0.038 # fraction + TNR: 0.962 # fraction + FNR: 0.154 # fraction + TP: 28244.0 # count + FP: 6917.0 # count + TN: 175197.0 # count + FN: 5130.0 # count + MAE: 0.031 # eV/atom + RMSE: 0.073 # eV/atom + R2: 0.842 # dimensionless + missing_preds: 0 # count + missing_percent: 0.00% # fraction diff --git a/models/grace/grace-2L-oam.yml b/models/grace/grace-2L-oam.yml index 87174ab1..a83e2471 100644 --- a/models/grace/grace-2L-oam.yml +++ b/models/grace/grace-2L-oam.yml @@ -66,3 +66,60 @@ metrics: pred_file: models/grace/grace-2L-oam/2025-01-28-wbm-IS2RE.csv.gz pred_file_url: pred_col: e_form_per_atom_grace + full_test_set: + F1: 0.861 # fraction + DAF: 5.046 # dimensionless + Precision: 0.866 # fraction + Recall: 0.856 # fraction + Accuracy: 0.953 # fraction + TPR: 0.856 # fraction + FPR: 0.027 # fraction + TNR: 0.973 # fraction + FNR: 0.144 # fraction + TP: 37754.0 # count + FP: 5846.0 # count + TN: 207025.0 # count + FN: 6338.0 # count + MAE: 0.023 # eV/atom + RMSE: 0.068 # eV/atom + R2: 0.856 # dimensionless + missing_preds: 2 # count + missing_percent: 0.00% # fraction + most_stable_10k: + F1: 0.985 # fraction + DAF: 6.352 # dimensionless + Precision: 0.971 # fraction + Recall: 1.0 # fraction + Accuracy: 0.971 # fraction + TPR: 1.0 # fraction + FPR: 1.0 # fraction + TNR: 0.0 # fraction + FNR: 0.0 # fraction + TP: 9711.0 # count + FP: 289.0 # count + TN: 0.0 # count + FN: 0.0 # count + MAE: 0.022 # eV/atom + RMSE: 0.081 # eV/atom + R2: 0.861 # dimensionless + missing_preds: 0 # count + missing_percent: 0.00% # fraction + unique_prototypes: + F1: 0.88 # fraction + DAF: 5.774 # dimensionless + Precision: 0.883 # fraction + Recall: 0.878 # fraction + Accuracy: 0.963 # fraction + TPR: 0.878 # fraction + FPR: 0.021 # fraction + TNR: 0.979 # fraction + FNR: 0.122 # fraction + TP: 29313.0 # count + FP: 3898.0 # count + TN: 178216.0 # count + FN: 4061.0 # count + MAE: 0.023 # eV/atom + RMSE: 0.068 # eV/atom + R2: 0.862 # dimensionless + missing_preds: 0 # count + missing_percent: 0.00% # fraction From 5db7a4b5515f9b178f4deb400bac57072eaf2b99 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Fri, 7 Feb 2025 11:18:16 -0500 Subject: [PATCH 13/17] calc both MP-corrected and uncorrected e_form_per_atom in models/grace/join_grace_preds.py --- models/grace/join_grace_preds.py | 87 +++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 23 deletions(-) diff --git a/models/grace/join_grace_preds.py b/models/grace/join_grace_preds.py index 896d940a..ab7b12a5 100644 --- a/models/grace/join_grace_preds.py +++ b/models/grace/join_grace_preds.py @@ -4,10 +4,13 @@ from glob import glob import pandas as pd +from pymatgen.core import Structure +from pymatgen.entries.compatibility import MaterialsProject2020Compatibility +from pymatgen.entries.computed_entries import ComputedStructureEntry from pymatviz.enums import Key from tqdm import tqdm -from matbench_discovery.data import as_dict_handler, df_wbm +from matbench_discovery.data import DataFiles, as_dict_handler, df_wbm from matbench_discovery.energy import calc_energy_from_e_refs, mp_elemental_ref_energies from matbench_discovery.enums import MbdKey, Task @@ -15,8 +18,8 @@ __date__ = "2025-02-06" -energy_column = "grace_energy" # or your actual column name. -e_form_grace_col = "e_form_per_atom_grace" # or your desired name +energy_col = "grace_energy" +e_form_grace_col = "e_form_per_atom_grace" struct_col = "grace_structure" @@ -42,7 +45,7 @@ def process_results(path: str) -> None: glob_pattern = f"{path}/production-*.json.gz" file_paths = glob(glob_pattern) - out_path = file_paths[0].rsplit("/", 1)[0] # Get directory from first file path. + out_dir = file_paths[0].rsplit("/", 1)[0] print(f"Found {len(file_paths):,} files for {glob_pattern = }") @@ -68,27 +71,67 @@ def process_results(path: str) -> None: ) df_grace = tot_df.set_index("material_id") # .drop(columns=[struct_col]) - df_grace[Key.formula] = df_wbm[Key.formula] - print("Calculating formation energies") - e_form_list = [] - for _, row in tqdm(df_grace.iterrows(), total=len(df_grace)): - e_form = calc_energy_from_e_refs( - row["formula"], + # Create ComputedStructureEntry objects with GRACE energies and structures + wbm_cse_path = DataFiles.wbm_computed_structure_entries.path + df_cse = pd.read_json(wbm_cse_path).set_index(Key.mat_id) + + df_cse[Key.computed_structure_entry] = [ + ComputedStructureEntry.from_dict(dct) + for dct in tqdm(df_cse[Key.computed_structure_entry], desc="Hydrate CSEs") + ] + + # %% transfer ML energies and relaxed structures WBM CSEs since MP2020 energy + # corrections applied below are structure-dependent (for oxides and sulfides) + cse: ComputedStructureEntry + for row in tqdm( + df_grace.itertuples(), total=len(df_grace), desc="ML energies to CSEs" + ): + mat_id, struct_dict, grace_energy, *_ = row + mlip_struct = Structure.from_dict(struct_dict) + cse = df_cse.loc[mat_id, Key.computed_structure_entry] + cse._energy = grace_energy # noqa: SLF001 cse._energy is the uncorrected energy + cse._structure = mlip_struct # noqa: SLF001 + df_grace.loc[mat_id, Key.computed_structure_entry] = cse + + # Apply MP2020 energy corrections + print("Applying MP2020 energy corrections") + processed = MaterialsProject2020Compatibility().process_entries( + df_grace[Key.computed_structure_entry], verbose=True, clean=True + ) + if len(processed) != len(df_grace): + raise ValueError( + f"not all entries processed: {len(processed)=} {len(df_grace)=}" + ) + + df_grace[e_form_grace_col] = [ + calc_energy_from_e_refs( + dict( + composition=row["formula"], + energy=row[Key.computed_structure_entry].energy, # use corrected energy + ), ref_energies=mp_elemental_ref_energies, - total_energy=row[energy_column], ) - e_form_list.append(e_form) + for _, row in tqdm(df_grace.iterrows(), total=len(df_grace)) + ] - df_grace[e_form_grace_col] = e_form_list + df_grace["e_form_per_atom_grace_uncorrected"] = [ + calc_energy_from_e_refs( + dict(energy=energy, composition=formula), + ref_energies=mp_elemental_ref_energies, + ) + for energy, formula in tqdm( + zip(df_grace[energy_col], df_grace["formula"]), + total=len(df_grace), + ) + ] # save relaxed structures - print("df_grace.columns=", df_grace.columns) df_grace.to_json( - f"{out_path}/{model_name}_{date}-wbm-IS2RE-FIRE.json.gz", + f"{out_dir}/{model_name}_{date}-wbm-IS2RE-FIRE.json.gz", default_handler=as_dict_handler, - ) # added model and date - df_grace = df_grace.drop(columns=[struct_col]) + ) + df_grace = df_grace.drop(columns=[struct_col, Key.computed_structure_entry]) df_wbm[[*df_grace]] = df_grace @@ -98,15 +141,13 @@ def process_results(path: str) -> None: print(f"{sum(bad_mask)=} is {sum(bad_mask) / len(df_wbm):.2%} of {n_preds:,}") df_grace = df_grace.round(4) - df_grace.select_dtypes("number").to_csv( - f"{out_path}/{model_name}_{date}.csv.gz" - ) # added model and date + df_grace.select_dtypes("number").to_csv(f"{out_dir}/{model_name}_{date}.csv.gz") df_grace.reset_index().to_json( - f"{out_path}/{model_name}_{date}.json.gz", default_handler=as_dict_handler - ) # added model and date + f"{out_dir}/{model_name}_{date}.json.gz", default_handler=as_dict_handler + ) df_bad = df_grace[bad_mask].copy() df_bad[MbdKey.e_form_dft] = df_wbm[MbdKey.e_form_dft] - df_bad.to_csv(f"{out_path}/{model_name}_{date}_bad.csv") # added model and date + df_bad.to_csv(f"{out_dir}/{model_name}_{date}_bad.csv") if __name__ == "__main__": From 9f1f0ce6518a01505ca09ffcae85fadfd0be3416 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Fri, 7 Feb 2025 11:18:40 -0500 Subject: [PATCH 14/17] contributing.md update recommended model artifact names --- contributing.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/contributing.md b/contributing.md index 2cf97a89..58dd1c89 100644 --- a/contributing.md +++ b/contributing.md @@ -18,24 +18,30 @@ There's also a [PyPI package](https://pypi.org/project/matbench-discovery) for f To submit a new model to this benchmark and add it to our leaderboard, please create a pull request to the [`main` branch][repo] that includes at least these 3 required files: -1. `--preds.csv.gz`: Your model's energy predictions for all ~250k WBM compounds as compressed CSV. The recommended way to create this file is with `pandas.DataFrame.to_csv("--wbm-IS2RE.csv.gz")`. See e.g. [`test_mace_discovery`](https://github.com/janosh/matbench-discovery/blob/-/models/mace/test_mace_discovery.py) for code that generates this file. +1. `--preds.csv.gz`: Your model's energy predictions for all ~250k WBM compounds as compressed CSV. The recommended way to create this file is with `pandas.DataFrame.to_csv("-wbm-IS2RE.csv.gz")`. See e.g. [`test_mace_discovery`](https://github.com/janosh/matbench-discovery/blob/-/models/mace/test_mace_discovery.py) for code that generates this file. ### Sharing Model Prediction Files - You should share your model's predictions through a cloud storage service (e.g. Figshare, Zenodo, Google Drive, Dropbox, AWS, etc.) and include the download links in your PR description. Your cloud storage directory should contain: + You should share your model's predictions through a cloud storage service (e.g. Figshare, Zenodo, Google Drive, Dropbox, AWS, etc.) and include the download links in your PR description. Your cloud storage directory should contain files with the following naming convention: `//-.{csv.gz|json.gz}`. For example, a in the case of MACE-MP-0, the file paths would be: - 1. `--wbm-geo-opt.json.gz`: The model's relaxed structures as compressed JSON containing: + - geometry optimization: `mace/mace-mp-0/2023-12-11-wbm-IS2RE-FIRE.json.gz` + - discovery: `mace/mace-mp-0/2023-12-11-wbm-IS2RE.csv.gz` + - phonons: `mace/mace-mp-0/2024-11-09-kappa-103-FIRE-dist=0.01-fmax=1e-4-symprec=1e-5.json.gz` + + The files should contain the following information: + + 1. `//-wbm-geo-opt-.json.gz`: The model's relaxed structures as compressed JSON containing: - Final relaxed structures (as ASE `Atoms` or pymatgen `Structures`) - Final energies (eV), forces (eV/Å), stress (eV/ų) and volume (ų) - Material IDs matching the WBM test set - 2. `--wbm-IS2RE.csv.gz`: A compressed CSV file with: + 2. `//-wbm-IS2RE.csv.gz`: A compressed CSV file with: - Material IDs matching the WBM test set - Final formation energies per atom (eV/atom) - 3. `--wbm-kappa.json.gz`: A compressed JSON file with: + 3. `//-kappa-103-.json.gz`: A compressed JSON file with: - Material IDs matching the WBM test set - Predicted thermal conductivity (κ) values (W/mK) @@ -78,7 +84,7 @@ To submit a new model to this benchmark and add it to our leaderboard, please cr df_traj.to_csv("trajectory.csv.gz") # Save final structure and trajectory data ``` -1. `test__discovery.(py|ipynb)`: The Python script that generated the WBM final energy predictions given the initial (unrelaxed) DFT structures. Ideally, this file should have comments explaining at a high level what the code is doing and how the model works so others can understand and reproduce your results. If the model deployed on this benchmark was trained specifically for this purpose (i.e. if you wrote any training/fine-tuning code while preparing your PR), please also include it as `train_.(py|ipynb)`. +1. `test__discovery.py`: The Python script that generated the WBM final energy predictions given the initial (unrelaxed) DFT structures. Ideally, this file should have comments explaining at a high level what the code is doing and how the model works so others can understand and reproduce your results. If the model deployed on this benchmark was trained specifically for this purpose (i.e. if you wrote any training/fine-tuning code while preparing your PR), please also include it as `train_.py`. 1. ``: A file to record all relevant metadata of your algorithm like model name and version, authors, package requirements, links to publications, notes, etc. Here's a template: ```yml From 9b4373ff5f4098b5e173584afa373c5f992870f4 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Fri, 7 Feb 2025 12:12:37 -0500 Subject: [PATCH 15/17] add parallel processing to analyze_geo_opt.py and improve error handling - uses ProcessPoolExecutor - add CLI option for specifying number of worker processes - improve logging and progress tracking --- matbench_discovery/metrics/geo_opt.py | 27 ++- matbench_discovery/structure.py | 22 ++- scripts/analyze_geo_opt.py | 269 +++++++++++++++----------- 3 files changed, 183 insertions(+), 135 deletions(-) diff --git a/matbench_discovery/metrics/geo_opt.py b/matbench_discovery/metrics/geo_opt.py index e6d74fd4..4cd54ba8 100644 --- a/matbench_discovery/metrics/geo_opt.py +++ b/matbench_discovery/metrics/geo_opt.py @@ -69,7 +69,7 @@ def write_geo_opt_metrics_to_yaml( round_trip_yaml.dump(model_metadata, file) -def calc_geo_opt_metrics(df_model_analysis: pd.DataFrame) -> pd.DataFrame: +def calc_geo_opt_metrics(df_model_analysis: pd.DataFrame) -> dict[str, float]: """Calculate geometry optimization metrics for a single model. Args: @@ -81,14 +81,13 @@ def calc_geo_opt_metrics(df_model_analysis: pd.DataFrame) -> pd.DataFrame: model_name (str): Name of the model being analyzed. Returns: - pd.DataFrame: DataFrame with geometry optimization metrics. - Shape = (1, n_metrics). Columns include: - - structure_rmsd_vs_dft: Mean RMSD between predicted and DFT structures - - n_sym_ops_mae: Mean absolute error in number of symmetry operations - - symmetry_decrease: Fraction of structures with decreased symmetry - - symmetry_match: Fraction of structures with matching symmetry - - symmetry_increase: Fraction of structures with increased symmetry - - n_structs: Number of structures evaluated + dict[str, float]: Geometry optimization metrics with keys: + - structure_rmsd_vs_dft: Mean RMSD between predicted and DFT structures + - n_sym_ops_mae: Mean absolute error in number of symmetry operations + - symmetry_decrease: Fraction of structures with decreased symmetry + - symmetry_match: Fraction of structures with matching symmetry + - symmetry_increase: Fraction of structures with increased symmetry + - n_structs: Number of structures evaluated """ # Get relevant columns spg_diff = df_model_analysis[MbdKey.spg_num_diff] @@ -96,7 +95,7 @@ def calc_geo_opt_metrics(df_model_analysis: pd.DataFrame) -> pd.DataFrame: rmsd = df_model_analysis[MbdKey.structure_rmsd_vs_dft] # Count total number of structures (excluding NaN values) - total = len(spg_diff.dropna()) + n_structs = len(spg_diff.dropna()) # Calculate RMSD and MAE metrics mean_rmsd = rmsd.mean() @@ -112,8 +111,8 @@ def calc_geo_opt_metrics(df_model_analysis: pd.DataFrame) -> pd.DataFrame: return { str(MbdKey.structure_rmsd_vs_dft): float(mean_rmsd), str(Key.n_sym_ops_mae): float(sym_ops_mae), - str(Key.symmetry_decrease): float(sym_decreased.sum() / total), - str(Key.symmetry_match): float(sym_matched.sum() / total), - str(Key.symmetry_increase): float(sym_increased.sum() / total), - str(Key.n_structures): total, + str(Key.symmetry_decrease): float(sym_decreased.sum() / n_structs), + str(Key.symmetry_match): float(sym_matched.sum() / n_structs), + str(Key.symmetry_increase): float(sym_increased.sum() / n_structs), + str(Key.n_structures): n_structs, } diff --git a/matbench_discovery/structure.py b/matbench_discovery/structure.py index 12323bda..23a9f4f0 100644 --- a/matbench_discovery/structure.py +++ b/matbench_discovery/structure.py @@ -43,7 +43,7 @@ def perturb_structure(struct: Structure, gamma: float = 1.5) -> Structure: def analyze_symmetry( structures: dict[str, Structure], *, - pbar: bool | dict[str, str] = True, + pbar: bool | dict[str, str | float | bool] = True, symprec: float = 1e-2, angle_tolerance: float | None = None, ) -> pd.DataFrame: @@ -52,8 +52,8 @@ def analyze_symmetry( Args: structures (dict[str, Structure | Atoms]): Map of material IDs to pymatgen Structures or ASE Atoms objects - pbar (bool | dict[str, str], optional): Whether to show progress bar. - Defaults to True. + pbar (bool | dict[str, str | float | bool], optional): Whether to show progress + bar. Defaults to True. symprec (float, optional): Symmetry precision of moyopy. Defaults to 1e-2. angle_tolerance (float, optional): Angle tolerance of moyopy (in radians unlike spglib which uses degrees!). Defaults to None. @@ -121,7 +121,7 @@ def pred_vs_ref_struct_symmetry( pred_structs: dict[str, Structure], ref_structs: dict[str, Structure], *, - pbar: bool | dict[str, str] = True, + pbar: bool | dict[str, str | float | bool] = True, ) -> pd.DataFrame: """Get RMSD and compare symmetry between ML and DFT reference structures. @@ -135,12 +135,17 @@ def pred_vs_ref_struct_symmetry( analyze_symmetry. pred_structs (dict[str, Structure]): Map material IDs to ML-relaxed structures ref_structs (dict[str, Structure]): Map material IDs to reference structures - pbar (bool | dict[str, str], optional): Whether to show progress bar. - Defaults to True. + pbar (bool | dict[str, str | float | bool], optional): Whether to show progress + bar. Defaults to True. Returns: pd.DataFrame: with added columns for symmetry differences """ + if df_sym_ref.index.name != Key.mat_id: + raise ValueError(f"{df_sym_ref.index.name=} must be {Key.mat_id!s}") + if df_sym_pred.index.name != Key.mat_id: + raise ValueError(f"{df_sym_pred.index.name=} must be {Key.mat_id!s}") + df_result = df_sym_pred.copy() # Calculate differences @@ -150,7 +155,10 @@ def pred_vs_ref_struct_symmetry( ) structure_matcher = StructureMatcher() - shared_ids = set(pred_structs) & set(ref_structs) + ref_ids, pred_ids = set(ref_structs), set(pred_structs) + shared_ids = ref_ids & pred_ids + if len(shared_ids) == 0: + raise ValueError(f"No shared IDs between:\n{pred_ids=}\n{ref_ids=}") # Initialize RMSD column df_result[MbdKey.structure_rmsd_vs_dft] = None diff --git a/scripts/analyze_geo_opt.py b/scripts/analyze_geo_opt.py index afd13547..a459465d 100644 --- a/scripts/analyze_geo_opt.py +++ b/scripts/analyze_geo_opt.py @@ -7,14 +7,18 @@ Example usage: python scripts/analyze_geo_opt.py --models mace_mp_0 m3gnet --symprec 1e-2 1e-5 python scripts/analyze_geo_opt.py --debug 10 # only analyze first 10 structures + python scripts/analyze_geo_opt.py --workers 4 # use 4 CPU cores """ # %% import argparse import importlib import importlib.metadata +import itertools +import multiprocessing as mp import os from collections.abc import Sequence +from concurrent.futures import ProcessPoolExecutor from typing import Any, Final import pandas as pd @@ -27,84 +31,17 @@ from matbench_discovery.models import MODEL_METADATA from matbench_discovery.structure import analyze_symmetry, pred_vs_ref_struct_symmetry -parser = argparse.ArgumentParser() -parser.add_argument( - "--models", - nargs="+", - default=["all"], - help="Model names to analyze. Use 'all' to analyze all models.", -) -parser.add_argument( - "--symprec", - nargs="+", - type=float, - default=[1e-2, 1e-5], - help="Symmetry precision values to analyze.", -) -parser.add_argument( - "--debug", - type=int, - default=0, - help="If > 0, only analyze this many structures.", -) -args = parser.parse_args() - -# activate debug mode by setting to any number > 0, only that many structures will be -# analyzed -debug_mode: Final[int] = args.debug -# List of symprec values to analyze -symprec_values: Final[Sequence[float]] = args.symprec - -# Get list of models to analyze -if "all" in args.models: - model_names = [model.name for model in Model] -else: - model_names = args.models - # Validate model names - valid_models = {model.name for model in Model} - for model_name in model_names: - if model_name not in valid_models: - raise ValueError( - f"Invalid model name: {model_name}. Valid models are: {valid_models}" - ) - -moyo_version = f"moyo={importlib.metadata.version('moyopy')}" -moyo_version = "moyo=0.3.1" - - -# %% Load WBM reference structures (this takes a while) -print("Loading WBM reference structures...") -wbm_cse_path = DataFiles.wbm_computed_structure_entries.path -df_wbm_structs: pd.DataFrame = pd.read_json(wbm_cse_path).set_index(Key.mat_id) -dft_structs: dict[str, Structure] = { - mat_id: Structure.from_dict(cse[Key.structure]) - for mat_id, cse in df_wbm_structs[Key.computed_structure_entry].items() -} - -if debug_mode: - df_wbm_structs = df_wbm_structs.head(debug_mode) - - -# %% Process DFT structures for each symprec value -dft_analysis_dict = {} -for symprec in symprec_values: - symprec_str = f"symprec={symprec:.0e}".replace("e-0", "e-") - dft_csv_path = f"{ROOT}/data/wbm/dft-geo-opt-{symprec_str}-{moyo_version}.csv.gz" - - if os.path.isfile(dft_csv_path): - dft_analysis_dict[symprec] = pd.read_csv(dft_csv_path) - else: - dft_analysis_dict[symprec] = analyze_symmetry( - dft_structs, - pbar=dict(desc=f"Getting DFT symmetries {symprec=}"), - symprec=symprec, - ) - dft_analysis_dict[symprec].to_csv(dft_csv_path) - -# %% Process each model sequentially -for idx, model_name in enumerate(model_names): - prog_str = f"{idx + 1}/{len(model_names)}:" +def analyze_model_symprec( + model_name: str, + symprec: float, + df_dft_analysis: pd.DataFrame, + dft_structs: dict[str, Structure], + debug_mode: int = 0, + moyo_version: str = "moyo=0.3.1", + pbar_pos: int = 0, # tqdm progress bar position +) -> None: + """Analyze a single model for a single symprec value.""" model = Model[model_name] model_metadata = MODEL_METADATA[model.label] @@ -114,18 +51,19 @@ # skip models that don't support geometry optimization if geo_opt_metrics in ("not applicable", "not available"): - continue + print(f"⚠️ {model.label} does not support geometry optimization") + return if not model.geo_opt_path: print(f"⚠️ {model.label} has no relaxed structures file") - continue + return if not os.path.isfile(ml_relaxed_structs_path := model.geo_opt_path): print( f"⚠️ {model.label}-relaxed structures not found, expected " f"at {ml_relaxed_structs_path}" ) - continue + return # Load model structures try: @@ -159,7 +97,7 @@ f"⚠️ {struct_col=} not found in {model.label}-relaxed structures loaded " f"from {ml_relaxed_structs_path}. Did you mean one of {struct_cols}?" ) - continue + return # Convert structures model_structs = { @@ -167,43 +105,146 @@ for mat_id, struct_dict in df_ml_structs[struct_col].items() } - # Process each symprec value + symprec_str = f"symprec={symprec:.0e}".replace("e-0", "e-") + geo_opt_filename = model.geo_opt_path.removesuffix(".json.gz") + geo_opt_csv_path = f"{geo_opt_filename}-{symprec_str}-{moyo_version}.csv.gz" + + if os.path.isfile(geo_opt_csv_path): + print(f"{model.label} already analyzed at {geo_opt_csv_path}") + return + + # Analyze symmetry for current symprec + pbar_desc = f"Process {pbar_pos}: Analyzing {model.label} for {symprec=}" + df_model_analysis = analyze_symmetry( + model_structs, + pbar=dict(desc=pbar_desc, position=pbar_pos, leave=True), + symprec=symprec, + ) + + # Compare with DFT reference + pbar_desc = f"Process {pbar_pos}:Comparing DFT vs {model.label} for {symprec=}" + # break here + df_ml_geo_analysis = pred_vs_ref_struct_symmetry( + df_model_analysis, + df_dft_analysis, + model_structs, + dft_structs, + pbar=dict(desc=pbar_desc, position=pbar_pos, leave=True), + ) + + # Save model results + df_ml_geo_analysis.to_csv(geo_opt_csv_path) + print(f"Completed {model.label} {symprec=} and saved results to {geo_opt_csv_path}") + + # Calculate metrics and write to YAML + df_metrics = geo_opt.calc_geo_opt_metrics(df_ml_geo_analysis) + print(f"\nCalculated metrics: {df_metrics}") # Debug print + geo_opt.write_geo_opt_metrics_to_yaml(df_metrics, model, symprec) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--models", + nargs="+", + default=["all"], + help="Model names to analyze. Use 'all' to analyze all models.", + ) + parser.add_argument( + "--symprec", + nargs="+", + type=float, + default=[1e-2, 1e-5], + help="Symmetry precision values to analyze.", + ) + parser.add_argument( + "--debug", + type=int, + default=0, + help="If > 0, only analyze this many structures.", + ) + parser.add_argument( + "--workers", + type=int, + default=max(1, mp.cpu_count() - 1), + help="Number of processes to use. Defaults to number of model-symprec combos.", + ) + args = parser.parse_args() + + # set to > 0 to activate debug mode, only that many structures will be analyzed + debug_mode: Final[int] = args.debug + # List of symprec values to analyze + symprec_values: Final[Sequence[float]] = args.symprec + + # Get list of models to analyze + if "all" in args.models: + model_names = [model.name for model in Model] + else: + model_names = args.models + # Validate model names + valid_models = {model.name for model in Model} + for model_name in model_names: + if model_name not in valid_models: + raise ValueError(f"Invalid {model_name=}, valid models: {valid_models}") + + moyo_version = f"moyo={importlib.metadata.version('moyopy')}" + + # %% Load WBM reference structures (this takes a while) + print("Loading WBM reference structures...") + wbm_cse_path = DataFiles.wbm_computed_structure_entries.path + df_wbm_structs: pd.DataFrame = pd.read_json(wbm_cse_path).set_index(Key.mat_id) + + if debug_mode: + df_wbm_structs = df_wbm_structs.head(debug_mode) + + dft_structs: dict[str, Structure] = { + mat_id: Structure.from_dict(cse[Key.structure]) + for mat_id, cse in df_wbm_structs[Key.computed_structure_entry].items() + } + + # %% Process DFT structures for each symprec value + dft_analysis_dict: dict[float, pd.DataFrame] = {} for symprec in symprec_values: symprec_str = f"symprec={symprec:.0e}".replace("e-0", "e-") - geo_opt_filename = model.geo_opt_path.removesuffix(".json.gz") - geo_opt_csv_path = f"{geo_opt_filename}-{symprec_str}-{moyo_version}.csv.gz" - - if os.path.isfile(geo_opt_csv_path): - print(f"{prog_str} {model.label} already analyzed at {geo_opt_csv_path}") - continue - - # Analyze symmetry for current symprec - model_analysis = analyze_symmetry( - model_structs, - pbar=dict(desc=f"{prog_str} {model.label} with {symprec=}"), - symprec=symprec, + dft_csv_path = ( + f"{ROOT}/data/wbm/dft-geo-opt-{symprec_str}-{moyo_version}.csv.gz" ) - # Compare with DFT reference - df_ml_geo_analysis = pred_vs_ref_struct_symmetry( - model_analysis, - dft_analysis_dict[symprec], - model_structs, - dft_structs, - pbar=dict( - desc=f"{prog_str} Comparing DFT vs {model.label} with {symprec=}" - ), - ) + if os.path.isfile(dft_csv_path): + dft_analysis_dict[symprec] = pd.read_csv(dft_csv_path).set_index(Key.mat_id) + else: + dft_analysis_dict[symprec] = analyze_symmetry( + dft_structs, + pbar=dict(desc=f"Getting DFT symmetries {symprec=}"), + symprec=symprec, + ) + dft_analysis_dict[symprec].to_csv(dft_csv_path) - # Save model results - df_ml_geo_analysis.to_csv(geo_opt_csv_path) - print( - f"{prog_str} Completed {model.label} {symprec=} and saved results " - f"to {geo_opt_csv_path}" - ) + # Create list of all model-symprec combinations + tasks = list(itertools.product(model_names, symprec_values)) + n_workers = min(len(tasks), args.workers) - # Calculate metrics and write to YAML - df_metrics = geo_opt.calc_geo_opt_metrics(df_ml_geo_analysis) - geo_opt.write_geo_opt_metrics_to_yaml(df_metrics, model, symprec) + # %% Process model-symprec combinations in parallel + print( + f"\nAnalyzing {len(tasks)} model-symprec combos using {n_workers} processes..." + ) + + with ProcessPoolExecutor(max_workers=n_workers) as executor: + futures = [ + executor.submit( + analyze_model_symprec, + model_name, + symprec, + dft_analysis_dict[symprec], + dft_structs, + debug_mode, + moyo_version, + pbar_pos=idx, # assign unique position to each task's progress bar + ) + for idx, (model_name, symprec) in enumerate(tasks) + ] + # Wait for all tasks to complete + for future in futures: + future.result() # This will raise any exceptions that occurred -print("All models processed!") + print(f"\nAll {len(tasks)} model-symprec combinations processed!") From 5ead8572e87aeffc2037942132d47ef08001aa1e Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Fri, 7 Feb 2025 12:40:21 -0500 Subject: [PATCH 16/17] add geo_opt metrics to GRACE Omat L1+L2 model YAMLs - add figshare pred_file_url for geo_opt, discovery and kappa_103 in grace-1L-oam.yml and grace-2L-oam.yml --- models/grace/grace-1L-oam.yml | 20 +++++++++++++++++--- models/grace/grace-2L-oam.yml | 20 +++++++++++++++++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/models/grace/grace-1L-oam.yml b/models/grace/grace-1L-oam.yml index 109b3144..50acac3c 100644 --- a/models/grace/grace-1L-oam.yml +++ b/models/grace/grace-1L-oam.yml @@ -57,14 +57,28 @@ metrics: kappa_103: κ_SRME: 0.516 # https://github.com/MPA2suite/k_SRME/pull/20 pred_file: models/grace/grace-1L-oam/2025-02-02-kappa-103-FIRE-dist=0.01-fmax=1e-4-symprec=1e-5.json.gz - pred_file_url: + pred_file_url: https://figshare.com/ndownloader/files/52204910 geo_opt: pred_file: models/grace/grace-1L-oam/2025-02-02-wbm-geo-opt.json.gz - pred_file_url: + pred_file_url: https://figshare.com/ndownloader/files/52204904 struct_col: grace_structure + symprec=1e-5: + rmsd: 0.0139 # Å + n_sym_ops_mae: 1.8528 # unitless + symmetry_decrease: 0.0329 # fraction + symmetry_match: 0.7336 # fraction + symmetry_increase: 0.2292 # fraction + n_structures: 256963 # count + symprec=1e-2: + rmsd: 0.0139 # Å + n_sym_ops_mae: 1.8157 # unitless + symmetry_decrease: 0.0576 # fraction + symmetry_match: 0.814 # fraction + symmetry_increase: 0.1216 # fraction + n_structures: 256963 # count discovery: pred_file: models/grace/grace-1L-oam/2025-02-02-wbm-IS2RE.csv.gz - pred_file_url: + pred_file_url: https://figshare.com/ndownloader/files/52204898 pred_col: e_form_per_atom_grace full_test_set: F1: 0.808 # fraction diff --git a/models/grace/grace-2L-oam.yml b/models/grace/grace-2L-oam.yml index a83e2471..9a04be64 100644 --- a/models/grace/grace-2L-oam.yml +++ b/models/grace/grace-2L-oam.yml @@ -57,14 +57,28 @@ metrics: kappa_103: κ_SRME: 0.294 # https://github.com/MPA2suite/k_SRME/pull/20 pred_file: models/grace/grace-2L-oam/2025-01-28-kappa-103-FIRE-dist=0.01-fmax=1e-4-symprec=1e-5.json.gz - pred_file_url: + pred_file_url: https://figshare.com/ndownloader/files/52204916 geo_opt: pred_file: models/grace/grace-2L-oam/2025-01-28-wbm-geo-opt.json.gz - pred_file_url: + pred_file_url: https://figshare.com/ndownloader/files/52204907 struct_col: grace_structure + symprec=1e-5: + rmsd: 0.0121 # Å + n_sym_ops_mae: 1.8673 # unitless + symmetry_decrease: 0.0325 # fraction + symmetry_match: 0.733 # fraction + symmetry_increase: 0.2301 # fraction + n_structures: 256963 # count + symprec=1e-2: + rmsd: 0.0121 # Å + n_sym_ops_mae: 1.7962 # unitless + symmetry_decrease: 0.0563 # fraction + symmetry_match: 0.8159 # fraction + symmetry_increase: 0.1209 # fraction + n_structures: 256963 # count discovery: pred_file: models/grace/grace-2L-oam/2025-01-28-wbm-IS2RE.csv.gz - pred_file_url: + pred_file_url: https://figshare.com/ndownloader/files/52204901 pred_col: e_form_per_atom_grace full_test_set: F1: 0.861 # fraction From 5d57787126a16ef2226ec561ebd17db3b4793340 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Fri, 7 Feb 2025 12:45:28 -0500 Subject: [PATCH 17/17] fix test_pred_vs_ref_struct_symmetry_with_structures --- site/package.json | 20 ++++++++++---------- tests/test_structure.py | 4 ++++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/site/package.json b/site/package.json index 4ab5b2da..91e95080 100644 --- a/site/package.json +++ b/site/package.json @@ -20,24 +20,24 @@ "devDependencies": { "@iconify/svelte": "^4.2.0", "@rollup/plugin-yaml": "^4.1.2", - "@stylistic/eslint-plugin": "^2.12.1", + "@stylistic/eslint-plugin": "^3.0.1", "@sveltejs/adapter-static": "^3.0.8", - "@sveltejs/kit": "^2.15.2", + "@sveltejs/kit": "^2.17.1", "@sveltejs/vite-plugin-svelte": "^3.1.2", "d3-array": "^3.2.4", "d3-scale": "^4.0.2", "d3-scale-chromatic": "^3.1.0", - "elementari": "^0.2.5", - "eslint": "^9.18.0", + "elementari": "^0.2.6", + "eslint": "^9.19.0", "eslint-plugin-svelte": "^2.46.1", "hastscript": "^9.0.0", "js-yaml": "^4.1.0", "jsdom": "^26.0.0", - "json-schema-to-typescript": "^15.0.3", - "katex": "^0.16.19", + "json-schema-to-typescript": "^15.0.4", + "katex": "^0.16.21", "mdsvex": "^0.12.3", "prettier": "^3.4.2", - "prettier-plugin-svelte": "^3.3.2", + "prettier-plugin-svelte": "^3.3.3", "rehype-autolink-headings": "^7.1.0", "rehype-katex-svelte": "^1.2.0", "rehype-slug": "^6.0.0", @@ -46,7 +46,7 @@ "remark-parse": "^11.0.0", "remark-rehype": "^11.1.1", "svelte": "^4.2.19", - "svelte-check": "^4.1.3", + "svelte-check": "^4.1.4", "svelte-multiselect": "11.0.0-rc.1", "svelte-preprocess": "^6.0.3", "svelte-toc": "^0.5.9", @@ -54,10 +54,10 @@ "svelte2tsx": "^0.7.34", "tslib": "^2.8.1", "typescript": "5.7.3", - "typescript-eslint": "^8.19.1", + "typescript-eslint": "^8.23.0", "unified": "^11.0.5", "vite": "^5.4.11", - "vitest": "^2.1.8" + "vitest": "^3.0.5" }, "prettier": { "semi": false, diff --git a/tests/test_structure.py b/tests/test_structure.py index 1efa688d..aa6907d1 100644 --- a/tests/test_structure.py +++ b/tests/test_structure.py @@ -120,6 +120,10 @@ def test_pred_vs_ref_struct_symmetry_with_structures( df_ml = pd.concat([df_ml, df_ml_sym], axis=1) df_dft = pd.concat([df_dft, df_dft_sym], axis=1) + # Set index name to material_id for both DataFrames + df_ml.index.name = Key.mat_id + df_dft.index.name = Key.mat_id + df_ml_sym[Key.spg_num] - df_dft_sym[Key.spg_num] # must use same keys for both structures to match them in RMSD calculation