From 8f8bf5dd9eff8e66d491ac9918acc48267fc05bf Mon Sep 17 00:00:00 2001 From: Marcus Tantakoun Date: Fri, 13 Dec 2024 23:17:17 -0500 Subject: [PATCH 01/10] added proper raise ImportError messages for LLM usage --- l2p/llm_builder.py | 32 +++++++++++++++++++++++++++++--- requirements.txt | 1 - setup.py | 7 +------ 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/l2p/llm_builder.py b/l2p/llm_builder.py index b5e536d..23da82b 100644 --- a/l2p/llm_builder.py +++ b/l2p/llm_builder.py @@ -87,12 +87,19 @@ def __init__(self, model: str, api_key: str | None = None, client=None, stop=Non # attempt to import necessary OPENAI modules try: from openai import OpenAI - import tiktoken except ImportError: raise ImportError( "The 'openai' library is required for OPENAI but is not installed. " "Install it using: `pip install openai`." ) + + try: + import tiktoken + except ImportError: + raise ImportError( + "The 'tiktoken' library is required for token processing but is not installed. " + "Install it using: `pip install tiktoken`." + ) # call the parent class constructor to handle model and api_key super().__init__(model, api_key) @@ -221,15 +228,34 @@ def __init__(self, model_path: str, max_tokens=4e3, temperature=0.01, top_p=0.9) self.out_tokens = 0 def _load_transformers(self): + + # attempt to import the `transformers` library try: import transformers + except ImportError: + raise ImportError( + "The 'transformers' library is required for HUGGING_FACE but is not installed. " + "Install it using: `pip install transformers`." + ) + + # attempt to import `AutoTokenizer` from `transformers` + try: from transformers import AutoTokenizer + except ImportError: + raise ImportError( + "The 'transformers.AutoTokenizer' module is required but is not installed properly. " + "Ensure that the 'transformers' library is installed correctly." + ) + + # attempt to import the `torch` library + try: import torch except ImportError: raise ImportError( - "The 'transformers' and 'torch' libraries are required for HUGGING_FACE but are not installed. " - "Install them using: `pip install transformers torch`." + "The 'torch' library is required for HUGGING_FACE but is not installed. " + "Install it using: `pip install torch`." ) + try: # Check if the model_path is valid by trying to load it self.model = transformers.pipeline( diff --git a/requirements.txt b/requirements.txt index 54f3416..fc03f4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -tiktoken retry pddl typing_extensions \ No newline at end of file diff --git a/setup.py b/setup.py index 5eb1150..338c097 100644 --- a/setup.py +++ b/setup.py @@ -13,14 +13,9 @@ author="Marcus Tantakoun", author_email="mtantakoun@gmail.com", install_requires=[ - "openai", - "tiktoken", "retry", "pddl", - "typing_extensions", - "transformers>=4.43.1", - "torch>=2.2", - "accelerate>=0.26.0", + "typing_extensions" ], license="MIT", url="https://github.com/AI-Planning/l2p", From cb89c2d43a337d03a999bb86d4a9acdc89d022a9 Mon Sep 17 00:00:00 2001 From: Marcus Tantakoun Date: Sat, 14 Dec 2024 01:54:40 -0500 Subject: [PATCH 02/10] cleaned up code using black library --- docs/conf.py | 46 +- l2p/__init__.py | 2 +- l2p/domain_builder.py | 548 +++++++------ l2p/feedback_builder.py | 761 +++++++++++------- l2p/llm_builder.py | 235 +++--- l2p/prompt_builder.py | 47 +- l2p/task_builder.py | 355 ++++---- l2p/utils/pddl_parser.py | 323 +++++--- l2p/utils/pddl_planner.py | 25 +- l2p/utils/pddl_types.py | 21 +- l2p/utils/pddl_validator.py | 397 +++++---- l2p/utils/utils.py | 11 +- paper_reconstructions/llm+dm/llm+dm.py | 185 +++-- .../llm+dm/prompts/pddl_prompt.txt | 2 + .../llm+dm/results/domain.pddl | 80 +- paper_reconstructions/llm+p/llm+p.py | 22 +- paper_reconstructions/llm+p/prompts/role.txt | 6 +- paper_reconstructions/nl2plan/nl2plan.py | 303 ++++--- .../nl2plan/prompts/blocksworld.txt | 6 + .../nl2plan/prompts/blocksworld_p1.txt | 1 + paper_reconstructions/proc2pddl/proc2pddl.py | 102 ++- setup.py | 6 +- tests/__init__.py | 2 +- tests/mock_llm.py | 2 +- tests/parse.py | 15 +- tests/test_domain_builder.py | 97 ++- tests/test_task_builder.py | 28 +- tests/usage/demonstration.py | 182 +++-- tests/usage/usage.py | 100 ++- 29 files changed, 2282 insertions(+), 1628 deletions(-) create mode 100644 paper_reconstructions/nl2plan/prompts/blocksworld.txt create mode 100644 paper_reconstructions/nl2plan/prompts/blocksworld_p1.txt diff --git a/docs/conf.py b/docs/conf.py index 066dfaa..03201c5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,30 +8,33 @@ import os import sys -sys.path.insert(0, os.path.abspath('..')) -project = 'L2P' -copyright = '2024, Marcus Tantakoun' -author = 'Marcus Tantakoun' -release = '0.1.0' +sys.path.insert(0, os.path.abspath("..")) + +project = "L2P" +copyright = "2024, Marcus Tantakoun" +author = "Marcus Tantakoun" +release = "0.1.0" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = ['sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', - 'sphinx.ext.napoleon', - 'myst_parser'] +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.mathjax", + "sphinx.ext.ifconfig", + "sphinx.ext.viewcode", + "sphinx.ext.githubpages", + "sphinx.ext.napoleon", + "myst_parser", +] myst_enable_extensions = [ - 'colon_fence', # Enables Markdown for code block with language + "colon_fence", # Enables Markdown for code block with language ] # Napoleon settings @@ -47,14 +50,13 @@ napoleon_use_param = True napoleon_use_rtype = True -templates_path = ['_templates'] -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_theme = 'furo' +html_theme = "furo" html_title = "l2p" -html_static_path = ['_static'] +html_static_path = ["_static"] diff --git a/l2p/__init__.py b/l2p/__init__.py index 1d77505..e6fb11d 100644 --- a/l2p/__init__.py +++ b/l2p/__init__.py @@ -3,4 +3,4 @@ from .feedback_builder import * from .prompt_builder import * from .llm_builder import * -from .utils import * \ No newline at end of file +from .utils import * diff --git a/l2p/domain_builder.py b/l2p/domain_builder.py index 936d170..e7c2772 100644 --- a/l2p/domain_builder.py +++ b/l2p/domain_builder.py @@ -8,15 +8,16 @@ from collections import OrderedDict import re, time + class DomainBuilder: def __init__( - self, - types: dict[str,str]=None, - type_hierarchy: dict[str,str]=None, - predicates: list[Predicate]=None, - nl_actions: dict[str,str]=None, - pddl_actions: list[Action]=None - ): + self, + types: dict[str, str] = None, + type_hierarchy: dict[str, str] = None, + predicates: list[Predicate] = None, + nl_actions: dict[str, str] = None, + pddl_actions: list[Action] = None, + ): """ Initializes a domain builder object @@ -34,15 +35,16 @@ def __init__( self.pddl_actions = pddl_actions """Extract functions""" + @require_llm def extract_type( - self, - model: LLM, - domain_desc: str, + self, + model: LLM, + domain_desc: str, prompt_template: str, - types: dict[str,str]=None, - max_retries: int=3 - ) -> tuple[dict[str,str], str]: + types: dict[str, str] = None, + max_retries: int = 3, + ) -> tuple[dict[str, str], str]: """ Extracts types with domain given @@ -60,39 +62,41 @@ def extract_type( # replace prompt placeholders types_str = format_dict(types) if types else "No types provided." - - prompt_template = prompt_template.replace('{domain_desc}', domain_desc) - prompt_template = prompt_template.replace('{types}', types_str) - + + prompt_template = prompt_template.replace("{domain_desc}", domain_desc) + prompt_template = prompt_template.replace("{types}", types_str) + # iterate through attempts in case of extraction failure for attempt in range(max_retries): try: model.reset_tokens() - llm_response = model.query(prompt=prompt_template) # prompt model + llm_response = model.query(prompt=prompt_template) # prompt model # extract respective types from response types = convert_to_dict(llm_response=llm_response) - + if types is not None: return types, llm_response print(f"Attempt {attempt + 1}/{max_retries}: Failed to extract types.") except Exception as e: - print(f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}...") - time.sleep(1) # add a delay before retrying - + print( + f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + ) + time.sleep(1) # add a delay before retrying + raise RuntimeError("Max retries exceeded. Failed to extract types.") - + @require_llm def extract_type_hierarchy( - self, - model: LLM, + self, + model: LLM, domain_desc: str, - prompt_template: str, - types: dict[str,str]=None, - max_retries: int=3 - ) -> tuple[dict[str,str], str]: + prompt_template: str, + types: dict[str, str] = None, + max_retries: int = 3, + ) -> tuple[dict[str, str], str]: """ Extracts type hierarchy from types list and domain given @@ -110,10 +114,10 @@ def extract_type_hierarchy( # replace prompt placeholders types_str = format_dict(types) if types else "No types provided." - - prompt_template = prompt_template.replace('{domain_desc}', domain_desc) - prompt_template = prompt_template.replace('{types}', types_str) - + + prompt_template = prompt_template.replace("{domain_desc}", domain_desc) + prompt_template = prompt_template.replace("{types}", types_str) + # iterate through attempts in case of extraction failure for attempt in range(max_retries): try: @@ -124,25 +128,27 @@ def extract_type_hierarchy( # extract respective types from response type_hierarchy = convert_to_dict(llm_response=llm_response) - + return type_hierarchy, llm_response - + except Exception as e: - print(f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}...") - time.sleep(1) # add a delay before retrying - + print( + f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + ) + time.sleep(1) # add a delay before retrying + raise RuntimeError("Max retries exceeded. Failed to extract type hierarchy.") - + @require_llm def extract_nl_actions( - self, + self, model: LLM, - domain_desc: str, - prompt_template: str, - types: dict[str,str]=None, - nl_actions: dict[str,str]=None, - max_retries: int=3 - ) -> tuple[dict[str,str], str]: + domain_desc: str, + prompt_template: str, + types: dict[str, str] = None, + nl_actions: dict[str, str] = None, + max_retries: int = 3, + ) -> tuple[dict[str, str], str]: """ Extract actions in natural language given domain description using LLM. @@ -158,14 +164,18 @@ def extract_nl_actions( nl_actions (dict[str, str]): a dictionary of extracted actions {action name: action description} llm_response (str): the raw string LLM response """ - + # replace prompt placeholders types_str = format_dict(types) if types else "No types provided." - nl_actions_str = "\n".join(f"{name}: {desc}" for name, desc in nl_actions.items()) if nl_actions else "No actions provided." - - prompt_template = prompt_template.replace('{domain_desc}', domain_desc) - prompt_template = prompt_template.replace('{types}', types_str) - prompt_template = prompt_template.replace('{nl_actions}', nl_actions_str) + nl_actions_str = ( + "\n".join(f"{name}: {desc}" for name, desc in nl_actions.items()) + if nl_actions + else "No actions provided." + ) + + prompt_template = prompt_template.replace("{domain_desc}", domain_desc) + prompt_template = prompt_template.replace("{types}", types_str) + prompt_template = prompt_template.replace("{nl_actions}", nl_actions_str) # iterate through attempts in case of extraction failure for attempt in range(max_retries): @@ -173,32 +183,36 @@ def extract_nl_actions( model.reset_tokens() - llm_response = model.query(prompt=prompt_template) # get LLM llm_response - + llm_response = model.query( + prompt=prompt_template + ) # get LLM llm_response + # extract respective types from response nl_actions = convert_to_dict(llm_response=llm_response) - + return nl_actions, llm_response - + except Exception as e: - print(f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}...") - time.sleep(1) # add a delay before retrying - + print( + f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + ) + time.sleep(1) # add a delay before retrying + raise RuntimeError("Max retries exceeded. Failed to extract NL actions.") - + @require_llm def extract_pddl_action( - self, - model: LLM, + self, + model: LLM, domain_desc: str, - prompt_template: str, + prompt_template: str, action_name: str, - action_desc: str=None, - action_list: dict[str,str]=None, - predicates: list[Predicate]=None, - types: dict[str,str]=None, - max_retries: int=3 - ) -> tuple[Action, list[Predicate], str]: + action_desc: str = None, + action_list: dict[str, str] = None, + predicates: list[Predicate] = None, + types: dict[str, str] = None, + max_retries: int = 3, + ) -> tuple[Action, list[Predicate], str]: """ Extract an action and predicates from a given action description using LLM @@ -221,46 +235,56 @@ def extract_pddl_action( # replace prompt placeholders types_str = format_dict(types) if types else "No types provided." - predicates_str = format_predicates(predicates) if predicates else "No predicates provided." - action_list_str = str(action_list) if action_list else "No other actions provided" - - prompt_template = prompt_template.replace('{domain_desc}', domain_desc) - prompt_template = prompt_template.replace('{action_list}', action_list_str) - prompt_template = prompt_template.replace('{action_name}', action_name) - prompt_template = prompt_template.replace('{action_desc}', action_desc if action_desc else "No description available.") - prompt_template = prompt_template.replace('{types}', types_str) - prompt_template = prompt_template.replace('{predicates}', predicates_str) - + predicates_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) + action_list_str = ( + str(action_list) if action_list else "No other actions provided" + ) + + prompt_template = prompt_template.replace("{domain_desc}", domain_desc) + prompt_template = prompt_template.replace("{action_list}", action_list_str) + prompt_template = prompt_template.replace("{action_name}", action_name) + prompt_template = prompt_template.replace( + "{action_desc}", action_desc if action_desc else "No description available." + ) + prompt_template = prompt_template.replace("{types}", types_str) + prompt_template = prompt_template.replace("{predicates}", predicates_str) + # iterate through attempts in case of extraction failure for attempt in range(max_retries): try: model.reset_tokens() - + llm_response = model.query(prompt=prompt_template) # extract respective types from response - action = parse_action(llm_response=llm_response, action_name=action_name) + action = parse_action( + llm_response=llm_response, action_name=action_name + ) new_predicates = parse_new_predicates(llm_response) return action, new_predicates, llm_response - + except Exception as e: - print(f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}...") - time.sleep(1) # add a delay before retrying - + print( + f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + ) + time.sleep(1) # add a delay before retrying + raise RuntimeError("Max retries exceeded. Failed to extract PDDL action.") - + @require_llm def extract_pddl_actions( - self, - model: LLM, + self, + model: LLM, domain_desc: str, - prompt_template: str, - nl_actions: dict[str,str]=None, - predicates: list[Predicate]=None, - types: dict[str,str]=None - ) -> tuple[list[Action], list[Predicate], str]: + prompt_template: str, + nl_actions: dict[str, str] = None, + predicates: list[Predicate] = None, + types: dict[str, str] = None, + ) -> tuple[list[Action], list[Predicate], str]: """ Extract all actions from a given action description using LLM @@ -277,64 +301,74 @@ def extract_pddl_actions( new_predicates (list[Predicate]): a list of new predicates llm_response (str): the raw string LLM response """ - + model.reset_tokens() - + # replace prompt placeholders types_str = format_dict(types) if types else "No types provided." - predicates_str = format_predicates(predicates) if predicates else "No predicates provided." - nl_actions_str = format_dict(nl_actions) if nl_actions else "No actions provided." - - prompt_template = prompt_template.replace('{domain_desc}', domain_desc) - prompt_template = prompt_template.replace('{types}', types_str) - prompt_template = prompt_template.replace('{predicates}', predicates_str) - prompt_template = prompt_template.replace('{nl_actions}', nl_actions_str) + predicates_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) + nl_actions_str = ( + format_dict(nl_actions) if nl_actions else "No actions provided." + ) + + prompt_template = prompt_template.replace("{domain_desc}", domain_desc) + prompt_template = prompt_template.replace("{types}", types_str) + prompt_template = prompt_template.replace("{predicates}", predicates_str) + prompt_template = prompt_template.replace("{nl_actions}", nl_actions_str) llm_response = model.query(prompt=prompt_template) - + # extract respective types from response - raw_actions = llm_response.split('## NEXT ACTION') - + raw_actions = llm_response.split("## NEXT ACTION") + actions = [] for i in raw_actions: # define the regex patterns - action_pattern = re.compile(r'\[([^\]]+)\]') - rest_of_string_pattern = re.compile(r'\[([^\]]+)\](.*)', re.DOTALL) - + action_pattern = re.compile(r"\[([^\]]+)\]") + rest_of_string_pattern = re.compile(r"\[([^\]]+)\](.*)", re.DOTALL) + # search for the action name action_match = action_pattern.search(i) action_name = action_match.group(1) if action_match else None - + # extract the rest of the string rest_match = rest_of_string_pattern.search(i) rest_of_string = rest_match.group(2).strip() if rest_match else None - - actions.append(parse_action(llm_response=rest_of_string, action_name=action_name)) - + + actions.append( + parse_action(llm_response=rest_of_string, action_name=action_name) + ) + # if user queries predicate creation via LLM try: new_predicates = parse_new_predicates(llm_response) - + if predicates: - new_predicates = [pred for pred in new_predicates if pred['name'] not in [p["name"] for p in predicates]] # remove re-defined predicates + new_predicates = [ + pred + for pred in new_predicates + if pred["name"] not in [p["name"] for p in predicates] + ] # remove re-defined predicates except Exception as e: # Log or print the exception if needed print(f"No new predicates: {e}") new_predicates = None - return actions, new_predicates, llm_response - + return actions, new_predicates, llm_response + @require_llm def extract_parameters( - self, - model: LLM, + self, + model: LLM, domain_desc: str, - prompt_template: str, - action_name: str, - action_desc: str, - types: dict[str,str]=None, - max_retries: int=3 - ) -> tuple[OrderedDict, list, str]: + prompt_template: str, + action_name: str, + action_desc: str, + types: dict[str, str] = None, + max_retries: int = 3, + ) -> tuple[OrderedDict, list, str]: """ Extracts parameters from single action description via LLM @@ -355,42 +389,44 @@ def extract_parameters( # replace prompt placeholders types_str = format_dict(types) if types else "No types provided." - - prompt_template = prompt_template.replace('{domain_desc}', domain_desc) - prompt_template = prompt_template.replace('{action_name}', action_name) - prompt_template = prompt_template.replace('{action_desc}', action_desc) - prompt_template = prompt_template.replace('{types}', types_str) - + + prompt_template = prompt_template.replace("{domain_desc}", domain_desc) + prompt_template = prompt_template.replace("{action_name}", action_name) + prompt_template = prompt_template.replace("{action_desc}", action_desc) + prompt_template = prompt_template.replace("{types}", types_str) + # iterate through attempts in case of extraction failure for attempt in range(max_retries): try: model.reset_tokens() - llm_response = model.query(prompt=prompt_template) # get LLM response - + llm_response = model.query(prompt=prompt_template) # get LLM response + # extract respective types from response param, param_raw = parse_params(llm_output=llm_response) return param, param_raw, llm_response - + except Exception as e: - print(f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}...") - time.sleep(1) # add a delay before retrying - + print( + f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + ) + time.sleep(1) # add a delay before retrying + raise RuntimeError("Max retries exceeded. Failed to extract parameters.") - + @require_llm def extract_preconditions( - self, - model: LLM, + self, + model: LLM, domain_desc: str, - prompt_template: str, - action_name: str, - action_desc: str, - params: list[str]=None, - predicates: list[Predicate]=None, - max_retries: int=3 - ) -> tuple[str, list[Predicate], str]: + prompt_template: str, + action_name: str, + action_desc: str, + params: list[str] = None, + predicates: list[Predicate] = None, + max_retries: int = 3, + ) -> tuple[str, list[Predicate], str]: """ Extracts preconditions from single action description via LLM @@ -411,47 +447,56 @@ def extract_preconditions( """ # replace prompt placeholders - predicates_str = format_predicates(predicates) if predicates else "No predicates provided." + predicates_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) params_str = "\n".join(params) if params else "No parameters provided." - prompt_template = prompt_template.replace('{domain_desc}', domain_desc) - prompt_template = prompt_template.replace('{action_name}', action_name) - prompt_template = prompt_template.replace('{action_desc}', action_desc) - prompt_template = prompt_template.replace('{parameters}', params_str) - prompt_template = prompt_template.replace('{predicates}', predicates_str) - + prompt_template = prompt_template.replace("{domain_desc}", domain_desc) + prompt_template = prompt_template.replace("{action_name}", action_name) + prompt_template = prompt_template.replace("{action_desc}", action_desc) + prompt_template = prompt_template.replace("{parameters}", params_str) + prompt_template = prompt_template.replace("{predicates}", predicates_str) + # iterate through attempts in case of extraction failure for attempt in range(max_retries): try: model.reset_tokens() - llm_response = model.query(prompt=prompt_template) # get LLM response - + llm_response = model.query(prompt=prompt_template) # get LLM response + # extract respective types from response - preconditions = llm_response.split("Preconditions\n")[1].split("##")[0].split("```")[1].strip(" `\n") + preconditions = ( + llm_response.split("Preconditions\n")[1] + .split("##")[0] + .split("```")[1] + .strip(" `\n") + ) new_predicates = parse_new_predicates(llm_output=llm_response) return preconditions, new_predicates, llm_response - + except Exception as e: - print(f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}...") - time.sleep(1) # add a delay before retrying - + print( + f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + ) + time.sleep(1) # add a delay before retrying + raise RuntimeError("Max retries exceeded. Failed to extract preconditions.") @require_llm def extract_effects( - self, - model: LLM, + self, + model: LLM, domain_desc: str, - prompt_template: str, - action_name: str, - action_desc: str, - params: list[str]=None, - precondition: str=None, - predicates: list[Predicate]=None, - max_retries: int=3 - ) -> tuple[str, list[Predicate], str]: + prompt_template: str, + action_name: str, + action_desc: str, + params: list[str] = None, + precondition: str = None, + predicates: list[Predicate] = None, + max_retries: int = 3, + ) -> tuple[str, list[Predicate], str]: """ Extracts effects from single action description via LLM @@ -473,46 +518,55 @@ def extract_effects( """ # replace prompt placeholders - predicates_str = format_predicates(predicates) if predicates else "No predicates provided." + predicates_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) params_str = "\n".join(params) if params else "No parameters provided." - - prompt_template = prompt_template.replace('{domain_desc}', domain_desc) - prompt_template = prompt_template.replace('{action_name}', action_name) - prompt_template = prompt_template.replace('{action_desc}', action_desc) - prompt_template = prompt_template.replace('{parameters}', params_str) - prompt_template = prompt_template.replace('{preconditions}', precondition) - prompt_template = prompt_template.replace('{predicates}', predicates_str) - + + prompt_template = prompt_template.replace("{domain_desc}", domain_desc) + prompt_template = prompt_template.replace("{action_name}", action_name) + prompt_template = prompt_template.replace("{action_desc}", action_desc) + prompt_template = prompt_template.replace("{parameters}", params_str) + prompt_template = prompt_template.replace("{preconditions}", precondition) + prompt_template = prompt_template.replace("{predicates}", predicates_str) + # iterate through attempts in case of extraction failure for attempt in range(max_retries): try: model.reset_tokens() - llm_response = model.query(prompt=prompt_template) # get LLM response - + llm_response = model.query(prompt=prompt_template) # get LLM response + # extract respective types from response - effects = llm_response.split("Effects\n")[1].split("##")[0].split("```")[1].strip(" `\n") + effects = ( + llm_response.split("Effects\n")[1] + .split("##")[0] + .split("```")[1] + .strip(" `\n") + ) new_predicates = parse_new_predicates(llm_output=llm_response) return effects, new_predicates, llm_response - + except Exception as e: - print(f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}...") - time.sleep(1) # add a delay before retrying - + print( + f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + ) + time.sleep(1) # add a delay before retrying + raise RuntimeError("Max retries exceeded. Failed to extract effects.") - + @require_llm def extract_predicates( - self, + self, model: LLM, domain_desc: str, - prompt_template: str, - types: dict[str,str]=None, - predicates: list[Predicate]=None, - nl_actions: dict[str,str]=None, - max_retries: int=3, - ) -> tuple[list[Predicate], str]: + prompt_template: str, + types: dict[str, str] = None, + predicates: list[Predicate] = None, + nl_actions: dict[str, str] = None, + max_retries: int = 3, + ) -> tuple[list[Predicate], str]: """ Extracts predicates via LLM @@ -532,67 +586,85 @@ def extract_predicates( # replace prompt placeholders types_str = format_dict(types) if types else "No types provided." - predicates_str = format_predicates(predicates) if predicates else "No predicates provided." - nl_actions_str = "\n".join(f"{name}: {desc}" for name, desc in nl_actions.items()) if nl_actions else "No actions provided." - - prompt_template = prompt_template.replace('{domain_desc}', domain_desc) - prompt_template = prompt_template.replace('{types}', types_str) - prompt_template = prompt_template.replace('{predicates}', predicates_str) - prompt_template = prompt_template.replace('{nl_actions}', nl_actions_str) - + predicates_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) + nl_actions_str = ( + "\n".join(f"{name}: {desc}" for name, desc in nl_actions.items()) + if nl_actions + else "No actions provided." + ) + + prompt_template = prompt_template.replace("{domain_desc}", domain_desc) + prompt_template = prompt_template.replace("{types}", types_str) + prompt_template = prompt_template.replace("{predicates}", predicates_str) + prompt_template = prompt_template.replace("{nl_actions}", nl_actions_str) + # iterate through attempts in case of extraction failure for attempt in range(max_retries): try: model.reset_tokens() - - llm_response = model.query(prompt=prompt_template) # prompt model - + + llm_response = model.query(prompt=prompt_template) # prompt model + # extract respective types from response new_predicates = parse_new_predicates(llm_output=llm_response) - + return new_predicates, llm_response - + except Exception as e: - print(f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}...") - time.sleep(1) # add a delay before retrying - - raise RuntimeError("Max retries exceeded. Failed to extract predicates.") + print( + f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + ) + time.sleep(1) # add a delay before retrying + raise RuntimeError("Max retries exceeded. Failed to extract predicates.") """Delete functions""" + def delete_type(self, name: str): """Deletes specific type from current model""" if self.types is not None: - self.types = {type_: desc for type_, desc in self.types.items() if type_ != name} + self.types = { + type_: desc for type_, desc in self.types.items() if type_ != name + } def delete_nl_action(self, name: str): """Deletes specific NL action from current model""" if self.nl_actions is not None: - self.nl_actions = {action_name: action_desc for action_name, action_desc in self.nl_actions.items() if action_name != name} + self.nl_actions = { + action_name: action_desc + for action_name, action_desc in self.nl_actions.items() + if action_name != name + } def delete_pddl_action(self, name: str): """Deletes specific PDDL action from current model""" if self.pddl_actions is not None: - self.pddl_actions = [action for action in self.pddl_actions if action['name'] != name] + self.pddl_actions = [ + action for action in self.pddl_actions if action["name"] != name + ] def delete_predicate(self, name: str): """Deletes specific predicate from current model""" if self.predicates is not None: - self.predicates = [predicate for predicate in self.predicates if predicate['name'] != name] - + self.predicates = [ + predicate for predicate in self.predicates if predicate["name"] != name + ] """Set functions""" - def set_types(self, types: dict[str,str]): + + def set_types(self, types: dict[str, str]): """Sets types for current model""" - self.types=types + self.types = types - def set_type_hierarchy(self, type_hierarchy: dict[str,str]): + def set_type_hierarchy(self, type_hierarchy: dict[str, str]): """Sets type hierarchy for current model""" - self.type_hierarchy=type_hierarchy + self.type_hierarchy = type_hierarchy - def set_nl_actions(self, nl_actions: dict[str,str]): + def set_nl_actions(self, nl_actions: dict[str, str]): """Sets NL actions for current model""" - self.nl_actions=nl_actions + self.nl_actions = nl_actions def set_pddl_action(self, pddl_action: Action): """Appends a PDDL action for current model""" @@ -603,6 +675,7 @@ def set_predicate(self, predicate: Predicate): self.predicates.append(predicate) """Get functions""" + def get_types(self): """Returns types from current model""" return self.types @@ -624,23 +697,23 @@ def get_predicates(self): return self.predicates def generate_domain( - self, - domain: str, - types: str, - predicates: str, - actions: list[Action], - requirements: list[str], - ) -> str: + self, + domain: str, + types: str, + predicates: str, + actions: list[Action], + requirements: list[str], + ) -> str: """ Generates PDDL domain from given information - + Args: domain (str): domain name types (str): domain types predicates (str): domain predicates actions (list[Action]): domain actions requirements (list[str]): domain requirements - + Returns: desc (str): PDDL domain """ @@ -651,12 +724,16 @@ def generate_domain( desc += f" (:predicates \n{indent(predicates)}\n )" desc += self.action_descs(actions) desc += "\n)" - desc = desc.replace("AND", "and").replace("OR", "or") # The python PDDL package can't handle capital AND and OR + desc = desc.replace("AND", "and").replace( + "OR", "or" + ) # The python PDDL package can't handle capital AND and OR return desc - + def action_desc(self, action: Action) -> str: """Helper function to format individual action descriptions""" - param_str = "\n".join([f"{name} - {type}" for name, type in action['parameters'].items()]) # name includes ? + param_str = "\n".join( + [f"{name} - {type}" for name, type in action["parameters"].items()] + ) # name includes ? desc = f"(:action {action['name']}\n" desc += f" :parameters (\n{param_str}\n )\n" desc += f" :precondition\n{action['preconditions']}\n" @@ -674,6 +751,3 @@ def action_descs(self, actions) -> str: def format_predicates(self, predicates: list[Predicate]) -> str: """Helper function that formats predicate list into string""" return "\n".join([pred["clean"].replace(":", " ; ") for pred in predicates]) - -if __name__ == "__main__": - pass \ No newline at end of file diff --git a/l2p/feedback_builder.py b/l2p/feedback_builder.py index 9a25b03..e5b2675 100644 --- a/l2p/feedback_builder.py +++ b/l2p/feedback_builder.py @@ -12,522 +12,668 @@ domain_builder = DomainBuilder() task_builder = TaskBuilder() + class FeedbackBuilder: @require_llm def get_feedback( - self, - model: LLM, - feedback_template: str, - feedback_type: str, - llm_response: str - ) -> tuple[bool, str]: + self, model: LLM, feedback_template: str, feedback_type: str, llm_response: str + ) -> tuple[bool, str]: """ This retrieves the type of feedback user requests and returns feedack message. - feedback_type takes in either "human" "llm" or "hybrid" which it both + feedback_type takes in either "human" "llm" or "hybrid" which it both """ model.reset_tokens() - + if feedback_type.lower() == "human": feedback_msg = self.human_feedback(llm_response) elif feedback_type.lower() == "llm": feedback_msg = model.query(prompt=feedback_template) elif feedback_type.lower() == "hybrid": feedback_msg = model.query(prompt=feedback_template) - response = "\nORIGINAL LLM OUTPUT:\n" + llm_response + "\nFEEDBACK:\n" + feedback_msg + response = ( + "\nORIGINAL LLM OUTPUT:\n" + + llm_response + + "\nFEEDBACK:\n" + + feedback_msg + ) feedback_msg.replace("no feedback".lower(), "") feedback_msg += self.human_feedback(response) else: - raise ValueError("Invalid feedback_type. Expected 'human', 'llm', or 'hybrid'.") - - if 'no feedback' in feedback_msg.lower() or len(feedback_msg.strip()) == 0: + raise ValueError( + "Invalid feedback_type. Expected 'human', 'llm', or 'hybrid'." + ) + + if "no feedback" in feedback_msg.lower() or len(feedback_msg.strip()) == 0: return True, feedback_msg - + return False, feedback_msg @require_llm def type_feedback( - self, - model: LLM, - domain_desc: str, - llm_response: str, - feedback_template: str, - feedback_type: str="llm", - types: dict[str,str]=None, - ) -> tuple[dict[str,str], str]: + self, + model: LLM, + domain_desc: str, + llm_response: str, + feedback_template: str, + feedback_type: str = "llm", + types: dict[str, str] = None, + ) -> tuple[dict[str, str], str]: """Makes LLM call using feedback prompt, then parses it into type format""" model.reset_tokens() type_str = format_dict(types) if types else "No types provided." - feedback_template = feedback_template.replace('{domain_desc}', domain_desc) - feedback_template = feedback_template.replace('{types}', type_str) - feedback_template = feedback_template.replace('{llm_response}', llm_response) + feedback_template = feedback_template.replace("{domain_desc}", domain_desc) + feedback_template = feedback_template.replace("{types}", type_str) + feedback_template = feedback_template.replace("{llm_response}", llm_response) + + no_fb, fb_msg = self.get_feedback( + model, feedback_template, feedback_type, llm_response + ) - no_fb, fb_msg = self.get_feedback(model, feedback_template, feedback_type, llm_response) - if not no_fb: prompt = ( f"\n\nYou now are revising your answer using feedback. Here is the feedback you outputted:\n{fb_msg}" f"\n\nFollow the same syntax format as the original output in your answer:\n{llm_response}" ) - types, llm_response = domain_builder.extract_type(model, domain_desc, prompt) + types, llm_response = domain_builder.extract_type( + model, domain_desc, prompt + ) return types, llm_response @require_llm def type_hierarchy_feedback( - self, - model: LLM, - domain_desc: str, - llm_response: str, - feedback_template: str, - feedback_type: str="llm", - type_hierarchy: dict[str,str]=None, - ) -> tuple[dict[str,str], str]: + self, + model: LLM, + domain_desc: str, + llm_response: str, + feedback_template: str, + feedback_type: str = "llm", + type_hierarchy: dict[str, str] = None, + ) -> tuple[dict[str, str], str]: """Makes LLM call using feedback prompt, then parses it into type hierarchy format""" model.reset_tokens() - type_str = format_dict(type_hierarchy) if type_hierarchy else "No types provided." + type_str = ( + format_dict(type_hierarchy) if type_hierarchy else "No types provided." + ) + + feedback_template = feedback_template.replace("{domain_desc}", domain_desc) + feedback_template = feedback_template.replace("{types}", type_str) + feedback_template = feedback_template.replace("{llm_response}", llm_response) + + no_fb, fb_msg = self.get_feedback( + model, feedback_template, feedback_type, llm_response + ) - feedback_template = feedback_template.replace('{domain_desc}', domain_desc) - feedback_template = feedback_template.replace('{types}', type_str) - feedback_template = feedback_template.replace('{llm_response}', llm_response) - - no_fb, fb_msg = self.get_feedback(model, feedback_template, feedback_type, llm_response) - if not no_fb: prompt = ( f"\n\nYou now are revising your answer using feedback. Here is the feedback you outputted:\n{fb_msg}" f"\n\nFollow the same syntax format as the original output in your answer:\n{llm_response}" ) - type_hierarchy, llm_response = domain_builder.extract_type_hierarchy(model, domain_desc, prompt) + type_hierarchy, llm_response = domain_builder.extract_type_hierarchy( + model, domain_desc, prompt + ) return type_hierarchy, llm_response @require_llm def nl_action_feedback( - self, - model: LLM, - domain_desc: str, - llm_response: str, - feedback_template: str, - feedback_type: str="llm", - nl_actions: dict[str,str]=None, - type_hierarchy: dict[str,str]=None, - ) -> tuple[dict[str,str], str]: + self, + model: LLM, + domain_desc: str, + llm_response: str, + feedback_template: str, + feedback_type: str = "llm", + nl_actions: dict[str, str] = None, + type_hierarchy: dict[str, str] = None, + ) -> tuple[dict[str, str], str]: """Makes LLM call using feedback prompt, then parses it into nl_action format""" model.reset_tokens() - type_str = format_dict(type_hierarchy) if type_hierarchy else "No types provided." - nl_action_str = format_dict(nl_actions) if nl_actions else "No actions provided." + type_str = ( + format_dict(type_hierarchy) if type_hierarchy else "No types provided." + ) + nl_action_str = ( + format_dict(nl_actions) if nl_actions else "No actions provided." + ) - feedback_template = feedback_template.replace('{domain_desc}', domain_desc) - feedback_template = feedback_template.replace('{llm_response}', llm_response) - feedback_template = feedback_template.replace('{types}', type_str) - feedback_template = feedback_template.replace('{nl_actions}', nl_action_str) + feedback_template = feedback_template.replace("{domain_desc}", domain_desc) + feedback_template = feedback_template.replace("{llm_response}", llm_response) + feedback_template = feedback_template.replace("{types}", type_str) + feedback_template = feedback_template.replace("{nl_actions}", nl_action_str) + + no_fb, fb_msg = self.get_feedback( + model, feedback_template, feedback_type, llm_response + ) - no_fb, fb_msg = self.get_feedback(model, feedback_template, feedback_type, llm_response) - if not no_fb: prompt = ( f"\n\nYou now are revising your answer using feedback. Here is the feedback you outputted:\n{fb_msg}" f"\n\nFollow the same syntax format as the original output in your answer:\n{llm_response}" ) - nl_actions, llm_response = domain_builder.extract_type_hierarchy(model, domain_desc, prompt) + nl_actions, llm_response = domain_builder.extract_type_hierarchy( + model, domain_desc, prompt + ) return nl_actions, llm_response @require_llm def pddl_action_feedback( - self, - model: LLM, - domain_desc: str, - llm_response: str, - feedback_template: str, - feedback_type: str="llm", - action: Action=None, - predicates: list[Predicate]=None, - types: dict[str,str]=None - ) -> tuple[Action, list[Predicate], str]: + self, + model: LLM, + domain_desc: str, + llm_response: str, + feedback_template: str, + feedback_type: str = "llm", + action: Action = None, + predicates: list[Predicate] = None, + types: dict[str, str] = None, + ) -> tuple[Action, list[Predicate], str]: """Makes LLM call using feedback prompt, then parses it into action format""" model.reset_tokens() type_str = format_dict(types) if types else "No types provided." - predicate_str = format_predicates(predicates) if predicates else "No predicates provided." - param_str = ", ".join([f"{name} - {type}" for name, type in action['parameters'].items()]) \ - if action else "No parameters provided" - action_name = action['name'] if action else "No action name provided" - preconditions_str = action['preconditions'] if action else "No preconditions provided." - effects_str = action['effects'] if action else "No effects provided." - - feedback_template = feedback_template.replace('{domain_desc}', domain_desc) - feedback_template = feedback_template.replace('{llm_response}', llm_response) - feedback_template = feedback_template.replace('{types}', type_str) - feedback_template = feedback_template.replace('{predicates}', predicate_str) - feedback_template = feedback_template.replace('{action_name}', action_name) - feedback_template = feedback_template.replace('{parameters}', param_str) - feedback_template = feedback_template.replace('{action_preconditions}', preconditions_str) - feedback_template = feedback_template.replace('{action_effects}', effects_str) - - no_fb, fb_msg = self.get_feedback(model, feedback_template, feedback_type, llm_response) - + predicate_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) + param_str = ( + ", ".join( + [f"{name} - {type}" for name, type in action["parameters"].items()] + ) + if action + else "No parameters provided" + ) + action_name = action["name"] if action else "No action name provided" + preconditions_str = ( + action["preconditions"] if action else "No preconditions provided." + ) + effects_str = action["effects"] if action else "No effects provided." + + feedback_template = feedback_template.replace("{domain_desc}", domain_desc) + feedback_template = feedback_template.replace("{llm_response}", llm_response) + feedback_template = feedback_template.replace("{types}", type_str) + feedback_template = feedback_template.replace("{predicates}", predicate_str) + feedback_template = feedback_template.replace("{action_name}", action_name) + feedback_template = feedback_template.replace("{parameters}", param_str) + feedback_template = feedback_template.replace( + "{action_preconditions}", preconditions_str + ) + feedback_template = feedback_template.replace("{action_effects}", effects_str) + + no_fb, fb_msg = self.get_feedback( + model, feedback_template, feedback_type, llm_response + ) + if not no_fb: prompt = ( f"\n\nYou now are revising your answer using feedback. Here is the feedback you outputted:\n{fb_msg}" f"\n\nFollow the same syntax format as the original output in your answer:\n{llm_response}" ) - action, predicates, llm_response = domain_builder.extract_pddl_action(model, domain_desc, prompt, action_name) + action, predicates, llm_response = domain_builder.extract_pddl_action( + model, domain_desc, prompt, action_name + ) return action, predicates, llm_response @require_llm def parameter_feedback( - self, - model: LLM, - domain_desc: str, - llm_response: str, - feedback_template: str, - feedback_type: str="llm", - parameter: OrderedDict=None, - action_name: str=None, - action_desc: str=None, - types: dict[str,str]=None - ) -> tuple[OrderedDict, OrderedDict, str]: + self, + model: LLM, + domain_desc: str, + llm_response: str, + feedback_template: str, + feedback_type: str = "llm", + parameter: OrderedDict = None, + action_name: str = None, + action_desc: str = None, + types: dict[str, str] = None, + ) -> tuple[OrderedDict, OrderedDict, str]: """Makes LLM call using feedback prompt, then parses it into parameter format""" model.reset_tokens() type_str = format_dict(types) if types else "No types provided." - param_str = "\n".join([f"{name} - {type}" for name, type in parameter.items()]) \ - if parameter else "No parameters provided" + param_str = ( + "\n".join([f"{name} - {type}" for name, type in parameter.items()]) + if parameter + else "No parameters provided" + ) action_name = action_name if action_name else "No action name provided." action_desc = action_desc if action_desc else "No action description provided." - - feedback_template = feedback_template.replace('{domain_desc}', domain_desc) - feedback_template = feedback_template.replace('{llm_response}', llm_response) - feedback_template = feedback_template.replace('{types}', type_str) - feedback_template = feedback_template.replace('{action_name}', action_name) - feedback_template = feedback_template.replace('{action_desc}', action_desc) - feedback_template = feedback_template.replace('{parameters}', param_str) - - no_fb, fb_msg = self.get_feedback(model, feedback_template, feedback_type, llm_response) - + + feedback_template = feedback_template.replace("{domain_desc}", domain_desc) + feedback_template = feedback_template.replace("{llm_response}", llm_response) + feedback_template = feedback_template.replace("{types}", type_str) + feedback_template = feedback_template.replace("{action_name}", action_name) + feedback_template = feedback_template.replace("{action_desc}", action_desc) + feedback_template = feedback_template.replace("{parameters}", param_str) + + no_fb, fb_msg = self.get_feedback( + model, feedback_template, feedback_type, llm_response + ) + if not no_fb: prompt = ( f"\n\nYou now are revising your answer using feedback. Here is the feedback you outputted:\n{fb_msg}" f"\n\nFollow the same syntax format as the original output in your answer:\n{llm_response}" ) - param, param_raw, llm_response = domain_builder.extract_parameters(model, domain_desc, prompt, action_name, action_desc, types) + param, param_raw, llm_response = domain_builder.extract_parameters( + model, domain_desc, prompt, action_name, action_desc, types + ) return param, param_raw, llm_response @require_llm def precondition_feedback( - self, - model: LLM, - domain_desc: str, - llm_response: str, - feedback_template: str, - feedback_type: str="llm", - parameter: OrderedDict=None, - preconditions: str=None, - action_name: str=None, - action_desc: str=None, - types: dict[str,str]=None, - predicates: list[Predicate]=None - ) -> tuple[str, list[Predicate], str]: + self, + model: LLM, + domain_desc: str, + llm_response: str, + feedback_template: str, + feedback_type: str = "llm", + parameter: OrderedDict = None, + preconditions: str = None, + action_name: str = None, + action_desc: str = None, + types: dict[str, str] = None, + predicates: list[Predicate] = None, + ) -> tuple[str, list[Predicate], str]: """Makes LLM call using feedback prompt, then parses it into precondition format""" - + model.reset_tokens() type_str = format_dict(types) if types else "No types provided." - predicate_str = format_predicates(predicates) if predicates else "No predicates provided." - param_str = "\n".join([f"{name} - {type}" for name, type in parameter.items()]) \ - if parameter else "No parameters provided" + predicate_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) + param_str = ( + "\n".join([f"{name} - {type}" for name, type in parameter.items()]) + if parameter + else "No parameters provided" + ) action_name = action_name if action_name else "No action name provided." action_desc = action_desc if action_desc else "No action description provided." - precondition_str = preconditions if preconditions else "No preconditions provided." - - feedback_template = feedback_template.replace('{domain_desc}', domain_desc) - feedback_template = feedback_template.replace('{llm_response}', llm_response) - feedback_template = feedback_template.replace('{types}', type_str) - feedback_template = feedback_template.replace('{predicates}', predicate_str) - feedback_template = feedback_template.replace('{action_name}', action_name) - feedback_template = feedback_template.replace('{action_desc}', action_desc) - feedback_template = feedback_template.replace('{parameters}', param_str) - feedback_template = feedback_template.replace('{action_preconditions}', precondition_str) - - no_fb, fb_msg = self.get_feedback(model, feedback_template, feedback_type, llm_response) - + precondition_str = ( + preconditions if preconditions else "No preconditions provided." + ) + + feedback_template = feedback_template.replace("{domain_desc}", domain_desc) + feedback_template = feedback_template.replace("{llm_response}", llm_response) + feedback_template = feedback_template.replace("{types}", type_str) + feedback_template = feedback_template.replace("{predicates}", predicate_str) + feedback_template = feedback_template.replace("{action_name}", action_name) + feedback_template = feedback_template.replace("{action_desc}", action_desc) + feedback_template = feedback_template.replace("{parameters}", param_str) + feedback_template = feedback_template.replace( + "{action_preconditions}", precondition_str + ) + + no_fb, fb_msg = self.get_feedback( + model, feedback_template, feedback_type, llm_response + ) + if not no_fb: prompt = ( f"\n\nYou now are revising your answer using feedback. Here is the feedback you outputted:\n{fb_msg}" f"\n\nFollow the same syntax format as the original output in your answer:\n{llm_response}" ) - preconditions, new_predicates, llm_response = domain_builder.extract_preconditions(model, - domain_desc, prompt, action_name, action_desc) + preconditions, new_predicates, llm_response = ( + domain_builder.extract_preconditions( + model, domain_desc, prompt, action_name, action_desc + ) + ) return preconditions, new_predicates, llm_response @require_llm def effect_feedback( - self, - model: LLM, - domain_desc: str, - llm_response: str, - feedback_template: str, - feedback_type: str="llm", - parameter: OrderedDict=None, - preconditions: str=None, - effects: str=None, - action_name: str=None, - action_desc: str=None, - types: dict[str,str]=None, - predicates: list[Predicate]=None - ) -> tuple[str, list[Predicate], str]: + self, + model: LLM, + domain_desc: str, + llm_response: str, + feedback_template: str, + feedback_type: str = "llm", + parameter: OrderedDict = None, + preconditions: str = None, + effects: str = None, + action_name: str = None, + action_desc: str = None, + types: dict[str, str] = None, + predicates: list[Predicate] = None, + ) -> tuple[str, list[Predicate], str]: """Makes LLM call using feedback prompt, then parses it into effects format""" model.reset_tokens() type_str = format_dict(types) if types else "No types provided." - predicate_str = format_predicates(predicates) if predicates else "No predicates provided." - param_str = "\n".join([f"{name} - {type}" for name, type in parameter.items()]) \ - if parameter else "No parameters provided" + predicate_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) + param_str = ( + "\n".join([f"{name} - {type}" for name, type in parameter.items()]) + if parameter + else "No parameters provided" + ) action_name = action_name if action_name else "No action name provided." action_desc = action_desc if action_desc else "No action description provided." - precondition_str = preconditions if preconditions else "No preconditions provided." + precondition_str = ( + preconditions if preconditions else "No preconditions provided." + ) effect_str = effects if effects else "No effects provided." - - feedback_template = feedback_template.replace('{domain_desc}', domain_desc) - feedback_template = feedback_template.replace('{llm_response}', llm_response) - feedback_template = feedback_template.replace('{types}', type_str) - feedback_template = feedback_template.replace('{predicates}', predicate_str) - feedback_template = feedback_template.replace('{action_name}', action_name) - feedback_template = feedback_template.replace('{action_desc}', action_desc) - feedback_template = feedback_template.replace('{parameters}', param_str) - feedback_template = feedback_template.replace('{action_preconditions}', precondition_str) - feedback_template = feedback_template.replace('{action_effects}', effect_str) - - no_fb, fb_msg = self.get_feedback(model, feedback_template, feedback_type, llm_response) - + + feedback_template = feedback_template.replace("{domain_desc}", domain_desc) + feedback_template = feedback_template.replace("{llm_response}", llm_response) + feedback_template = feedback_template.replace("{types}", type_str) + feedback_template = feedback_template.replace("{predicates}", predicate_str) + feedback_template = feedback_template.replace("{action_name}", action_name) + feedback_template = feedback_template.replace("{action_desc}", action_desc) + feedback_template = feedback_template.replace("{parameters}", param_str) + feedback_template = feedback_template.replace( + "{action_preconditions}", precondition_str + ) + feedback_template = feedback_template.replace("{action_effects}", effect_str) + + no_fb, fb_msg = self.get_feedback( + model, feedback_template, feedback_type, llm_response + ) + if not no_fb: prompt = ( f"\n\nYou now are revising your answer using feedback. Here is the feedback you outputted:\n{fb_msg}" f"\n\nFollow the same syntax format as the original output in your answer:\n{llm_response}" ) - effects, new_predicates, llm_response = domain_builder.extract_effects(model, - domain_desc, prompt, action_name, action_desc) + effects, new_predicates, llm_response = domain_builder.extract_effects( + model, domain_desc, prompt, action_name, action_desc + ) return effects, new_predicates, llm_response @require_llm def predicate_feedback( - self, - model: LLM, - domain_desc: str, - llm_response: str, - feedback_template: str, - feedback_type: str="llm", - types: dict[str,str]=None, - predicates: list[Predicate]=None, - nl_actions: dict[str,str]=None - ) -> tuple[list[Predicate], str]: + self, + model: LLM, + domain_desc: str, + llm_response: str, + feedback_template: str, + feedback_type: str = "llm", + types: dict[str, str] = None, + predicates: list[Predicate] = None, + nl_actions: dict[str, str] = None, + ) -> tuple[list[Predicate], str]: """Makes LLM call using feedback prompt, then parses it into predicates format""" - + model.reset_tokens() type_str = format_dict(types) if types else "No types provided." - predicate_str = format_predicates(predicates) if predicates else "No predicates provided." - nl_action_str = format_dict(nl_actions) if nl_actions else "No actions provided." - - feedback_template = feedback_template.replace('{domain_desc}', domain_desc) - feedback_template = feedback_template.replace('{llm_response}', llm_response) - feedback_template = feedback_template.replace('{types}', type_str) - feedback_template = feedback_template.replace('{predicates}', predicate_str) - feedback_template = feedback_template.replace('{nl_actions}', nl_action_str) + predicate_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) + nl_action_str = ( + format_dict(nl_actions) if nl_actions else "No actions provided." + ) + + feedback_template = feedback_template.replace("{domain_desc}", domain_desc) + feedback_template = feedback_template.replace("{llm_response}", llm_response) + feedback_template = feedback_template.replace("{types}", type_str) + feedback_template = feedback_template.replace("{predicates}", predicate_str) + feedback_template = feedback_template.replace("{nl_actions}", nl_action_str) + + no_fb, fb_msg = self.get_feedback( + model, feedback_template, feedback_type, llm_response + ) - no_fb, fb_msg = self.get_feedback(model, feedback_template, feedback_type, llm_response) - if not no_fb: prompt = ( f"\n\nYou now are revising your answer using feedback. Here is the feedback you outputted:\n{fb_msg}" f"\n\nFollow the same syntax format as the original output in your answer:\n{llm_response}" ) - new_predicates, llm_response = domain_builder.extract_predicates(model, domain_desc, prompt) + new_predicates, llm_response = domain_builder.extract_predicates( + model, domain_desc, prompt + ) return new_predicates, llm_response @require_llm def task_feedback( - self, - model: LLM, - problem_desc: str, - llm_response: str, - feedback_template: str, - feedback_type: str="llm", - predicates: list[Predicate]=None, - types: dict[str,str]=None, - objects: dict[str,str]=None, - initial: list[dict[str,str]]=None, - goal: list[dict[str,str]]=None, - ) -> tuple[dict[str,str], list[dict[str,str]], list[dict[str,str]], str]: + self, + model: LLM, + problem_desc: str, + llm_response: str, + feedback_template: str, + feedback_type: str = "llm", + predicates: list[Predicate] = None, + types: dict[str, str] = None, + objects: dict[str, str] = None, + initial: list[dict[str, str]] = None, + goal: list[dict[str, str]] = None, + ) -> tuple[dict[str, str], list[dict[str, str]], list[dict[str, str]], str]: """Makes LLM call using feedback prompt, then parses it into object, initial, and goal format""" - + model.reset_tokens() type_str = format_dict(types) if types else "No types provided." - predicate_str = format_predicates(predicates) if predicates else "No predicates provided." - objects_str = "\n".join([f"{obj} - {type}" for obj, type in objects.items()]) if objects else "No objects provided." - initial_state_str = task_builder.format_initial(initial) if initial else "No initial state provided." - goal_state_str = task_builder.format_goal(goal) if goal else "No goal state provided." - - feedback_template = feedback_template.replace('{problem_desc}', problem_desc) - feedback_template = feedback_template.replace('{llm_response}', llm_response) - feedback_template = feedback_template.replace('{types}', type_str) - feedback_template = feedback_template.replace('{predicates}', predicate_str) - feedback_template = feedback_template.replace('{objects}', objects_str) - feedback_template = feedback_template.replace('{initial_state}', initial_state_str) - feedback_template = feedback_template.replace('{goal_state}', goal_state_str) - - no_fb, fb_msg = self.get_feedback(model, feedback_template, feedback_type, llm_response) - + predicate_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) + objects_str = ( + "\n".join([f"{obj} - {type}" for obj, type in objects.items()]) + if objects + else "No objects provided." + ) + initial_state_str = ( + task_builder.format_initial(initial) + if initial + else "No initial state provided." + ) + goal_state_str = ( + task_builder.format_goal(goal) if goal else "No goal state provided." + ) + + feedback_template = feedback_template.replace("{problem_desc}", problem_desc) + feedback_template = feedback_template.replace("{llm_response}", llm_response) + feedback_template = feedback_template.replace("{types}", type_str) + feedback_template = feedback_template.replace("{predicates}", predicate_str) + feedback_template = feedback_template.replace("{objects}", objects_str) + feedback_template = feedback_template.replace( + "{initial_state}", initial_state_str + ) + feedback_template = feedback_template.replace("{goal_state}", goal_state_str) + + no_fb, fb_msg = self.get_feedback( + model, feedback_template, feedback_type, llm_response + ) + if not no_fb: prompt = ( f"\n\nYou now are revising your answer using feedback. Here is the feedback you outputted:\n{fb_msg}" f"\n\nFollow the same syntax format as the original output in your answer:\n{llm_response}" ) - objects, initial, goal, _ = task_builder.extract_task(model, problem_desc, prompt) + objects, initial, goal, _ = task_builder.extract_task( + model, problem_desc, prompt + ) return objects, initial, goal, fb_msg @require_llm def objects_feedback( - self, - model: LLM, + self, + model: LLM, problem_desc: str, llm_response: str, - feedback_template: str, - feedback_type: str="llm", - type_hierarchy: dict[str,str]=None, - predicates: list[Predicate]=None, - objects: dict[str,str]=None - ) -> tuple[dict[str,str], str]: + feedback_template: str, + feedback_type: str = "llm", + type_hierarchy: dict[str, str] = None, + predicates: list[Predicate] = None, + objects: dict[str, str] = None, + ) -> tuple[dict[str, str], str]: """Makes LLM call using feedback prompt, then parses it into objects format""" - + model.reset_tokens() - type_str = format_dict(type_hierarchy) if type_hierarchy else "No types provided." - predicate_str = format_predicates(predicates) if predicates else "No predicates provided." - objects_str = "\n".join([f"{obj} - {type}" for obj, type in objects.items()]) if objects else "No objects provided." - - feedback_template = feedback_template.replace('{problem_desc}', problem_desc) - feedback_template = feedback_template.replace('{llm_response}', llm_response) - feedback_template = feedback_template.replace('{types}', type_str) - feedback_template = feedback_template.replace('{predicates}', predicate_str) - feedback_template = feedback_template.replace('{objects}', objects_str) - - no_fb, fb_msg = self.get_feedback(model, feedback_template, feedback_type, llm_response) - + type_str = ( + format_dict(type_hierarchy) if type_hierarchy else "No types provided." + ) + predicate_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) + objects_str = ( + "\n".join([f"{obj} - {type}" for obj, type in objects.items()]) + if objects + else "No objects provided." + ) + + feedback_template = feedback_template.replace("{problem_desc}", problem_desc) + feedback_template = feedback_template.replace("{llm_response}", llm_response) + feedback_template = feedback_template.replace("{types}", type_str) + feedback_template = feedback_template.replace("{predicates}", predicate_str) + feedback_template = feedback_template.replace("{objects}", objects_str) + + no_fb, fb_msg = self.get_feedback( + model, feedback_template, feedback_type, llm_response + ) + if not no_fb: prompt = ( f"\n\nYou now are revising your answer using feedback. Here is the feedback you outputted:\n{fb_msg}" f"\n\nFollow the same syntax format as the original output in your answer:\n{llm_response}" ) - objects, llm_response = task_builder.extract_objects(model, problem_desc, prompt) + objects, llm_response = task_builder.extract_objects( + model, problem_desc, prompt + ) return objects, llm_response @require_llm def initial_state_feedback( - self, - model: LLM, + self, + model: LLM, problem_desc: str, llm_response: str, - feedback_template: str, - feedback_type: str="llm", - type_hierarchy: dict[str,str]=None, - predicates: list[Predicate]=None, - objects: dict[str,str]=None, - initial: list[dict[str,str]]=None - ) -> tuple[list[dict[str,str]], str]: + feedback_template: str, + feedback_type: str = "llm", + type_hierarchy: dict[str, str] = None, + predicates: list[Predicate] = None, + objects: dict[str, str] = None, + initial: list[dict[str, str]] = None, + ) -> tuple[list[dict[str, str]], str]: """Makes LLM call using feedback prompt, then parses it into initial states format""" - + model.reset_tokens() - type_str = format_dict(type_hierarchy) if type_hierarchy else "No types provided." - predicate_str = format_predicates(predicates) if predicates else "No predicates provided." - objects_str = "\n".join([f"{obj} - {type}" for obj, type in objects.items()]) if objects else "No objects provided." - initial_state_str = task_builder.format_initial(initial) if initial else "No initial state provided." - - feedback_template = feedback_template.replace('{problem_desc}', problem_desc) - feedback_template = feedback_template.replace('{llm_response}', llm_response) - feedback_template = feedback_template.replace('{types}', type_str) - feedback_template = feedback_template.replace('{predicates}', predicate_str) - feedback_template = feedback_template.replace('{objects}', objects_str) - feedback_template = feedback_template.replace('{initial_state}', initial_state_str) - - no_fb, fb_msg = self.get_feedback(model, feedback_template, feedback_type, llm_response) - + type_str = ( + format_dict(type_hierarchy) if type_hierarchy else "No types provided." + ) + predicate_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) + objects_str = ( + "\n".join([f"{obj} - {type}" for obj, type in objects.items()]) + if objects + else "No objects provided." + ) + initial_state_str = ( + task_builder.format_initial(initial) + if initial + else "No initial state provided." + ) + + feedback_template = feedback_template.replace("{problem_desc}", problem_desc) + feedback_template = feedback_template.replace("{llm_response}", llm_response) + feedback_template = feedback_template.replace("{types}", type_str) + feedback_template = feedback_template.replace("{predicates}", predicate_str) + feedback_template = feedback_template.replace("{objects}", objects_str) + feedback_template = feedback_template.replace( + "{initial_state}", initial_state_str + ) + + no_fb, fb_msg = self.get_feedback( + model, feedback_template, feedback_type, llm_response + ) + if not no_fb: prompt = ( f"\n\nYou now are revising your answer using feedback. Here is the feedback you outputted:\n{fb_msg}" f"\n\nFollow the same syntax format as the original output in your answer:\n{llm_response}" ) - initial, llm_response = task_builder.extract_initial_state(model, problem_desc, prompt) + initial, llm_response = task_builder.extract_initial_state( + model, problem_desc, prompt + ) return initial, llm_response @require_llm def goal_state_feedback( - self, - model: LLM, + self, + model: LLM, problem_desc: str, llm_response: str, - feedback_template: str, - feedback_type: str="llm", - type_hierarchy: dict[str,str]=None, - predicates: list[Predicate]=None, - objects: dict[str,str]=None, - initial: list[dict[str,str]]=None, - goal: list[dict[str,str]]=None - ) -> tuple[list[dict[str,str]], str]: + feedback_template: str, + feedback_type: str = "llm", + type_hierarchy: dict[str, str] = None, + predicates: list[Predicate] = None, + objects: dict[str, str] = None, + initial: list[dict[str, str]] = None, + goal: list[dict[str, str]] = None, + ) -> tuple[list[dict[str, str]], str]: """Makes LLM call using feedback prompt, then parses it into goal states format""" - + model.reset_tokens() - type_str = format_dict(type_hierarchy) if type_hierarchy else "No types provided." - predicate_str = format_predicates(predicates) if predicates else "No predicates provided." - objects_str = "\n".join([f"{obj} - {type}" for obj, type in objects.items()]) if objects else "No objects provided." - initial_state_str = task_builder.format_initial(initial) if initial else "No initial state provided." - goal_state_str = task_builder.format_goal(goal) if goal else "No goal state provided." - - feedback_template = feedback_template.replace('{problem_desc}', problem_desc) - feedback_template = feedback_template.replace('{llm_response}', llm_response) - feedback_template = feedback_template.replace('{types}', type_str) - feedback_template = feedback_template.replace('{predicates}', predicate_str) - feedback_template = feedback_template.replace('{objects}', objects_str) - feedback_template = feedback_template.replace('{initial_state}', initial_state_str) - feedback_template = feedback_template.replace('{initial_state}', goal_state_str) - - no_fb, fb_msg = self.get_feedback(model, feedback_template, feedback_type, llm_response) - + type_str = ( + format_dict(type_hierarchy) if type_hierarchy else "No types provided." + ) + predicate_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) + objects_str = ( + "\n".join([f"{obj} - {type}" for obj, type in objects.items()]) + if objects + else "No objects provided." + ) + initial_state_str = ( + task_builder.format_initial(initial) + if initial + else "No initial state provided." + ) + goal_state_str = ( + task_builder.format_goal(goal) if goal else "No goal state provided." + ) + + feedback_template = feedback_template.replace("{problem_desc}", problem_desc) + feedback_template = feedback_template.replace("{llm_response}", llm_response) + feedback_template = feedback_template.replace("{types}", type_str) + feedback_template = feedback_template.replace("{predicates}", predicate_str) + feedback_template = feedback_template.replace("{objects}", objects_str) + feedback_template = feedback_template.replace( + "{initial_state}", initial_state_str + ) + feedback_template = feedback_template.replace("{initial_state}", goal_state_str) + + no_fb, fb_msg = self.get_feedback( + model, feedback_template, feedback_type, llm_response + ) + if not no_fb: prompt = ( f"\n\nYou now are revising your answer using feedback. Here is the feedback you outputted:\n{fb_msg}" f"\n\nFollow the same syntax format as the original output in your answer:\n{llm_response}" ) - goal, llm_response = task_builder.extract_goal_state(model, problem_desc, prompt) + goal, llm_response = task_builder.extract_goal_state( + model, problem_desc, prompt + ) return goal, llm_response @@ -547,8 +693,5 @@ def human_feedback(self, info: str): if resp.strip().lower() == "no feedback": return "no feedback" - - return resp -if __name__ == "__main__": - pass \ No newline at end of file + return resp diff --git a/l2p/llm_builder.py b/l2p/llm_builder.py index 23da82b..39a924b 100644 --- a/l2p/llm_builder.py +++ b/l2p/llm_builder.py @@ -11,33 +11,38 @@ LOG: logging.Logger = logging.getLogger(__name__) + def require_llm(func): """ Decorator to check if an LLM instance is provided and catch errors. """ + @functools.wraps(func) def wrapper(*args, **kwargs): # check if LLM instance is provided - model = kwargs.get('model', None) + model = kwargs.get("model", None) if model is None: # attempt tp get model from positional argument for arg in args: if isinstance(arg, LLM): model = arg break - + if not model: raise ValueError("An LLM instance must be provided to use this method.") - + # try-catch to ensure LLM is even used properly try: return func(*args, **kwargs) except Exception as e: - print(f"An error occurred in {func.__name__}: {e}.\n You must provide an LLM engine OR proper configuration to use L2P. Refer to https://github.com/AI-Planning/l2p.") + print( + f"An error occurred in {func.__name__}: {e}.\n You must provide an LLM engine OR proper configuration to use L2P. Refer to https://github.com/AI-Planning/l2p." + ) raise - + return wrapper + class LLM(ABC): def __init__(self, model: str, api_key: str | None = None) -> None: if model not in self.valid_models(): @@ -46,23 +51,23 @@ def __init__(self, model: str, api_key: str | None = None) -> None: ) self.model: str = model self.api_key: str | None = api_key - - @abstractmethod + + @abstractmethod def query(self, prompt: str) -> str: """ Abstract method to query an LLM with a given prompt and return the response. - + Args: prompt (str): The prompt to send to the LLM Returns: str: The response from the LLM """ pass - + def query_with_system_prompt(self, system_prompt: str, prompt: str) -> str: """ Abstract methody to query an LLM with a given prompt and system prompt and return the response. - + Args: system_prompt (str): The system prompt to send to the LLM prompt (str): The prompt to send to the LLM @@ -70,7 +75,7 @@ def query_with_system_prompt(self, system_prompt: str, prompt: str) -> str: str: The response from the LLM """ return self.query(system_prompt + "\n" + prompt) - + def valid_models(self) -> list[str]: """ List of valid model parameters, e.g., 'gpt4o-mini' for GPT @@ -80,10 +85,21 @@ def valid_models(self) -> list[str]: class OPENAI(LLM): """Accessing OpenAI""" - - def __init__(self, model: str, api_key: str | None = None, client=None, stop=None, max_tokens=4e3, - temperature=0, top_p=1, frequency_penalty=0.0, presence_penalty=0.0, seed=0) -> None: - + + def __init__( + self, + model: str, + api_key: str | None = None, + client=None, + stop=None, + max_tokens=4e3, + temperature=0, + top_p=1, + frequency_penalty=0.0, + presence_penalty=0.0, + seed=0, + ) -> None: + # attempt to import necessary OPENAI modules try: from openai import OpenAI @@ -92,47 +108,57 @@ def __init__(self, model: str, api_key: str | None = None, client=None, stop=Non "The 'openai' library is required for OPENAI but is not installed. " "Install it using: `pip install openai`." ) - + try: import tiktoken except ImportError: raise ImportError( "The 'tiktoken' library is required for token processing but is not installed. " "Install it using: `pip install tiktoken`." - ) - + ) + # call the parent class constructor to handle model and api_key super().__init__(model, api_key) - + # initialize the OpenAI client or use the one provided self.client = client if client else OpenAI(api_key=api_key) - + # store other parameters self.temperature = temperature self.top_p = top_p self.freq_penalty = frequency_penalty self.presence_penalty = presence_penalty self.stop = stop - + self.context_length = { - "gpt-3.5-turbo-0125": 16e3, # 16k tokens - "gpt-3.5-turbo-instruct": 4e3, # 4k tokens - "gpt-4-1106-preview": 128e3, # 128k tokens - "gpt-4-turbo-2024-04-09": 128e3, # 128k tokens - "gpt-4": 8192, # ~8k tokens - "gpt-4-32k": 32768, # ~32k tokens - "gpt-4o": 32768, # ~32k tokens - "gpt-4o-mini": 32768, # ~32k tokens + "gpt-3.5-turbo-0125": 16e3, # 16k tokens + "gpt-3.5-turbo-instruct": 4e3, # 4k tokens + "gpt-4-1106-preview": 128e3, # 128k tokens + "gpt-4-turbo-2024-04-09": 128e3, # 128k tokens + "gpt-4": 8192, # ~8k tokens + "gpt-4-32k": 32768, # ~32k tokens + "gpt-4o": 32768, # ~32k tokens + "gpt-4o-mini": 32768, # ~32k tokens }[model] - + self.max_tokens = max_tokens if max_tokens is not None else self.context_length - self.tok = tiktoken.get_encoding("cl100k_base") # For GPT3.5+ + self.tok = tiktoken.get_encoding("cl100k_base") # For GPT3.5+ self.in_tokens = 0 self.out_tokens = 0 - + @retry(tries=2, delay=60) - def connect_openai(self, client, model, messages, temperature, max_tokens, - top_p, frequency_penalty, presence_penalty, stop): + def connect_openai( + self, + client, + model, + messages, + temperature, + max_tokens, + top_p, + frequency_penalty, + presence_penalty, + stop, + ): return client.chat.completions.create( model=model, messages=messages, @@ -141,22 +167,35 @@ def connect_openai(self, client, model, messages, temperature, max_tokens, top_p=top_p, frequency_penalty=frequency_penalty, presence_penalty=presence_penalty, - stop=stop + stop=stop, ) @override - def query(self, prompt: str, messages=None, end_when_error=False, max_retry=5, est_margin = 200) -> str: + def query( + self, + prompt: str, + messages=None, + end_when_error=False, + max_retry=5, + est_margin=200, + ) -> str: if prompt is None and messages is None: raise ValueError("Prompt and messages cannot both be None") if messages is not None: messages = messages else: - messages = [{'role': 'user', 'content': prompt}] + messages = [{"role": "user", "content": prompt}] # calculate # of tokens to request. At most self.max_tokens, and prompt + request < self.context_length - current_tokens = int(sum([len(self.tok.encode(m['content'])) for m in messages])) # estimate current usage - requested_tokens = int(min(self.max_tokens, self.context_length - current_tokens - est_margin)) # request with safety margin - print(f"Requesting {requested_tokens} tokens from {self.model} (estimated {current_tokens - est_margin} prompt tokens with a safety margin of {est_margin} tokens)") + current_tokens = int( + sum([len(self.tok.encode(m["content"])) for m in messages]) + ) # estimate current usage + requested_tokens = int( + min(self.max_tokens, self.context_length - current_tokens - est_margin) + ) # request with safety margin + print( + f"Requesting {requested_tokens} tokens from {self.model} (estimated {current_tokens - est_margin} prompt tokens with a safety margin of {est_margin} tokens)" + ) self.in_tokens += current_tokens # request response @@ -167,7 +206,7 @@ def query(self, prompt: str, messages=None, end_when_error=False, max_retry=5, e if n_retry >= max_retry: break try: - print(f'[INFO] connecting to the LLM ({requested_tokens} tokens)...') + print(f"[INFO] connecting to the LLM ({requested_tokens} tokens)...") response = self.connect_openai( client=self.client, model=self.model, @@ -177,30 +216,34 @@ def query(self, prompt: str, messages=None, end_when_error=False, max_retry=5, e top_p=self.top_p, frequency_penalty=self.freq_penalty, presence_penalty=self.presence_penalty, - stop=self.stop + stop=self.stop, ) - llm_output = response.choices[0].message.content # response['choices'][0]['message']['content'] + llm_output = response.choices[ + 0 + ].message.content # response['choices'][0]['message']['content'] conn_success = True except Exception as e: - print(f'[ERROR] LLM error: {e}') + print(f"[ERROR] LLM error: {e}") if end_when_error: break if not conn_success: - raise ConnectionError(f'Failed to connect to the LLM after {max_retry} retries') - - response_tokens = len(self.tok.encode(llm_output)) # Estimate response tokens + raise ConnectionError( + f"Failed to connect to the LLM after {max_retry} retries" + ) + + response_tokens = len(self.tok.encode(llm_output)) # Estimate response tokens self.out_tokens += response_tokens return llm_output - + def get_tokens(self) -> tuple[int, int]: return self.in_tokens, self.out_tokens - + def reset_tokens(self): self.in_tokens = 0 self.out_tokens = 0 - - @override + + @override def valid_models(self) -> list[str]: """ List of valid model parameters for OpenAI. @@ -214,7 +257,7 @@ def valid_models(self) -> list[str]: "gpt-4", "gpt-4-32k", "gpt-4o", - "gpt-4o-mini" + "gpt-4o-mini", ] @@ -226,9 +269,9 @@ def __init__(self, model_path: str, max_tokens=4e3, temperature=0.01, top_p=0.9) self.top_p = top_p self.in_tokens = 0 self.out_tokens = 0 - + def _load_transformers(self): - + # attempt to import the `transformers` library try: import transformers @@ -237,7 +280,7 @@ def _load_transformers(self): "The 'transformers' library is required for HUGGING_FACE but is not installed. " "Install it using: `pip install transformers`." ) - + # attempt to import `AutoTokenizer` from `transformers` try: from transformers import AutoTokenizer @@ -246,7 +289,7 @@ def _load_transformers(self): "The 'transformers.AutoTokenizer' module is required but is not installed properly. " "Ensure that the 'transformers' library is installed correctly." ) - + # attempt to import the `torch` library try: import torch @@ -267,17 +310,19 @@ def _load_transformers(self): self.tokenizer = AutoTokenizer.from_pretrained(self.model_path) except OSError as e: # if model_path is not found, raise an error - raise ValueError(f"Model path '{self.model_path}' could not be found. Please ensure the model exists.") + raise ValueError( + f"Model path '{self.model_path}' could not be found. Please ensure the model exists." + ) except Exception as e: # catch any other exceptions and raise a generic error raise RuntimeError(f"An error occurred while loading the model: {str(e)}") - + # Retry decorator to handle retries on request @retry(tries=2, delay=60) def connect_huggingface(self, input, temperature, max_tokens, top_p, numSample): if self.model is None or self.tokenizer is None: self._load_transformers() - + if numSample > 1: responses = [] sequences = self.model( @@ -289,14 +334,14 @@ def connect_huggingface(self, input, temperature, max_tokens, top_p, numSample): return_full_text=False, temperature=temperature, top_p=top_p, - pad_token_id=self.tokenizer.eos_token_id + pad_token_id=self.tokenizer.eos_token_id, ) for seq in sequences: - response = seq['generated_text'] + response = seq["generated_text"] responses.append(response) return responses - + else: sequences = self.model( input, @@ -306,83 +351,63 @@ def connect_huggingface(self, input, temperature, max_tokens, top_p, numSample): return_full_text=False, temperature=temperature, top_p=top_p, - pad_token_id=self.tokenizer.eos_token_id + pad_token_id=self.tokenizer.eos_token_id, ) seq = sequences[0] - response = seq['generated_text'] - + response = seq["generated_text"] + return response - + @override def query(self, prompt: str, numSample=1, max_retry=3, est_margin=200) -> str: if prompt is None: raise ValueError("Prompt cannot be None") - + # Estimate current usage of tokens current_tokens = len(self.tokenizer.encode(prompt)) - requested_tokens = min(self.max_tokens, self.max_tokens - current_tokens - est_margin) - - print(f"Requesting {requested_tokens} tokens from {self.model} (estimated {current_tokens - est_margin} prompt tokens with a safety margin of {est_margin} tokens)") - + requested_tokens = min( + self.max_tokens, self.max_tokens - current_tokens - est_margin + ) + + print( + f"Requesting {requested_tokens} tokens from {self.model} (estimated {current_tokens - est_margin} prompt tokens with a safety margin of {est_margin} tokens)" + ) + # Retry logic for Hugging Face request n_retry = 0 conn_success = False while not conn_success and n_retry < max_retry: n_retry += 1 try: - print(f"[INFO] Connecting to Hugging Face model ({requested_tokens} tokens)...") + print( + f"[INFO] Connecting to Hugging Face model ({requested_tokens} tokens)..." + ) llm_output = self.connect_huggingface( input=prompt, temperature=self.temperature, max_tokens=requested_tokens, top_p=self.top_p, - numSample=numSample + numSample=numSample, ) conn_success = True except Exception as e: print(f"[ERROR] Hugging Face error: {e}") if n_retry >= max_retry: - raise ConnectionError(f"Failed to connect to the Hugging Face model after {max_retry} retries") + raise ConnectionError( + f"Failed to connect to the Hugging Face model after {max_retry} retries" + ) # Token management response_tokens = len(self.tokenizer.encode(llm_output)) self.out_tokens += response_tokens self.in_tokens += current_tokens - + return llm_output - + def get_tokens(self) -> tuple[int, int]: return self.in_tokens, self.out_tokens - + def reset_tokens(self): self.in_tokens = 0 self.out_tokens = 0 - - -if __name__ == '__main__': - - # test out OpenAI GPT - api_key = os.environ.get('OPENAI_API_KEY') - model_name = "gpt-4o-mini" - openai_llm = OPENAI(model=model_name, api_key=api_key) - - prompt = "What is the capital of France?" - response = openai_llm.query(prompt) - print(f"Response from {model_name}: {response}") - - # test out Huggingface model - parser = argparse.ArgumentParser(description="Define Parameters") - parser.add_argument('-test_dataset', action='store_true') # test custom prompt by default, set flag to run predictions over a specific dataset - parser.add_argument("--temp", type=float, default=0.01, help = "temperature for sampling") - parser.add_argument("--max_len", type=int, default=4e3, help = "max number of tokens in answer") - parser.add_argument("--num_sample", type=int, default=1, help = "number of answers to sample") - parser.add_argument("--model_path", type=str, default="/path/to/model", help = "path to llm") - args = parser.parse_args() - - huggingface_model = HUGGING_FACE(model_path=args.model_path, max_tokens=args.max_len, temperature=args.temp) - - prompt = "What is the capital of the United States?" - input = f"""<|begin_of_text|><|start_header_id|>system<|end_header_id|>{prompt}<|eot_id|>\n""" - response = huggingface_model.query(prompt=input, numSample=args.num_sample, max_retry=3) - print(f"Response: {response}") diff --git a/l2p/prompt_builder.py b/l2p/prompt_builder.py index 0f20774..52297b9 100644 --- a/l2p/prompt_builder.py +++ b/l2p/prompt_builder.py @@ -5,12 +5,19 @@ import os + class PromptBuilder: - def __init__(self, role: str=None, technique: str=None, examples: list=[], task: str=None): - self.role = role # role for LLM to follow (i.e. PDDL predicate constructor) - self.technique = technique # prompting technique (i.e. CoT) - self.examples = examples # n-shot examples for LLM to follow - self.task = task # dynamic placeholder given information to LLM + def __init__( + self, + role: str = None, + technique: str = None, + examples: list = [], + task: str = None, + ): + self.role = role # role for LLM to follow (i.e. PDDL predicate constructor) + self.technique = technique # prompting technique (i.e. CoT) + self.examples = examples # n-shot examples for LLM to follow + self.task = task # dynamic placeholder given information to LLM def set_role(self, role): """Sets the role for the LLM to perform task""" @@ -43,7 +50,6 @@ def set_task(self, task): """ self.task = task - def get_role(self): """Returns role of the prompt given""" return self.role @@ -60,7 +66,6 @@ def get_task(self): """Returns dynamic placeholder task prompt""" return self.task - def remove_role(self): """Removes role prompt""" self.role = None @@ -77,7 +82,6 @@ def remove_task(self): """Removes dynamic placeholder task prompt""" self.task = None - def generate_prompt(self): """Generates the whole prompt in proper format""" prompt = "" @@ -101,30 +105,3 @@ def generate_prompt(self): prompt += f"[TASK]:\nHere is the task to solve:\n{self.task}\n\n" return prompt.strip() - - -# USAGE EXAMPLE -if __name__ == "__main__": - def open_file(filepath): - with open(filepath, 'r') as file: - return file.read() - - role_file_path = 'data/prompt_templates/type_extraction/role.txt' - role_desc = open_file(role_file_path) - tech_file_path = 'data/prompt_templates/type_extraction/technique.txt' - tech_desc = open_file(tech_file_path) - task_file_path = 'data/prompt_templates/type_extraction/task.txt' - task_desc = open_file(task_file_path) - domain_file_path = 'data/domains/logistics.txt' - domain_desc = open_file(domain_file_path) - - # Directory where example files are stored - examples_dir = 'data/prompt_templates/type_extraction/examples/' - example_files = [f for f in os.listdir(examples_dir) if os.path.isfile(os.path.join(examples_dir, f))] - - # Read all example files - examples = [open_file(os.path.join(examples_dir, f)) for f in example_files] - - type_extraction_prompt = PromptBuilder(role_desc, tech_desc, examples, task_desc, domain_desc) - - print(type_extraction_prompt.get_prompt()) \ No newline at end of file diff --git a/l2p/task_builder.py b/l2p/task_builder.py index 6b4dd46..6e4136c 100644 --- a/l2p/task_builder.py +++ b/l2p/task_builder.py @@ -1,18 +1,20 @@ """ This file contains collection of functions for PDDL task generation purposes """ + from .utils import * from .llm_builder import LLM from .llm_builder import require_llm import time + class TaskBuilder: def __init__( - self, - objects: dict[str,str]=None, - initial: list[dict[str,str]]=None, - goal: list[dict[str,str]]=None - ): + self, + objects: dict[str, str] = None, + initial: list[dict[str, str]] = None, + goal: list[dict[str, str]] = None, + ): """ Initializes a task builder object @@ -21,21 +23,23 @@ def __init__( initial (list[dict[str,str]]): current initial states in model goal (list[dict[str,str]]): current goal states in model """ - - self.objects=objects - self.initial=initial - self.goal=goal + + self.objects = objects + self.initial = initial + self.goal = goal """Extract functions""" + @require_llm - def extract_objects(self, - model: LLM, + def extract_objects( + self, + model: LLM, problem_desc: str, prompt_template: str, - types: dict[str,str]=None, - predicates: list[Predicate]=None, - max_retries: int=3 - ) -> tuple[dict[str,str], str]: + types: dict[str, str] = None, + predicates: list[Predicate] = None, + max_retries: int = 3, + ) -> tuple[dict[str, str], str]: """ Extracts objects with given predicates in current model @@ -54,44 +58,48 @@ def extract_objects(self, """ # replace prompt placeholders - predicate_str = format_predicates(predicates) if predicates else "No predicates provided." + predicate_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) types_str = "\n".join(types) if types else "No types provided." - prompt_template = prompt_template.replace('{types}', types_str) - prompt_template = prompt_template.replace('{predicates}', predicate_str) - prompt_template = prompt_template.replace('{problem_desc}', problem_desc) - + prompt_template = prompt_template.replace("{types}", types_str) + prompt_template = prompt_template.replace("{predicates}", predicate_str) + prompt_template = prompt_template.replace("{problem_desc}", problem_desc) + # iterate through attempts in case of extraction failure for attempt in range(max_retries): try: model.reset_tokens() - llm_response = model.query(prompt=prompt_template) # get LLM response - + llm_response = model.query(prompt=prompt_template) # get LLM response + # extract respective types from response objects = parse_objects(llm_response) return objects, llm_response - + except Exception as e: - print(f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}...") - time.sleep(1) # add a delay before retrying - + print( + f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + ) + time.sleep(1) # add a delay before retrying + raise RuntimeError("Max retries exceeded. Failed to extract objects.") @require_llm def extract_initial_state( - self, - model: LLM, + self, + model: LLM, problem_desc: str, prompt_template: str, - types: dict[str,str]=None, - predicates: list[Predicate]=None, - objects: dict[str,str]=None, - initial: list[dict[str,str]]=None, - goal: list[dict[str,str]]=None, - max_retries: int=3 - ) -> tuple[list[dict[str,str]], str]: + types: dict[str, str] = None, + predicates: list[Predicate] = None, + objects: dict[str, str] = None, + initial: list[dict[str, str]] = None, + goal: list[dict[str, str]] = None, + max_retries: int = 3, + ) -> tuple[list[dict[str, str]], str]: """ Extracts initial states with given predicates, objects, and states in current model @@ -113,19 +121,25 @@ def extract_initial_state( """ # replace prompt placeholders - predicate_str = format_predicates(predicates) if predicates else "No predicates provided." + predicate_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) types_str = "\n".join(types) if types else "No types provided." - objects_str = self.format_objects(objects) if objects else "No objects provided." - initial_str = self.format_initial(initial) if initial else "No initial state provided." + objects_str = ( + self.format_objects(objects) if objects else "No objects provided." + ) + initial_str = ( + self.format_initial(initial) if initial else "No initial state provided." + ) goal_str = self.format_goal(goal) if goal else "No goal state provided." - prompt_template = prompt_template.replace('{types}', types_str) - prompt_template = prompt_template.replace('{predicates}', predicate_str) - prompt_template = prompt_template.replace('{objects}', objects_str) - prompt_template = prompt_template.replace('{initial_state}', initial_str) - prompt_template = prompt_template.replace('{goal_state}', goal_str) - prompt_template = prompt_template.replace('{problem_desc}', problem_desc) - + prompt_template = prompt_template.replace("{types}", types_str) + prompt_template = prompt_template.replace("{predicates}", predicate_str) + prompt_template = prompt_template.replace("{objects}", objects_str) + prompt_template = prompt_template.replace("{initial_state}", initial_str) + prompt_template = prompt_template.replace("{goal_state}", goal_str) + prompt_template = prompt_template.replace("{problem_desc}", problem_desc) + # iterate through attempts in case of extraction failure for attempt in range(max_retries): try: @@ -137,26 +151,28 @@ def extract_initial_state( initial = parse_initial(llm_response) return initial, llm_response - + except Exception as e: - print(f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}...") - time.sleep(1) # add a delay before retrying - + print( + f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + ) + time.sleep(1) # add a delay before retrying + raise RuntimeError("Max retries exceeded. Failed to extract initial states.") - + @require_llm def extract_goal_state( - self, - model: LLM, + self, + model: LLM, problem_desc: str, prompt_template: str, - types: dict[str,str]=None, - predicates: list[Predicate]=None, - objects: dict[str,str]=None, - initial: list[dict[str,str]]=None, - goal: list[dict[str,str]]=None, - max_retries: int=3 - ) -> tuple[list[dict[str,str]], str]: + types: dict[str, str] = None, + predicates: list[Predicate] = None, + objects: dict[str, str] = None, + initial: list[dict[str, str]] = None, + goal: list[dict[str, str]] = None, + max_retries: int = 3, + ) -> tuple[list[dict[str, str]], str]: """ Extracts goal states with given predicates, objects, and states in current model @@ -178,19 +194,25 @@ def extract_goal_state( """ # replace prompt placeholders - predicate_str = format_predicates(predicates) if predicates else "No predicates provided." + predicate_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) types_str = "\n".join(types) if types else "No types provided." - objects_str = self.format_objects(objects) if objects else "No objects provided." - initial_str = self.format_initial(initial) if initial else "No initial state provided." + objects_str = ( + self.format_objects(objects) if objects else "No objects provided." + ) + initial_str = ( + self.format_initial(initial) if initial else "No initial state provided." + ) goal_str = self.format_goal(goal) if goal else "No goal state provided." - prompt_template = prompt_template.replace('{types}', types_str) - prompt_template = prompt_template.replace('{predicates}', predicate_str) - prompt_template = prompt_template.replace('{objects}', objects_str) - prompt_template = prompt_template.replace('{initial_state}', initial_str) - prompt_template = prompt_template.replace('{goal_state}', goal_str) - prompt_template = prompt_template.replace('{problem_desc}', problem_desc) - + prompt_template = prompt_template.replace("{types}", types_str) + prompt_template = prompt_template.replace("{predicates}", predicate_str) + prompt_template = prompt_template.replace("{objects}", objects_str) + prompt_template = prompt_template.replace("{initial_state}", initial_str) + prompt_template = prompt_template.replace("{goal_state}", goal_str) + prompt_template = prompt_template.replace("{problem_desc}", problem_desc) + # iterate through attempts in case of extraction failure for attempt in range(max_retries): try: @@ -202,24 +224,26 @@ def extract_goal_state( goal = parse_goal(llm_response) return goal, llm_response - + except Exception as e: - print(f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}...") - time.sleep(1) # add a delay before retrying - + print( + f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + ) + time.sleep(1) # add a delay before retrying + raise RuntimeError("Max retries exceeded. Failed to extract goal states.") @require_llm def extract_task( - self, - model: LLM, + self, + model: LLM, problem_desc: str, - prompt_template: str, - types: dict[str,str]=None, - predicates: list[Predicate]=None, - actions: list[Action]=None, - max_retries: int=3 - ) -> tuple[dict[str,str], list[dict[str,str]], list[dict[str,str]], str]: + prompt_template: str, + types: dict[str, str] = None, + predicates: list[Predicate] = None, + actions: list[Action] = None, + max_retries: int = 3, + ) -> tuple[dict[str, str], list[dict[str, str]], list[dict[str, str]], str]: """ Extracts whole task specification in current model @@ -243,50 +267,67 @@ def extract_task( model.reset_tokens() # replace prompt placeholders - predicate_str = format_predicates(predicates) if predicates else "No predicates provided." + predicate_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) types_str = "\n".join(types) if types else "No types provided." - action_str = self.format_action(actions=actions) if actions else "No actions provided." + action_str = ( + self.format_action(actions=actions) if actions else "No actions provided." + ) + + prompt_template = prompt_template.replace("{types}", types_str) + prompt_template = prompt_template.replace("{predicates}", predicate_str) + prompt_template = prompt_template.replace("{actions}", action_str) + prompt_template = prompt_template.replace("{problem_desc}", problem_desc) - prompt_template = prompt_template.replace('{types}', types_str) - prompt_template = prompt_template.replace('{predicates}', predicate_str) - prompt_template = prompt_template.replace('{actions}', action_str) - prompt_template = prompt_template.replace('{problem_desc}', problem_desc) - # iterate through attempts in case of extraction failure for attempt in range(max_retries): try: model.reset_tokens() - + llm_response = model.query(prompt=prompt_template) + print(llm_response) + print("END OF LLM RESPONSE") + # extract respective types from response objects = parse_objects(llm_response) + print(objects) + print("----------") + initial = parse_initial(llm_response) + print(initial) + print("----------") + goal = parse_goal(llm_response) + print(goal) + print("----------") return objects, initial, goal, llm_response - + except Exception as e: - print(f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}...") - time.sleep(1) # add a delay before retrying - + print( + f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + ) + time.sleep(1) # add a delay before retrying + raise RuntimeError("Max retries exceeded. Failed to extract task.") @require_llm def extract_nl_conditions( - self, - model: LLM, + self, + model: LLM, problem_desc: str, prompt_template: str, - types: dict[str,str]=None, - predicates: list[Predicate]=None, - actions: list[Action]=None, - objects: dict[str,str]=None, - max_retries: int=3 - ) -> str: + types: dict[str, str] = None, + predicates: list[Predicate] = None, + actions: list[Action] = None, + objects: dict[str, str] = None, + max_retries: int = 3, + ) -> str: """ Extracts initial and goal states in natural language - + Args: model (LLM): LLM problem_desc (str): problem description @@ -303,69 +344,81 @@ def extract_nl_conditions( """ # replace prompt placeholders - predicate_str = format_predicates(predicates) if predicates else "No predicates provided." + predicate_str = ( + format_predicates(predicates) if predicates else "No predicates provided." + ) types_str = "\n".join(types) if types else "No types provided." - objects_str = self.format_objects(objects) if objects else "No objects provided." - action_str = self.format_action(actions=actions) if actions else "No actions provided." - - prompt_template = prompt_template.replace('{problem_desc}', problem_desc) - prompt_template = prompt_template.replace('{actions}', action_str) - prompt_template = prompt_template.replace('{types}', types_str) - prompt_template = prompt_template.replace('{predicates}', predicate_str) - prompt_template = prompt_template.replace('{objects}', objects_str) - + objects_str = ( + self.format_objects(objects) if objects else "No objects provided." + ) + action_str = ( + self.format_action(actions=actions) if actions else "No actions provided." + ) + + prompt_template = prompt_template.replace("{problem_desc}", problem_desc) + prompt_template = prompt_template.replace("{actions}", action_str) + prompt_template = prompt_template.replace("{types}", types_str) + prompt_template = prompt_template.replace("{predicates}", predicate_str) + prompt_template = prompt_template.replace("{objects}", objects_str) + # iterate through attempts in case of extraction failure for attempt in range(max_retries): try: model.reset_tokens() llm_response = model.query(prompt=prompt_template) - + return llm_response - + except Exception as e: - print(f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}...") - time.sleep(1) # add a delay before retrying - - raise RuntimeError("Max retries exceeded. Failed to extract NL task states.") + print( + f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + ) + time.sleep(1) # add a delay before retrying + raise RuntimeError("Max retries exceeded. Failed to extract NL task states.") """Delete function""" + def delete_objects(self, name): if self.objects is not None: - self.objects = {var: type_ for var, type_ in self.objects.items() if var != name} - + self.objects = { + var: type_ for var, type_ in self.objects.items() if var != name + } + def delete_initial_state(self, state: dict[str, str]): if self.initial is not None: self.initial = [s for s in self.initial if s != state] - + def delete_goal_state(self, state: dict[str, str]): if self.goal is not None: self.goal = [s for s in self.goal if s != state] - """Set functions""" - def set_objects(self, objects: dict[str,str]): + + def set_objects(self, objects: dict[str, str]): self.set_objects = objects - def set_initial(self, initial: dict[str,str]): + def set_initial(self, initial: dict[str, str]): self.set_initial = initial def set_goal(self, goal: str): self.goal = goal """Get functions""" - def get_objects(self) -> dict[str,str]: + + def get_objects(self) -> dict[str, str]: return self.objects - def get_initial(self) -> dict[str,str]: + def get_initial(self) -> dict[str, str]: return self.initial - + def get_goal(self) -> str: return self.goal - - def generate_task(self, domain: str, problem: str, objects: str, initial: str, goal: str): + def generate_task( + self, domain: str, problem: str, objects: str, initial: str, goal: str + ): # Write problem file desc = "(define\n" desc += f" (problem {problem})\n" @@ -374,13 +427,15 @@ def generate_task(self, domain: str, problem: str, objects: str, initial: str, g desc += f" (:init\n{indent(initial)}\n )\n\n" desc += f" (:goal\n{indent(goal)}\n )\n\n" desc += ")" - desc = desc.replace("AND","and").replace("OR","or") + desc = desc.replace("AND", "and").replace("OR", "or") return desc - + def format_action(self, actions: list[Action]) -> str: desc = "" for action in actions: - param_str = "\n".join([f"{name} - {type}" for name, type in action['parameters'].items()]) # name includes ? + param_str = "\n".join( + [f"{name} - {type}" for name, type in action["parameters"].items()] + ) # name includes ? desc += f"(:action {action['name']}\n" desc += f" :parameters (\n{indent(param_str,2)}\n )\n" desc += f" :precondition\n{indent(action['preconditions'],2)}\n" @@ -388,30 +443,36 @@ def format_action(self, actions: list[Action]) -> str: desc += ")\n" return desc - def format_objects(self, objects: dict[str,str]) -> str: + def format_objects(self, objects: dict[str, str]) -> str: objects = "\n".join([f"{obj} - {type}" for obj, type in objects.items()]) return objects - def format_initial(self, initial_states: list[dict[str,str]]) -> str: - inner_str = [f"({state['name']} {' '.join(state['params'])})" for state in initial_states] # The main part of each predicate - full_str = [f"(not {inner})" if state["neg"] else inner for state, inner in zip(initial_states, inner_str)] # add `not` if needed - initial_states_str = "\n".join(full_str) # combine the states into a single string - + def format_initial(self, initial_states: list[dict[str, str]]) -> str: + inner_str = [ + f"({state['name']} {' '.join(state['params'])})" for state in initial_states + ] # The main part of each predicate + full_str = [ + f"(not {inner})" if state["neg"] else inner + for state, inner in zip(initial_states, inner_str) + ] # add `not` if needed + initial_states_str = "\n".join( + full_str + ) # combine the states into a single string + return initial_states_str - - def format_goal(self, goal_states: list[dict[str,str]]) -> str: + + def format_goal(self, goal_states: list[dict[str, str]]) -> str: goal_states_str = "(AND \n" # loop through each dictionary in the list for item in goal_states: # extract the name and parameters from the dictionary - name = item['name'] - params = " ".join(item['params']) - goal_states_str += f" ({name} {params}) \n" # append the predicate in the desired format + name = item["name"] + params = " ".join(item["params"]) + goal_states_str += ( + f" ({name} {params}) \n" # append the predicate in the desired format + ) goal_states_str += ")" - - return goal_states_str -if __name__ == "__main__": - pass \ No newline at end of file + return goal_states_str diff --git a/l2p/utils/pddl_parser.py b/l2p/utils/pddl_parser.py index 1ae183b..dc25711 100644 --- a/l2p/utils/pddl_parser.py +++ b/l2p/utils/pddl_parser.py @@ -7,61 +7,73 @@ from copy import deepcopy import re, ast, json + def parse_params(llm_output): """ Parses parameters from LLM into Python format (refer to example templates to see how these parameters should be formatted in LLM response). - + LLM output header should contain '### Parameters' along with structured content. """ params_info = OrderedDict() - params_heading = llm_output.split('Parameters')[1].strip().split('###')[0] + params_heading = llm_output.split("Parameters")[1].strip().split("###")[0] params_str = combine_blocks(params_heading) params_raw = [] - for line in params_str.split('\n'): - if line.strip() == '' or ('.' not in line and not line.strip().startswith('-')): - print(f"[WARNING] checking param object types - empty line or not a valid line: '{line}'") + for line in params_str.split("\n"): + if line.strip() == "" or ("." not in line and not line.strip().startswith("-")): + print( + f"[WARNING] checking param object types - empty line or not a valid line: '{line}'" + ) continue - if not (line.split('.')[0].strip().isdigit() or line.startswith('-')): + if not (line.split(".")[0].strip().isdigit() or line.startswith("-")): print(f"[WARNING] checking param object types - not a valid line: '{line}'") continue try: params_raw.append(line.strip()) - p_info = [e for e in line.split(':')[0].split(' ') if e != ''] + p_info = [e for e in line.split(":")[0].split(" ") if e != ""] param_name, param_type = p_info[1].strip(" `"), p_info[3].strip(" `") params_info[param_name] = param_type except Exception: - print(f'[WARNING] checking param object types - fail to parse: {line}') + print(f"[WARNING] checking param object types - fail to parse: {line}") break return params_info, params_raw + def parse_new_predicates(llm_output) -> list[Predicate]: """ Parses new predicates from LLM into Python format (refer to example templates to see how these predicates should be formatted in LLM response). - + LLM output header should contain '### New Predicates' along with structured content. """ new_predicates = list() try: - predicate_heading = llm_output.split('New Predicates\n')[1].strip().split('###')[0] + predicate_heading = ( + llm_output.split("New Predicates\n")[1].strip().split("###")[0] + ) except: - raise Exception("Could not find the 'New Predicates' section in the output. Provide the entire response, including all headings even if some are unchanged.") + raise Exception( + "Could not find the 'New Predicates' section in the output. Provide the entire response, including all headings even if some are unchanged." + ) predicate_output = combine_blocks(predicate_heading) - for p_line in predicate_output.split('\n'): - if ('.' not in p_line or not p_line.split('.')[0].strip().isdigit()) and not (p_line.startswith('-') or p_line.startswith('(')): + for p_line in predicate_output.split("\n"): + if ("." not in p_line or not p_line.split(".")[0].strip().isdigit()) and not ( + p_line.startswith("-") or p_line.startswith("(") + ): if len(p_line.strip()) > 0: print(f'[WARNING] unable to parse the line: "{p_line}"') continue - predicate_info = p_line.split(': ')[0].strip(" 1234567890.(-)`").split(' ') + predicate_info = p_line.split(": ")[0].strip(" 1234567890.(-)`").split(" ") predicate_name = predicate_info[0] - predicate_desc = p_line.split(': ')[1].strip() if ": " in p_line else '' + predicate_desc = p_line.split(": ")[1].strip() if ": " in p_line else "" # get the predicate type info if len(predicate_info) > 1: predicate_type_info = predicate_info[1:] - predicate_type_info = [l.strip(" ()`") for l in predicate_type_info if l.strip(" ()`")] + predicate_type_info = [ + l.strip(" ()`") for l in predicate_type_info if l.strip(" ()`") + ] else: predicate_type_info = [] params = OrderedDict() @@ -70,61 +82,77 @@ def parse_new_predicates(llm_output) -> list[Predicate]: for p in predicate_type_info: if next_is_type: - if p.startswith('?'): - print(f"[WARNING] `{p}` is not a valid type for a variable, but it is being treated as one. Should be checked by syntax check later.") + if p.startswith("?"): + print( + f"[WARNING] `{p}` is not a valid type for a variable, but it is being treated as one. Should be checked by syntax check later." + ) for up in upcoming_params: params[up] = p next_is_type = False upcoming_params = [] - elif p == '-': + elif p == "-": next_is_type = True - elif p.startswith('?'): - upcoming_params.append(p) # the next type will be for this variable + elif p.startswith("?"): + upcoming_params.append(p) # the next type will be for this variable else: - print(f"[WARNING] `{p}` is not corrrectly formatted. Assuming it's a variable name.") + print( + f"[WARNING] `{p}` is not corrrectly formatted. Assuming it's a variable name." + ) upcoming_params.append(f"?{p}") if next_is_type: - print(f"[WARNING] The last type is not specified for `{p_line}`. Undefined are discarded.") + print( + f"[WARNING] The last type is not specified for `{p_line}`. Undefined are discarded." + ) if len(upcoming_params) > 0: - print(f"[WARNING] The last {len(upcoming_params)} is not followed by a type name for {upcoming_params}. These are discarded") + print( + f"[WARNING] The last {len(upcoming_params)} is not followed by a type name for {upcoming_params}. These are discarded" + ) # generate a clean version of the predicate clean = f"({predicate_name} {' '.join([f'{k} - {v}' for k, v in params.items()])}): {predicate_desc}" # drop the index/dot - p_line = p_line.strip(" 1234567890.-`") - new_predicates.append({ - 'name': predicate_name, - 'desc': predicate_desc, - 'raw': p_line, - 'params': params, - 'clean': clean, - }) + p_line = p_line.strip(" 1234567890.-`") + new_predicates.append( + { + "name": predicate_name, + "desc": predicate_desc, + "raw": p_line, + "params": params, + "clean": clean, + } + ) return new_predicates + def parse_predicates(all_predicates): """ This function assumes the predicate definitions adhere to PDDL grammar. + Assigns `params` to the predicate arguments properly. This should be run + after retrieving a predicate list to ensure predicates are set correctly. """ all_predicates = deepcopy(all_predicates) for i, pred in enumerate(all_predicates): - if 'params' in pred: + if "params" in pred: continue - pred_def = pred['raw'].split(': ')[0] + pred_def = pred["raw"].split(": ")[0] pred_def = pred_def.strip(" ()`") # drop any leading/strange formatting - split_predicate = pred_def.split(' ')[1:] # discard the predicate name - split_predicate = [e for e in split_predicate if e != ''] + split_predicate = pred_def.split(" ")[1:] # discard the predicate name + split_predicate = [e for e in split_predicate if e != ""] - pred['params'] = OrderedDict() + pred["params"] = OrderedDict() for j, p in enumerate(split_predicate): if j % 3 == 0: - assert '?' in p, f'invalid predicate definition: {pred_def}' - assert split_predicate[j+1] == '-', f'invalid predicate definition: {pred_def}' - param_name, param_obj_type = p, split_predicate[j+2] - pred['params'][param_name] = param_obj_type + assert "?" in p, f"invalid predicate definition: {pred_def}" + assert ( + split_predicate[j + 1] == "-" + ), f"invalid predicate definition: {pred_def}" + param_name, param_obj_type = p, split_predicate[j + 2] + pred["params"][param_name] = param_obj_type return all_predicates + def parse_action(llm_response: str, action_name: str) -> Action: """ Parse an action from a given LLM output. @@ -138,15 +166,35 @@ def parse_action(llm_response: str, action_name: str) -> Action: """ parameters, _ = parse_params(llm_response) try: - preconditions = llm_response.split("Preconditions\n")[1].split("###")[0].split("```")[1].strip(" `\n") + preconditions = ( + llm_response.split("Preconditions\n")[1] + .split("###")[0] + .split("```")[1] + .strip(" `\n") + ) except: - raise Exception("Could not find the 'Preconditions' section in the output. Provide the entire response, including all headings even if some are unchanged.") + raise Exception( + "Could not find the 'Preconditions' section in the output. Provide the entire response, including all headings even if some are unchanged." + ) try: - effects = llm_response.split("Effects\n")[1].split("###")[0].split("```")[1].strip(" `\n") + effects = ( + llm_response.split("Effects\n")[1] + .split("###")[0] + .split("```")[1] + .strip(" `\n") + ) except: - raise Exception("Could not find the 'Effects' section in the output. Provide the entire response, including all headings even if some are unchanged.") - return {"name": action_name, "parameters": parameters, "preconditions": preconditions, "effects": effects} - + raise Exception( + "Could not find the 'Effects' section in the output. Provide the entire response, including all headings even if some are unchanged." + ) + return { + "name": action_name, + "parameters": parameters, + "preconditions": preconditions, + "effects": effects, + } + + def parse_objects(llm_response: str) -> dict[str, str]: """ Extract objects from LLM response and returns dictionary string pairs object(name, type) @@ -157,18 +205,26 @@ def parse_objects(llm_response: str) -> dict[str, str]: Returns: - dict[str,str]: objects """ - + objects_head = extract_heading(llm_response, "OBJECTS") objects_raw = combine_blocks(objects_head) - objects_clean = clear_comments(objects_raw, comments=[':','//','#',';','(']) # Remove comments - objects = {obj.split(" - ")[0].strip(" `"): obj.split(" - ")[1].strip(" `").lower() for obj in objects_clean.split("\n") if obj.strip()} + objects_clean = clear_comments( + text=objects_raw, comments=[":", "//", "#", ";", "(", ")"] + ) # Remove comments + + objects = { + obj.split(" - ")[0].strip(" `"): obj.split(" - ")[1].strip(" `").lower() + for obj in objects_clean.split("\n") + if obj.strip() + } return objects -def parse_initial(llm_response: str) -> list[dict[str,str]]: + +def parse_initial(llm_response: str) -> list[dict[str, str]]: """ Extracts state (PDDL-init) from LLM response and returns it as a list of dict strings - + Args: llm_response (str): The LLM output. @@ -177,17 +233,23 @@ def parse_initial(llm_response: str) -> list[dict[str,str]]: """ state_head = extract_heading(llm_response, "INITIAL") state_raw = combine_blocks(state_head) + print("STATE RAW") + print(state_raw) + print("END STATE RAW") + state_clean = clear_comments(state_raw) states = [] for line in state_clean.split("\n"): line = line.strip("- `()") - if not line: # Skip empty lines + if not line: # Skip empty lines continue name = line.split(" ")[0] if name == "not": neg = True - name = line.split(" ")[1].strip("()") # Remove the `not` and the parentheses + name = line.split(" ")[1].strip( + "()" + ) # Remove the `not` and the parentheses params = line.split(" ")[2:] else: neg = False @@ -196,10 +258,11 @@ def parse_initial(llm_response: str) -> list[dict[str,str]]: return states -def parse_goal(llm_response: str) -> list[dict[str,str]]: + +def parse_goal(llm_response: str) -> list[dict[str, str]]: """ Extracts goal (PDDL-goal) from LLM response and returns it as a string - + Args: llm_response (str): The LLM output. @@ -209,53 +272,68 @@ def parse_goal(llm_response: str) -> list[dict[str,str]]: goal_head = extract_heading(llm_response, "GOAL") if goal_head.count("```") != 2: - raise ValueError("Could not find exactly one block in the goal section of the LLM output. The goal has to be specified in a single block and as valid PDDL using the `and` and `not` operators. Likely this is caused by a too long response and limited context length. If so, try to shorten the message and exclude objects which aren't needed for the task.") - goal_raw = goal_head.split("```")[1].strip() # Only a single block in the goal + raise ValueError( + "Could not find exactly one block in the goal section of the LLM output. The goal has to be specified in a single block and as valid PDDL using the `and` and `not` operators. Likely this is caused by a too long response and limited context length. If so, try to shorten the message and exclude objects which aren't needed for the task." + ) + goal_raw = goal_head.split("```")[1].strip() # Only a single block in the goal goal_clean = clear_comments(goal_raw) - goal_pure = goal_clean.replace("and", "").replace("AND", "").replace("not", "").replace("NOT", "") + goal_pure = ( + goal_clean.replace("and", "") + .replace("AND", "") + .replace("not", "") + .replace("NOT", "") + ) goal = [] for line in goal_pure.split("\n"): line = line.strip(" ()") - if not line: # Skip empty lines + if not line: # Skip empty lines continue name = line.split(" ")[0] params = line.split(" ")[1:] if len(line.split(" ")) > 1 else [] goal.append({"name": name, "params": params}) - return goal # Since the goal uses `and` and `not` recombining it is difficult + return goal # Since the goal uses `and` and `not` recombining it is difficult -def prune_types(types: dict[str,str], predicates: list[Predicate], actions: list[Action]) -> dict[str,str]: - """ - Prune types that are not used in any predicate or action. +def prune_types( + types: dict[str, str], predicates: list[Predicate], actions: list[Action] +) -> dict[str, str]: + """ + Prune types that are not used in any predicate or action. - Args: - types (list[str]): A list of types. - predicates (list[Predicate]): A list of predicates. - actions (list[Action]): A list of actions. + Args: + types (list[str]): A list of types. + predicates (list[Predicate]): A list of predicates. + actions (list[Action]): A list of actions. - Returns: - list[str]: The pruned list of types. - """ + Returns: + list[str]: The pruned list of types. + """ - used_types = {} - for type in types: - for pred in predicates: - if type.split(' ')[0] in pred['params'].values(): + used_types = {} + for type in types: + for pred in predicates: + if type.split(" ")[0] in pred["params"].values(): + used_types[type] = types[type] + break + else: + for action in actions: + if type.split(" ")[0] in action["parameters"].values(): used_types[type] = types[type] break - else: - for action in actions: - if type.split(' ')[0] in action['parameters'].values(): - used_types[type] = types[type] - break - if type.split(' ')[0] in action['preconditions'] or type.split(' ')[0] in action['effects']: # If the type is included in a "forall" or "exists" statement - used_types[type] = types[type] - break - return used_types - -def prune_predicates(predicates: list[Predicate], actions: list[Action]) -> list[Predicate]: + if ( + type.split(" ")[0] in action["preconditions"] + or type.split(" ")[0] in action["effects"] + ): # If the type is included in a "forall" or "exists" statement + used_types[type] = types[type] + break + return used_types + + +def prune_predicates( + predicates: list[Predicate], actions: list[Action] +) -> list[Predicate]: """ Remove predicates that are not used in any action. @@ -271,13 +349,13 @@ def prune_predicates(predicates: list[Predicate], actions: list[Action]) -> list for pred in predicates: for action in actions: - # Add a space or a ")" to avoid partial matches + # Add a space or a ")" to avoid partial matches names = [f"{pred['name']} ", f"{pred['name']})"] for name in names: - if name in action['preconditions'] or name in action['effects']: - if pred['name'] not in seen_predicate_names: + if name in action["preconditions"] or name in action["effects"]: + if pred["name"] not in seen_predicate_names: used_predicates.append(pred) - seen_predicate_names.add(pred['name']) + seen_predicate_names.add(pred["name"]) break return used_predicates @@ -286,17 +364,26 @@ def prune_predicates(predicates: list[Predicate], actions: list[Action]) -> list def extract_heading(llm_output: str, heading: str): """Extract the text between the heading and the next second level heading in the LLM output.""" if heading not in llm_output: - print("#"*10, "LLM Output", "#"*10) + print("#" * 10, "LLM Output", "#" * 10) print(llm_output) - print("#"*30) - raise ValueError(f"Could not find heading {heading} in the LLM output. Likely this is caused by a too long response and limited context length. If so, try to shorten the message and exclude objects which aren't needed for the task.") - heading_str = llm_output.split(heading)[1].split("\n### ")[0].strip() # Get the text between the heading and the next heading + print("#" * 30) + raise ValueError( + f"Could not find heading {heading} in the LLM output. Likely this is caused by a too long response and limited context length. If so, try to shorten the message and exclude objects which aren't needed for the task." + ) + heading_str = ( + llm_output.split(heading)[1].split("\n### ")[0].strip() + ) # Get the text between the heading and the next heading return heading_str -def convert_to_dict(llm_response: str) -> dict[str,str]: + +def convert_to_dict(llm_response: str) -> dict[str, str]: """Converts string into Python dictionary format.""" - dict_pattern = re.compile(r'{.*}', re.DOTALL) # regular expression to find the JSON-like dictionary structure - match = dict_pattern.search(llm_response) # search for the pattern in the llm_response + dict_pattern = re.compile( + r"{.*}", re.DOTALL + ) # regular expression to find the JSON-like dictionary structure + match = dict_pattern.search( + llm_response + ) # search for the pattern in the llm_response # safely evaluate the string to convert it into a Python dictionary if match: @@ -311,34 +398,49 @@ def convert_to_dict(llm_response: str) -> dict[str,str]: print("No dictionary found in the llm_response.") return None -def clear_comments(text: str, comments = [':','//','#',';']) -> str: + +def clear_comments(text: str, comments=[":", "//", "#", ";"]) -> str: """Remove comments from the text.""" for comment in comments: text = "\n".join([line.split(comment)[0] for line in text.split("\n")]) return text + def combine_blocks(heading_str: str): """Combine the inside of blocks from the heading string into a single string.""" possible_blocks = heading_str.split("```") - blocks = [possible_blocks[i] for i in range(1, len(possible_blocks), 2)] # obtain string between ``` + blocks = [ + possible_blocks[i] for i in range(1, len(possible_blocks), 2) + ] # obtain string between ``` + combined = "\n".join(blocks) - return combined.replace("\n\n", "\n").strip() # remove leading/trailing whitespace and internal empty lines + + return combined.replace( + "\n\n", "\n" + ).strip() # remove leading/trailing whitespace and internal empty lines + def format_dict(dictionary): """Formats dictionary in JSON format easier for readability""" return json.dumps(dictionary, indent=4) -def format_types(type_hierarchy: dict[str,str]) -> dict[str,str]: + +def format_types(type_hierarchy: dict[str, str]) -> dict[str, str]: """Formats Python dictionary hierarchy to PDDL formatted dictionary""" + def process_node(node, parent_type=None): current_type = list(node.keys())[0] description = node[current_type] parent_type = parent_type if parent_type else current_type - name = f"{current_type} - {parent_type}" if current_type != parent_type else f"{current_type}" + name = ( + f"{current_type} - {parent_type}" + if current_type != parent_type + else f"{current_type}" + ) desc = f"; {description}" - + result[name] = desc for child in node.get("children", []): @@ -348,6 +450,7 @@ def process_node(node, parent_type=None): process_node(type_hierarchy) return result + def format_predicates(predicates: list[Predicate]) -> str: """Formats list of predicates easier for readability""" if not predicates: @@ -356,18 +459,8 @@ def format_predicates(predicates: list[Predicate]) -> str: f"{i + 1}. {pred['name']}: {pred.get('desc', 'No description provided') or 'No description provided'}" for i, pred in enumerate(predicates) ) - + + def indent(string: str, level: int = 2): """Indent string helper function to format PDDL domain/task""" return " " * level + string.replace("\n", f"\n{' ' * level}") - - -if __name__ == '__main__': - string = """ - ### Action Parameters\n - ```\n - - ?v - vehicle: The vehicle travelling\n - - ?from - location: The location travelling from\n - - ?to - location: The location travelling to - ``` - """ \ No newline at end of file diff --git a/l2p/utils/pddl_planner.py b/l2p/utils/pddl_planner.py index 694df62..68b0019 100644 --- a/l2p/utils/pddl_planner.py +++ b/l2p/utils/pddl_planner.py @@ -25,6 +25,7 @@ DRIVER_INPUT_ERROR = 36 DRIVER_UNSUPPORTED = 37 + class FastDownward: def run_fast_downward(self, domain_file, problem_file, plan_file="sas_plan"): @@ -35,18 +36,20 @@ def run_fast_downward(self, domain_file, problem_file, plan_file="sas_plan"): result = subprocess.run( [downward_path, "--alias", "lama-first", domain_file, problem_file], capture_output=True, - text=True + text=True, ) exitcodes = [result.returncode] if result.returncode == SUCCESS: # Planning succeeded - with open(plan_file, 'w') as f: + with open(plan_file, "w") as f: f.write(result.stdout) print("Planning succeeded!") - print("All run components successfully terminated (translator: completed, search: found a plan, validate: validated a plan)") - + print( + "All run components successfully terminated (translator: completed, search: found a plan, validate: validated a plan)" + ) + # Extract the plan steps from the output plan_output = self.extract_plan_steps(result.stdout) if plan_output: @@ -62,7 +65,7 @@ def run_fast_downward(self, domain_file, problem_file, plan_file="sas_plan"): return False, str(e) def extract_plan_steps(self, output): - plan_steps = re.findall(r'^\w+.*\(.*\)', output, re.MULTILINE) + plan_steps = re.findall(r"^\w+.*\(.*\)", output, re.MULTILINE) return "\n".join(plan_steps) def handle_error(self, exitcode, plan_found): @@ -119,7 +122,9 @@ def generate_portfolio_exitcode(self, exitcodes): print("Exit codes: {}".format(exitcodes)) exitcodes = set(exitcodes) - unrecoverable_codes = [code for code in exitcodes if self.is_unrecoverable(code)] + unrecoverable_codes = [ + code for code in exitcodes if self.is_unrecoverable(code) + ] # There are unrecoverable exit codes. if unrecoverable_codes: @@ -154,11 +159,3 @@ def generate_portfolio_exitcode(self, exitcodes): return (SEARCH_OUT_OF_TIME, False) assert False, "Error: Unhandled exit codes: {}".format(exitcodes) - -if __name__ == "__main__": - - planner = FastDownward() - - domain = "paper_reconstructions/proc2pddl/results/domain.pddl" - problem = "paper_reconstructions/proc2pddl/results/problems/problem-start-fire.pddl" - planner.run_fast_downward(domain, problem) diff --git a/l2p/utils/pddl_types.py b/l2p/utils/pddl_types.py index 83ec62b..2f04322 100644 --- a/l2p/utils/pddl_types.py +++ b/l2p/utils/pddl_types.py @@ -6,8 +6,11 @@ from collections import OrderedDict from dataclasses import dataclass -ParameterList = NewType('ParameterList', OrderedDict[str, str]) # {param_name: param_type} -ObjectList = NewType('ObjectList', dict[str, str]) # {obj_name: obj_type} +ParameterList = NewType( + "ParameterList", OrderedDict[str, str] +) # {param_name: param_type} +ObjectList = NewType("ObjectList", dict[str, str]) # {obj_name: obj_type} + class Predicate(TypedDict): name: str @@ -16,13 +19,15 @@ class Predicate(TypedDict): params: ParameterList clean: str + class Action(TypedDict): name: str raw: str parameters: ParameterList preconditions: str effects: str - + + # Domain details data class including predicates and actions @dataclass class DomainDetails: @@ -34,18 +39,20 @@ class DomainDetails: predicates: List[Predicate] # List of Predicate objects actions: List[Action] # List of Action objects + # Problem details data class @dataclass class ProblemDetails: name: str problem_desc: str problem_pddl: str - objects: tuple[dict[str,str], str] - initial: tuple[dict[str,str], str] - goal: tuple[dict[str,str], str] + objects: tuple[dict[str, str], str] + initial: tuple[dict[str, str], str] + goal: tuple[dict[str, str], str] + # Plan details data class @dataclass class PlanDetails: plan_pddl: str - plan_nl: str \ No newline at end of file + plan_nl: str diff --git a/l2p/utils/pddl_validator.py b/l2p/utils/pddl_validator.py index 7db1ec4..bd6f997 100644 --- a/l2p/utils/pddl_validator.py +++ b/l2p/utils/pddl_validator.py @@ -6,190 +6,222 @@ from .pddl_parser import parse_params, parse_new_predicates, parse_predicates from .pddl_types import Predicate + class SyntaxValidator: # PARAMETER CHECKS - def validate_params(self, parameters: OrderedDict, types: dict[str,str]) -> tuple[bool,str]: + def validate_params( + self, parameters: OrderedDict, types: dict[str, str] + ) -> tuple[bool, str]: """Checks whether a PDDL action parameter contains types found in object types.""" - + for param_name in parameters: param_type = parameters[param_name] if not any(param_type in t for t in types.keys()): feedback_msg = f'There is an invalid object type `{param_type}` for the parameter {param_name} not found in the types {types.keys()}. If you need to use a new type, you can emulate it with an "is_{{type}} ?o - object" precondition. Please revise the PDDL model to fix this error.' return False, feedback_msg - + feedback_msg = "PASS: All parameter types found in object types." return True, feedback_msg # PREDICATE CHECKS - - def validate_types_predicates(self, predicates: list[dict], types: dict[str, str]) -> tuple[bool, str]: + + def validate_types_predicates( + self, predicates: list[dict], types: dict[str, str] + ) -> tuple[bool, str]: """Check if predicate name is found within any type definitions""" invalid_predicates = list() for pred in predicates: - pred_name = pred['name'].lower() - + pred_name = pred["name"].lower() + for type_key in types.keys(): # extract the actual type name, disregarding hierarchical or descriptive parts - type_name = type_key.split(' - ')[0].strip().lower() - + type_name = type_key.split(" - ")[0].strip().lower() + # check if the predicate name is exactly the same as the type name if pred_name == type_name: invalid_predicates.append(pred_name) if invalid_predicates: - feedback_msg = 'ERROR: The following predicate(s) have the same name(s) as existing object types:' + feedback_msg = "ERROR: The following predicate(s) have the same name(s) as existing object types:" for pred_i, pred_name in enumerate(invalid_predicates): - feedback_msg += f'\n{pred_i + 1}. {pred_name}' - feedback_msg += '\nPlease rename these predicates.' + feedback_msg += f"\n{pred_i + 1}. {pred_name}" + feedback_msg += "\nPlease rename these predicates." return False, feedback_msg feedback_msg = "PASS: All predicate names are unique to object type names" return True, feedback_msg - def validate_duplicate_predicates(self, curr_predicates: list[Predicate], new_predicates: list[Predicate]) -> tuple[bool,str]: + def validate_duplicate_predicates( + self, curr_predicates: list[Predicate], new_predicates: list[Predicate] + ) -> tuple[bool, str]: """Checks if predicates have the same name but different parameters""" - - curr_pred_dict = {pred['name'].lower(): pred for pred in curr_predicates} - + + curr_pred_dict = {pred["name"].lower(): pred for pred in curr_predicates} + duplicated_predicates = list() for new_pred in new_predicates: # check if the name is already used - if new_pred['name'].lower() in curr_pred_dict: - - curr = curr_pred_dict[new_pred['name'].lower()] - - if len(curr['params']) != len(new_pred['params']) or any([t1 != t2 for t1, t2 in zip(curr['params'], new_pred['params'])]): + if new_pred["name"].lower() in curr_pred_dict: + + curr = curr_pred_dict[new_pred["name"].lower()] + + if len(curr["params"]) != len(new_pred["params"]) or any( + [t1 != t2 for t1, t2 in zip(curr["params"], new_pred["params"])] + ): # if the params are the same, then it's not a problem - duplicated_predicates.append((new_pred['raw'], curr_pred_dict[new_pred['name'].lower()]['raw'])) + duplicated_predicates.append( + ( + new_pred["raw"], + curr_pred_dict[new_pred["name"].lower()]["raw"], + ) + ) if len(duplicated_predicates) > 0: - feedback_msg = f'The following predicate(s) have the same name(s) as existing predicate(s):' + feedback_msg = f"The following predicate(s) have the same name(s) as existing predicate(s):" for pred_i, duplicated_pred_info in enumerate(duplicated_predicates): new_pred_full, existing_pred_full = duplicated_pred_info feedback_msg += f'\n{pred_i + 1}. {new_pred_full.replace(":", ",")}; existing predicate with the same name: {existing_pred_full.replace(":", ",")}' - feedback_msg += '\n\nYou should reuse existing predicates whenever possible. If you are reusing existing predicate(s), you shouldn\'t list them under \'New Predicates\'. If existing predicates are not enough and you are devising new predicate(s), please use names that are different from existing ones.' - feedback_msg += '\n\nPlease revise the PDDL model to fix this error.\n\n' - feedback_msg += 'Parameters:' + feedback_msg += "\n\nYou should reuse existing predicates whenever possible. If you are reusing existing predicate(s), you shouldn't list them under 'New Predicates'. If existing predicates are not enough and you are devising new predicate(s), please use names that are different from existing ones." + feedback_msg += "\n\nPlease revise the PDDL model to fix this error.\n\n" + feedback_msg += "Parameters:" return False, feedback_msg feedback_msg = "PASS: All predicates are unique to each other." return True, feedback_msg - def validate_format_predicates(self, predicates: list[dict], types: dict[str, str]) -> tuple[bool, str]: + def validate_format_predicates( + self, predicates: list[dict], types: dict[str, str] + ) -> tuple[bool, str]: """Checks for any PDDL syntax found within predicates""" - + all_invalid_params = [] for pred in predicates: - pred_def = pred['raw'].split(': ')[0] - pred_def = pred_def.strip(" ()`") # discard parentheses and similar - split_predicate = pred_def.split(' ')[1:] # discard the predicate name - split_predicate = [e for e in split_predicate if e != ''] + pred_def = pred["raw"].split(": ")[0] + pred_def = pred_def.strip(" ()`") # discard parentheses and similar + split_predicate = pred_def.split(" ")[1:] # discard the predicate name + split_predicate = [e for e in split_predicate if e != ""] for i, p in enumerate(split_predicate): if i % 3 == 0: - if '?' not in p: - feedback_msg = f'There are syntax errors in the definition of the new predicate {pred_def}. Check for any missing \'?\' variables, or missing type declarations. Please revise its definition and output the entire PDDL action model again. Note that you need to strictly follow the syntax of PDDL.' + if "?" not in p: + feedback_msg = f"There are syntax errors in the definition of the new predicate {pred_def}. Check for any missing '?' variables, or missing type declarations. Please revise its definition and output the entire PDDL action model again. Note that you need to strictly follow the syntax of PDDL." return False, feedback_msg else: - if i + 1 >= len(split_predicate) or split_predicate[i+1] != '-': - feedback_msg = f'There are syntax errors in the definition of the new predicate {pred_def}. Please revise its definition and output the entire PDDL action model again. Note that you need to define the object type of each parameter and strictly follow the syntax of PDDL.' + if ( + i + 1 >= len(split_predicate) + or split_predicate[i + 1] != "-" + ): + feedback_msg = f"There are syntax errors in the definition of the new predicate {pred_def}. Please revise its definition and output the entire PDDL action model again. Note that you need to define the object type of each parameter and strictly follow the syntax of PDDL." return False, feedback_msg - + if i + 2 >= len(split_predicate): - feedback_msg = f'There are syntax errors in the definition of the new predicate {pred_def}. Please revise its definition and output the entire PDDL action model again. Note that you need to define the object type of each parameter and strictly follow the syntax of PDDL.' + feedback_msg = f"There are syntax errors in the definition of the new predicate {pred_def}. Please revise its definition and output the entire PDDL action model again. Note that you need to define the object type of each parameter and strictly follow the syntax of PDDL." return False, feedback_msg - - param_obj_type = split_predicate[i+2].lower() - + + param_obj_type = split_predicate[i + 2].lower() + # Extract the base type names from the keys in types - valid_types = {type_key.split(' - ')[0].strip().lower() for type_key in types.keys()} - + valid_types = { + type_key.split(" - ")[0].strip().lower() + for type_key in types.keys() + } + # Check if the parameter object type is in the set of valid types if param_obj_type not in valid_types: all_invalid_params.append((param_obj_type, p, pred_def)) if all_invalid_params: - feedback_msg = 'There are invalid object types in the predicates:' + feedback_msg = "There are invalid object types in the predicates:" for param_obj_type, p, pred_def in all_invalid_params: - feedback_msg += f'\n- `{param_obj_type}` for the parameter `{p}` in the definition of the predicate `{pred_def}` not found in types: {valid_types}.' - feedback_msg += '\nPlease revise these definitions and output the entire PDDL action model again.' + feedback_msg += f"\n- `{param_obj_type}` for the parameter `{p}` in the definition of the predicate `{pred_def}` not found in types: {valid_types}." + feedback_msg += "\nPlease revise these definitions and output the entire PDDL action model again." return False, feedback_msg - + feedback_msg = "PASS: All predicates are formatted correctly." return True, feedback_msg - + def validate_pddl_usage_predicates( - self, - pddl: str, - predicates: list[Predicate], + self, + pddl: str, + predicates: list[Predicate], action_params: list[str], - types: dict[str,str], - part='preconditions' - ) -> tuple[bool, str]: + types: dict[str, str], + part="preconditions", + ) -> tuple[bool, str]: """ This function checks three types of errors: - (i) check if the num of params given matches the num of params in predicate definition - (ii) check if there is any param that is not listed under `Parameters:` - (iii) check if the param type matches that in the predicate definition """ + def get_ordinal_suffix(_num): - return {1: 'st', 2: 'nd', 3: 'rd'}.get(_num % 10, 'th') if _num not in (11, 12, 13) else 'th' + return ( + {1: "st", 2: "nd", 3: "rd"}.get(_num % 10, "th") + if _num not in (11, 12, 13) + else "th" + ) + + pred_names = {predicates[i]["name"]: i for i in range(len(predicates))} + pddl_elems = [e for e in pddl.split(" ") if e != ""] - pred_names = {predicates[i]['name']: i for i in range(len(predicates))} - pddl_elems = [e for e in pddl.split(' ') if e != ''] - idx = 0 while idx < len(pddl_elems): - if pddl_elems[idx] == '(' and idx + 1 < len(pddl_elems): + if pddl_elems[idx] == "(" and idx + 1 < len(pddl_elems): if pddl_elems[idx + 1] in pred_names: - + curr_pred_name = pddl_elems[idx + 1] curr_pred_params = list() target_pred_info = predicates[pred_names[curr_pred_name]] - + # read params idx += 2 - while idx < len(pddl_elems) and pddl_elems[idx] != ')': + while idx < len(pddl_elems) and pddl_elems[idx] != ")": curr_pred_params.append(pddl_elems[idx]) idx += 1 # (i) check if the num of params are correct - n_expected_param = len(target_pred_info['params']) + n_expected_param = len(target_pred_info["params"]) if n_expected_param != len(curr_pred_params): feedback_msg = f'In the {part}, the predicate `{curr_pred_name}` requires {n_expected_param} parameters but {len(curr_pred_params)} parameters were provided. Object type should not be declared in the {part}, but just the variable. For example, "(drive ?a ?from)" does not contain its object types, just variables. Do not change the predicates. Please revise the PDDL model to fix this error.' return False, feedback_msg - + # (ii) check if there is any unknown param for curr_param in curr_pred_params: - + if curr_param not in action_params[0]: - feedback_msg = f'In the {part} and in the predicate `{curr_pred_name}`, there is an unknown parameter `{curr_param}`. You should define all parameters (i.e., name and type) under the `### Action Parameters` list. Please revise the PDDL model to fix this error (and other potentially similar errors).' + feedback_msg = f"In the {part} and in the predicate `{curr_pred_name}`, there is an unknown parameter `{curr_param}`. You should define all parameters (i.e., name and type) under the `### Action Parameters` list. Please revise the PDDL model to fix this error (and other potentially similar errors)." return False, feedback_msg - + # (iii) check if the object types are correct - target_param_types = [target_pred_info['params'][t_p] for t_p in target_pred_info['params']] + target_param_types = [ + target_pred_info["params"][t_p] + for t_p in target_pred_info["params"] + ] for param_idx, target_type in enumerate(target_param_types): curr_param = curr_pred_params[param_idx] claimed_type = action_params[0][curr_param] if not self.validate_type(target_type, claimed_type, types): - feedback_msg = f'There is a syntax error in the {part.lower()}, the {param_idx+1}-{get_ordinal_suffix(param_idx+1)} parameter of `{curr_pred_name}` should be a `{target_type}` but a `{claimed_type}` was given. Please use the correct predicate or devise new one(s) if needed (but note that you should use existing predicates as much as possible).' + feedback_msg = f"There is a syntax error in the {part.lower()}, the {param_idx+1}-{get_ordinal_suffix(param_idx+1)} parameter of `{curr_pred_name}` should be a `{target_type}` but a `{claimed_type}` was given. Please use the correct predicate or devise new one(s) if needed (but note that you should use existing predicates as much as possible)." return False, feedback_msg idx += 1 - + feedback_msg = "PASS: all correct use of predicates." return True, feedback_msg - def validate_usage_predicates(self, llm_response: str, curr_predicates: list[Predicate], types: dict[str,str]): + def validate_usage_predicates( + self, llm_response: str, curr_predicates: list[Predicate], types: dict[str, str] + ): """ This function performs very basic check over whether the predicates are used in a valid way. This check should be performed at the end. """ - + # parse predicates new_predicates = parse_new_predicates(llm_response) curr_predicates.extend(new_predicates) @@ -199,209 +231,243 @@ def validate_usage_predicates(self, llm_response: str, curr_predicates: list[Pre params_info = parse_params(llm_response) # check preconditions - precond_str = llm_response.split('Preconditions')[1].split('```\n')[1].strip() - precond_str = precond_str.replace('\n', ' ').replace('(', ' ( ').replace(')', ' ) ') - - validation_info = self.validate_pddl_usage_predicates(precond_str, curr_predicates, params_info, types, part='preconditions') + precond_str = llm_response.split("Preconditions")[1].split("```\n")[1].strip() + precond_str = ( + precond_str.replace("\n", " ").replace("(", " ( ").replace(")", " ) ") + ) + + validation_info = self.validate_pddl_usage_predicates( + precond_str, curr_predicates, params_info, types, part="preconditions" + ) if not validation_info[0]: return validation_info # check effects - if llm_response.split('Effects')[1].count('```\n') < 2: - return True, 'invalid_predicate_usage' - eff_str = llm_response.split('Effects')[1].split('```\n')[1].strip() - eff_str = eff_str.replace('\n', ' ').replace('(', ' ( ').replace(')', ' ) ') - return self.validate_pddl_usage_predicates(eff_str, curr_predicates, params_info, types, part='effects') - - def validate_overflow_predicates(self, llm_response: str, limit: int) -> tuple[bool, str]: + if llm_response.split("Effects")[1].count("```\n") < 2: + return True, "invalid_predicate_usage" + eff_str = llm_response.split("Effects")[1].split("```\n")[1].strip() + eff_str = eff_str.replace("\n", " ").replace("(", " ( ").replace(")", " ) ") + return self.validate_pddl_usage_predicates( + eff_str, curr_predicates, params_info, types, part="effects" + ) + + def validate_overflow_predicates( + self, llm_response: str, limit: int + ) -> tuple[bool, str]: """ Checks if LLM output contains too many predicates in precondition/effects (based on users assigned limit) """ - assert '\nPreconditions:' in llm_response, llm_response - precond_str = llm_response.split('\nPreconditions:')[1].split('```\n')[1].strip() - if len(precond_str.split('\n')) > limit: - feedback_msg = f'FAIL: You seem to have generated an action model with an unusually long list of preconditions. Please include only the relevant preconditions/effects and keep the action model concise.\n\nParameters:' + assert "\nPreconditions:" in llm_response, llm_response + precond_str = ( + llm_response.split("\nPreconditions:")[1].split("```\n")[1].strip() + ) + if len(precond_str.split("\n")) > limit: + feedback_msg = f"FAIL: You seem to have generated an action model with an unusually long list of preconditions. Please include only the relevant preconditions/effects and keep the action model concise.\n\nParameters:" return False, feedback_msg - - eff_str = llm_response.split('Effects')[1].split('```\n')[1].strip() - if len(eff_str.split('\n')) > limit: - feedback_msg = f'FAIL: You seem to have generated an action model with an unusually long list of effects. Please include only the relevant preconditions/effects and keep the action model concise.\n\nParameters:' + + eff_str = llm_response.split("Effects")[1].split("```\n")[1].strip() + if len(eff_str.split("\n")) > limit: + feedback_msg = f"FAIL: You seem to have generated an action model with an unusually long list of effects. Please include only the relevant preconditions/effects and keep the action model concise.\n\nParameters:" return False, feedback_msg feedback_msg = "PASS: predicate output is fine." return True, feedback_msg - - def validate_task_objects(self, objects: dict[str,str], types: dict[str,str]) -> tuple[bool, str]: + def validate_task_objects( + self, objects: dict[str, str], types: dict[str, str] + ) -> tuple[bool, str]: """ Parameters: - objects (dict[str,str]): a dictionary of the task objects. - types (dict[str,str]): a dictionary of the domain types. - + Returns: - check, feedback_msg (bool, str) - - Checks following cases: + + Checks following cases: (i) if object type is the same as type (ii) if object name is the same as type """ - + valid = True feedback_msgs = [] - + for obj_name, obj_type in objects.items(): obj_type_found = False - + for type_key in types.keys(): current_type, parent_type = type_key.split(" - ") - + # checks if obj_type is found in types if obj_type == current_type or obj_type == parent_type: obj_type_found = True - + # checks if obj_name matches either current_type or parent_type if obj_name == current_type: - feedback_msgs.append(f"ERROR: Object variable '{obj_name}' matches the type name '{current_type}', change it to be unique from types: {types.keys()}") + feedback_msgs.append( + f"ERROR: Object variable '{obj_name}' matches the type name '{current_type}', change it to be unique from types: {types.keys()}" + ) valid = False break if obj_name == parent_type: - feedback_msgs.append(f"ERROR: Object variable '{obj_name}' matches the type name '{parent_type}', change it to be unique from types: {types.keys()}") + feedback_msgs.append( + f"ERROR: Object variable '{obj_name}' matches the type name '{parent_type}', change it to be unique from types: {types.keys()}" + ) valid = False break - + # clause that checks if obj_type is found in types if not obj_type_found: - feedback_msgs.append(f"ERROR: Object variable '{obj_name}' has an invalid type '{obj_type}' not found in types: {types.keys()}") + feedback_msgs.append( + f"ERROR: Object variable '{obj_name}' has an invalid type '{obj_type}' not found in types: {types.keys()}" + ) valid = False - - feedback_msg = "\n".join(feedback_msgs) if not valid else "PASS: all objects are valid." - + + feedback_msg = ( + "\n".join(feedback_msgs) if not valid else "PASS: all objects are valid." + ) + return valid, feedback_msg - + def validate_task_states( self, - states: list[dict[str,str]], - objects: dict[str,str], + states: list[dict[str, str]], + objects: dict[str, str], predicates: list[Predicate], - state_type: str="initial" - ) -> tuple[bool, str]: + state_type: str = "initial", + ) -> tuple[bool, str]: """ Parameters: - states (list[dict[str,str]]): a list of dictionaries of the state states. - parameters (OrderedDict): parameters of the current action. - types (dict[str,str]): a dictionary of the domain types. - + Returns: - check, feedback_msg (bool, str) - - Checks following cases: + + Checks following cases: (i) if predicates in states are found in predicates in domain (ii) if object variables in states are found in task object list """ - + valid = True feedback_msgs = [] - + # loop through each state for state in states: - + # (i) check if predicates in states are found in predicates in domain matched_preds = False - state_name = state['name'] # retrieve predicate name from state - + state_name = state["name"] # retrieve predicate name from state + # loop through each predicate name from domain for pred in predicates: # check if predicate in state is found in predicate domain - if state_name == pred['name']: + if state_name == pred["name"]: matched_preds = True - + # if no matches, then that state is missusing a predicate - not found in domain if matched_preds == False: - feedback_msgs.append(f"ERROR: In the {state_type} state, '({state['name']} {' '.join(state['params'])})' contains '{state_name}' predicate, which is not found in {[p['name'] for p in predicates]}, predicate in state is missused.") + feedback_msgs.append( + f"ERROR: In the {state_type} state, '({state['name']} {' '.join(state['params'])})' contains '{state_name}' predicate, which is not found in {[p['name'] for p in predicates]}, predicate in state is missused." + ) valid = False - + # (ii) check if object variables in states are found in task object list - state_params = state['params'] # retrieve variables from state - + state_params = state["params"] # retrieve variables from state + # loop through each parameter in current state for state_p in state_params: - + matched_params = False for obj_name, obj_type in objects.items(): # check if parameter is found in object names if state_p == obj_name: matched_params = True - + if matched_params == False: - feedback_msgs.append(f"ERROR: In the {state_type} state, '({state['name']} {' '.join(state['params'])})' contains parameter '{state_p}' not found in '{objects.keys()}'.") + feedback_msgs.append( + f"ERROR: In the {state_type} state, '({state['name']} {' '.join(state['params'])})' contains parameter '{state_p}' not found in '{objects.keys()}'." + ) valid = False - feedback_msg = "\n".join(feedback_msgs) if not valid else "PASS: all objects are valid." - + feedback_msg = ( + "\n".join(feedback_msgs) if not valid else "PASS: all objects are valid." + ) + return valid, feedback_msg - - + def validate_header(self, llm_response: str): """Checks if domain headers and formatted code block syntax are found in LLM output""" - - for header in ['Parameters', 'Preconditions', 'Effects', 'New Predicates']: + + for header in ["Parameters", "Preconditions", "Effects", "New Predicates"]: if header not in llm_response: - feedback_msg = f'FAIL: The header `{header}` is missing in the PDDL model. Please include the header `{header}` in the PDDL model.' + feedback_msg = f"FAIL: The header `{header}` is missing in the PDDL model. Please include the header `{header}` in the PDDL model." return False, feedback_msg - for header in ['Parameters', 'Preconditions', 'Effects']: - if llm_response.split(f"{header}")[1].split("##")[0].count('```\n') < 2: + for header in ["Parameters", "Preconditions", "Effects"]: + if llm_response.split(f"{header}")[1].split("##")[0].count("```\n") < 2: feedback_msg = f'FAIL: The header `{header}` is missing in the formalised code block. Please include a "```" section in the {header} section.' return False, feedback_msg - + feedback_msg = "PASS: headers are identified properly in LLM output." return True, feedback_msg - - def validate_unsupported_keywords(self, llm_response: str, unsupported_keywords: list[str]) -> tuple[bool, str]: + + def validate_unsupported_keywords( + self, llm_response: str, unsupported_keywords: list[str] + ) -> tuple[bool, str]: """Checks whether PDDL model uses unsupported logic keywords""" for key in unsupported_keywords: - if f'{key}' in llm_response: - feedback_msg = f'ERROR: The precondition or effect contains the keyword {key}.' + if f"{key}" in llm_response: + feedback_msg = ( + f"ERROR: The precondition or effect contains the keyword {key}." + ) return False, feedback_msg feedback_msg = "PASS: Unsupported keywords not found in PDDL model." return True, feedback_msg - + def validate_keyword_usage(self, llm_response: str): """Checks if action effects uses unsupported universal condition keywords""" - + if not "Action Effects" in llm_response: feedback_msg = "PASS" return True, feedback_msg heading = llm_response.split("Action Effects")[1].split("```\n")[1].strip() - for keyword in ['forall', 'exists', "if "]: + for keyword in ["forall", "exists", "if "]: if keyword in heading: - feedback_msg = f'The keyword `{keyword}` is not supported in the action effects.' + feedback_msg = ( + f"The keyword `{keyword}` is not supported in the action effects." + ) return False, feedback_msg - + feedback_msg = "PASS: unsupported keywords are not found in the action effects." return True, feedback_msg - - + def validate_new_action_creation(self, llm_response: str) -> tuple[bool, str]: """Checks if the LLM attempts to create a new action (so two or more actions defined in the same response)""" - - if llm_response.count('## Action Parameters') > 1 or llm_response.count('## Preconditions') > 1 or llm_response.count('## Effects') > 1 or llm_response.count('## New Predicates') > 1: + + if ( + llm_response.count("## Action Parameters") > 1 + or llm_response.count("## Preconditions") > 1 + or llm_response.count("## Effects") > 1 + or llm_response.count("## New Predicates") > 1 + ): feedback_msg = "It's not possible to create new actions at this time. Please only define the requested action." return False, feedback_msg - + feedback_msg = "PASS: no new actions created" return True, feedback_msg def validate_type(self, target_type, claimed_type, types): """ Check if the claimed_type is valid for the target_type according to the type hierarchy. - + Parameters: - target_type (str): The type that is expected for the parameter. - claimed_type (str): The type that is provided in the PDDL. - types (dict[str, str]): A dictionary mapping subtypes to their supertypes. - + Returns: - bool: True if claimed_type is valid, False otherwise. """ @@ -411,12 +477,12 @@ def validate_type(self, target_type, claimed_type, types): # Iterate through the types hierarchy to check if claimed_type is a subtype of target_type current_type = claimed_type - + # Extract all types from the keys in the types dictionary all_types = set() type_hierarchy = {} for key in types.keys(): - main_type, *subtype = key.split(' - ') + main_type, *subtype = key.split(" - ") all_types.add(main_type.strip()) if subtype: all_types.add(subtype[0].strip()) @@ -424,13 +490,15 @@ def validate_type(self, target_type, claimed_type, types): while current_type in all_types: # find the key that starts with the current type - - parent_type_entry = next((k for k in types.keys() if k.startswith(f'{current_type} - ')), None) - + + parent_type_entry = next( + (k for k in types.keys() if k.startswith(f"{current_type} - ")), None + ) + if parent_type_entry: # extract the parent type from the key - super_type = parent_type_entry.split(' - ')[1].strip() - + super_type = parent_type_entry.split(" - ")[1].strip() + if super_type == target_type: return True current_type = super_type @@ -438,6 +506,3 @@ def validate_type(self, target_type, claimed_type, types): break return False - -if __name__ == '__main__': - pass \ No newline at end of file diff --git a/l2p/utils/utils.py b/l2p/utils/utils.py index e9725dc..5df3a80 100644 --- a/l2p/utils/utils.py +++ b/l2p/utils/utils.py @@ -1,7 +1,10 @@ import os, json + def load_file(file_path): - _, ext = os.path.splitext(file_path) - with open(file_path, 'r') as file: - if ext == '.json': return json.load(file) - else: return file.read().strip() \ No newline at end of file + _, ext = os.path.splitext(file_path) + with open(file_path, "r") as file: + if ext == ".json": + return json.load(file) + else: + return file.read().strip() diff --git a/paper_reconstructions/llm+dm/llm+dm.py b/paper_reconstructions/llm+dm/llm+dm.py index 542cbf8..c780e45 100644 --- a/paper_reconstructions/llm+dm/llm+dm.py +++ b/paper_reconstructions/llm+dm/llm+dm.py @@ -1,5 +1,3 @@ - - """ Paper: "Leveraging Pre-trained Large Language Models to Construct and Utilize World Models for Model-based Task Planning" Guan et al. (2023) Source code: https://github.com/GuanSuns/LLMs-World-Models-for-Planning @@ -15,177 +13,206 @@ from copy import deepcopy from l2p import * + def open_txt(file_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: file = file.read().strip() return file + def open_json(file_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: file = json.load(file) return file def construct_action_model( - domain_desc, - prompt_template, - action_name, - action_desc, - predicate_list, - max_iterations=3, - syntax_validator=None - ) -> tuple[Action,list[Predicate],str]: - + domain_desc, + prompt_template, + action_name, + action_desc, + predicate_list, + max_iterations=3, + syntax_validator=None, +) -> tuple[Action, list[Predicate], str]: """ This function constructs an action model for a single action. Specifically, it runs through syntax validator to refine model in a certain set amount of iterations. - + Returns: - pddl_action (Action): Action model class that contains params, preconditions, effects, and additional info - predicate_list (list[Predicate]): list of Predicate classes - llm_response (str): raw output from LLM """ - + no_syntax_error = False i_iter = 0 - + # create action model, check for syntax error while not no_syntax_error and i_iter < max_iterations: i_iter += 1 - + # generate action model pddl_action, new_predicates, llm_response = domain_builder.extract_pddl_action( - model=openai_llm, + model=openai_llm, domain_desc=domain_desc, - prompt_template=prompt_template, + prompt_template=prompt_template, action_name=action_name, action_desc=action_desc, predicates=predicate_list, - types=hierarchy_requirements["hierarchy"] - ) - - + types=hierarchy_requirements["hierarchy"], + ) + # if syntax validator check is set on if syntax_validator is not None: syntax_valid = False while not syntax_valid: - # perform syntax check on action model - no_syntax_error, feedback_msg = syntax_validator.validate_usage_predicates(llm_response, predicate_list, hierarchy_requirements["hierarchy"]) - - # if there is syntax error, run through feedback mechanism to retrieve new action model - if no_syntax_error is False: - # Update the prompt with the feedback - prompt_template += "\n\nHere is the PDDL action you outputted:\n" + str(pddl_action) - if len(new_predicates) > 0: - prompt_template += "\n\nHere are the predicates you created from that action:\n" + format_predicates(new_predicates) - prompt_template += "\n\nHere is the feedback you outputted:\n" + feedback_msg - - # Generate a new PDDL action model based on the feedback - pddl_action, new_predicates, llm_response = domain_builder.extract_pddl_action( - model=openai_llm, + # perform syntax check on action model + no_syntax_error, feedback_msg = ( + syntax_validator.validate_usage_predicates( + llm_response, + predicate_list, + hierarchy_requirements["hierarchy"], + ) + ) + + # if there is syntax error, run through feedback mechanism to retrieve new action model + if no_syntax_error is False: + # Update the prompt with the feedback + prompt_template += ( + "\n\nHere is the PDDL action you outputted:\n" + + str(pddl_action) + ) + if len(new_predicates) > 0: + prompt_template += ( + "\n\nHere are the predicates you created from that action:\n" + + format_predicates(new_predicates) + ) + prompt_template += ( + "\n\nHere is the feedback you outputted:\n" + feedback_msg + ) + + # Generate a new PDDL action model based on the feedback + pddl_action, new_predicates, llm_response = ( + domain_builder.extract_pddl_action( + model=openai_llm, domain_desc=domain_desc, - prompt_template=prompt_template, + prompt_template=prompt_template, action_name=action_name, action_desc=action_desc, predicates=predicate_list, - types=hierarchy_requirements["hierarchy"] + types=hierarchy_requirements["hierarchy"], ) - else: - syntax_valid = True - + ) + else: + syntax_valid = True + new_predicates = parse_new_predicates(llm_response) predicate_list.extend(new_predicates) return pddl_action, predicate_list, llm_response -if __name__ == "__main__": - +if __name__ == "__main__": + # setup prompt templates - action_model = open_json('paper_reconstructions/llm+dm/prompts/action_model.json') - domain_desc = open_txt('paper_reconstructions/llm+dm/prompts/domain_desc.txt') - hierarchy_requirements = open_json('paper_reconstructions/llm+dm/prompts/hierarchy_requirements.json') - prompt_template = open_txt('paper_reconstructions/llm+dm/prompts/pddl_prompt.txt') + action_model = open_json("paper_reconstructions/llm+dm/prompts/action_model.json") + domain_desc = open_txt("paper_reconstructions/llm+dm/prompts/domain_desc.txt") + hierarchy_requirements = open_json( + "paper_reconstructions/llm+dm/prompts/hierarchy_requirements.json" + ) + prompt_template = open_txt("paper_reconstructions/llm+dm/prompts/pddl_prompt.txt") # setup LLM engine engine = "gpt-4o-mini" - api_key = os.environ.get('OPENAI_API_KEY') + api_key = os.environ.get("OPENAI_API_KEY") openai_llm = OPENAI(model=engine, api_key=api_key) # setup L2P libraries domain_builder = DomainBuilder() syntax_validator = SyntaxValidator() - domain = 'logistics' # using logistics domain for this example - + domain = "logistics" # using logistics domain for this example + max_iterations = 2 max_feedback = 1 - + actions = list(action_model.keys()) predicate_list = list() - + """ Action-by-action algorithm: iteratively generates an action model (parameters, precondition, effects) one at a time. At the same time, it is generating new predicates if needed and is added to a dynamic list. At the end of the iterations, it is ran again once more to create the action models agains, but with using the new predicate list. This algorithm can iterative as many times as needed until no new predicates are added to the list. This is an action model refinement algorithm, that refines itself by a growing predicate list. """ - + # iterate however many times for i_iter in range(max_iterations): prev_predicate_list = deepcopy(predicate_list) - + action_list = [] - + # iterate through each action for i_action, action in enumerate(actions): - + # replace prompt with dynamic predicate list if len(predicate_list) == 0: # if no predicates in list - prompt_template = prompt_template.replace('{predicates}', '\nNo predicate has been defined yet') + prompt_template = prompt_template.replace( + "{predicates}", "\nNo predicate has been defined yet" + ) else: # replace with current predicates readable_results = "" for i, p in enumerate(predicate_list): readable_results += f'\n{i + 1}. {p["raw"]}' - - prompt_template = prompt_template.replace('{predicates}', readable_results) - + + prompt_template = prompt_template.replace( + "{predicates}", readable_results + ) + # construct action model pddl_action, predicate_list, llm_output = construct_action_model( - domain_desc, - prompt_template, - action, - action_model[action]['desc'], - predicate_list, + domain_desc, + prompt_template, + action, + action_model[action]["desc"], + predicate_list, max_iterations=max_feedback, - syntax_validator=syntax_validator - ) + syntax_validator=syntax_validator, + ) action_list.append(pddl_action) - + # at the end of the action-by-action algorithm, clean predicates, types, and build parse PDDL domain predicate_list = prune_predicates(predicates=predicate_list, actions=action_list) - predicate_str = "\n".join([pred["clean"].replace(":", " ; ") for pred in predicate_list]) - + predicate_str = "\n".join( + [pred["clean"].replace(":", " ; ") for pred in predicate_list] + ) + # prune types if not found in action interfaces - types = {name: description for name, description in hierarchy_requirements["hierarchy"].items() if name} + types = { + name: description + for name, description in hierarchy_requirements["hierarchy"].items() + if name + } types_str = "\n".join(types) - + # generate domain pddl_domain = domain_builder.generate_domain( - domain=domain, + domain=domain, requirements=hierarchy_requirements["requirements"], types=types_str, predicates=predicate_str, - actions=action_list - ) - + actions=action_list, + ) + domain_file = "paper_reconstructions/llm+dm/results/domain.pddl" - + # save domain file with open(domain_file, "w") as f: f.write(pddl_domain) - \ No newline at end of file + + print(pddl_domain) diff --git a/paper_reconstructions/llm+dm/prompts/pddl_prompt.txt b/paper_reconstructions/llm+dm/prompts/pddl_prompt.txt index 43efd20..36e7f24 100644 --- a/paper_reconstructions/llm+dm/prompts/pddl_prompt.txt +++ b/paper_reconstructions/llm+dm/prompts/pddl_prompt.txt @@ -4,6 +4,8 @@ Here are two examples from the classical BlocksWorld domain for demonstrating th Domain information: BlocksWorld is a planning domain in artificial intelligence. The AI agent here is a mechanical robot arm that can pick and place the blocks. Only one block may be moved at a time: it may either be placed on the table or placed atop another block. Because of this, any blocks that are, at a given time, under another block cannot be moved. There is only one type of object in this domain, and that is the block. +End your final answers underneath the headers: '### Action Parameters,' '### Action Preconditions,' '### Action Effects,' and '### New Predicates' with ''' ''' comment blocks in PDDL. Follow the exact example syntax as the following: + Example 1 Action: This action enables the robot to put a block onto the table. For example, the robot puts block_1 onto the table. diff --git a/paper_reconstructions/llm+dm/results/domain.pddl b/paper_reconstructions/llm+dm/results/domain.pddl index 53886a8..2b06ea9 100644 --- a/paper_reconstructions/llm+dm/results/domain.pddl +++ b/paper_reconstructions/llm+dm/results/domain.pddl @@ -1,30 +1,30 @@ (define (domain logistics) -(:requirements - :strips :typing :equality :negative-preconditions :disjunctive-preconditions :universal-preconditions :conditional-effects -) - (:types -truck -plane -package -city -location + (:requirements + :strips :typing :equality :negative-preconditions :disjunctive-preconditions :universal-preconditions :conditional-effects + ) + (:types + truck + plane + package + city + location ) (:predicates -(truck-at ?t - truck ?l - location) ; true if the truck ?t is currently located at location ?l -(package-at ?p - package ?l - location) ; true if the package ?p is currently located at location ?l -(truck-has-space ?t - truck) ; true if the truck ?t has space to load more packages -(truck-holding ?t - truck ?p - package) ; true if the truck ?t is currently holding the package ?p -(at-airport ?a - plane ?l - location) ; true if the airplane ?a is at the airport located at ?l -(location-connected ?from - location ?to - location, ?c - city) ; true if location ?from is directly connected to location ?to in city ?c -(connected-cities ?from_city - city ?to_city - city) ; true if there is a direct connection between city ?from_city and city ?to_city -(package-on-ground ?p - package) ; true if the package ?p is on the ground and ready to be loaded -(airplane-at-airport ?a - plane) ; true if the airplane ?a is at the designated airport location -(airplane-has-space ?a - plane) ; true if the airplane ?a has space available to load more packages -(airplane-has-package ?a - plane ?p - package) ; true if the airplane ?a is carrying the package ?p -(plane-at ?a - plane ?c - city) ; true if the airplane ?a is located in city ?c -(package-on-plane ?p - package ?a - plane) ; true if the package ?p is currently on airplane ?a -(location-in-city ?l - location ?c - city) ; true if the location ?l is situated in city ?c + (truck-at ?t - truck ?l - location) ; true if the truck ?t is located at location ?l + (package-at ?p - package ?l - location) ; true if the package ?p is located at location ?l + (truck-has-space ?t - truck) ; true if the truck ?t has space to load more packages + (truck-has-package ?t - truck ?p - package) ; true if the truck ?t is carrying the package ?p + (truck-holding ?t - truck ?p - package) ; true if the truck ?t is holding the package ?p + (airplane-at ?a - plane ?l - location) ; true if the airplane ?a is located at location ?l + (airplane-full ?a - plane) ; true if the airplane ?a cannot carry more packages + (airplane-has-package ?a - plane ?p - package) ; true if the airplane ?a is carrying the package ?p + (airplane-holding ?a - plane ?p - package) ; true if the airplane ?a is currently holding the package ?p + (at-airplane ?a - plane ?l - location) ; true if the airplane ?a is located at the location ?l + (location-connected ?l1 - location ?l2 - location ?c - city) ; true if location ?l1 is directly connected to location ?l2 in city ?c + (at-airport ?plane - plane ?city - city) ; true if the airplane ?plane is at the airport in city ?city + (truck-at-location ?t - truck ?l - location) ; true if the truck ?t is at the location ?l + (package-at-location ?p - package ?l - location) ; true if the package ?p is at the location ?l ) (:action load_truck @@ -42,7 +42,7 @@ location :effect (and (not (package-at ?p ?l)) - (truck-holding ?t ?p) + (truck-has-package ?t ?p) ) ) @@ -55,12 +55,12 @@ location :precondition (and (truck-holding ?t ?p) - (truck-at ?t ?l) + (truck-at-location ?t ?l) ) :effect (and (not (truck-holding ?t ?p)) - (package-at ?p ?l) + (package-at-location ?p ?l) ) ) @@ -68,16 +68,17 @@ location :parameters ( ?p - package ?a - plane +?l - location ) :precondition (and - (package-on-ground ?p) - (airplane-at-airport ?a) - (airplane-has-space ?a) + (package-at ?p ?l) + (airplane-at ?a ?l) + (not (airplane-full ?a)) ) :effect (and - (not (package-on-ground ?p)) + (not (package-at ?p ?l)) (airplane-has-package ?a ?p) ) ) @@ -87,17 +88,15 @@ location ?p - package ?a - plane ?l - location -?c - city ) :precondition (and - (plane-at ?a ?c) - (package-on-plane ?p ?a) - (location-in-city ?l ?c) + (airplane-holding ?a ?p) + (at-airplane ?a ?l) ) :effect (and - (not (package-on-plane ?p ?a)) + (not (airplane-holding ?a ?p)) (package-at ?p ?l) ) ) @@ -124,18 +123,21 @@ location (:action fly_airplane :parameters ( ?plane - plane +?from_airport - location +?to_airport - location ?from_city - city ?to_city - city ) :precondition (and - (at-airport ?plane ?from_city) - (connected-cities ?from_city ?to_city) + (at-airport ?from_airport ?from_city) + (at-airport ?to_airport ?to_city) + (airplane-at ?plane ?from_airport) ) :effect (and - (not (at-airport ?plane ?from_city)) - (at-airport ?plane ?to_city) + (not (airplane-at ?plane ?from_airport)) + (airplane-at ?plane ?to_airport) ) ) ) \ No newline at end of file diff --git a/paper_reconstructions/llm+p/llm+p.py b/paper_reconstructions/llm+p/llm+p.py index fedfac5..c528b7b 100644 --- a/paper_reconstructions/llm+p/llm+p.py +++ b/paper_reconstructions/llm+p/llm+p.py @@ -14,16 +14,18 @@ from l2p import * from l2p.utils.pddl_planner import FastDownward + def open_file(file_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: file = file.read().strip() return file + if __name__ == "__main__": - + # setup L2P requirements engine = "gpt-4o-mini" - api_key = os.environ.get('OPENAI_API_KEY') + api_key = os.environ.get("OPENAI_API_KEY") openai_llm = OPENAI(model=engine, api_key=api_key) planner = FastDownward() @@ -41,16 +43,23 @@ def open_file(file_path): # extract PDDL from prompt objects, initial, goal, llm_response = task_builder.extract_task( - model=openai_llm, + model=openai_llm, problem_desc=problem_desc, - prompt_template=prompt_builder.generate_prompt()) + prompt_template=prompt_builder.generate_prompt(), + ) # construct PDDL components into PDDL problem file object_str = task_builder.format_objects(objects) initial_state_str = task_builder.format_initial(initial) goal_state_str = task_builder.format_goal(goal) - pddl_problem = task_builder.generate_task("blocksworld-4ops", "blocksworld-4ops_problem", object_str, initial_state_str, goal_state_str) + pddl_problem = task_builder.generate_task( + "blocksworld-4ops", + "blocksworld-4ops_problem", + object_str, + initial_state_str, + goal_state_str, + ) # write down PDDL problem file problem_file = "paper_reconstructions/llm+p/results/problem.pddl" @@ -62,4 +71,3 @@ def open_file(file_path): # run planner planner.run_fast_downward(domain_file=domain_file, problem_file=problem_file) - \ No newline at end of file diff --git a/paper_reconstructions/llm+p/prompts/role.txt b/paper_reconstructions/llm+p/prompts/role.txt index 92d5cc2..6eceb87 100644 --- a/paper_reconstructions/llm+p/prompts/role.txt +++ b/paper_reconstructions/llm+p/prompts/role.txt @@ -9,17 +9,17 @@ Do not, under any circumstance, output the answers in PDDL format. Final answer """ ## Problem description -## OBJECTS +### OBJECTS ``` truck1 - truck ``` -## INITIAL +### INITIAL ``` (at truck1 chicago_depot): truck1 is at the chicago_depot ``` -## GOAL +### GOAL ``` (AND ; all the following should be done (finalised house1) ; house 1 is done diff --git a/paper_reconstructions/nl2plan/nl2plan.py b/paper_reconstructions/nl2plan/nl2plan.py index 88853a2..3fc66f0 100644 --- a/paper_reconstructions/nl2plan/nl2plan.py +++ b/paper_reconstructions/nl2plan/nl2plan.py @@ -10,48 +10,50 @@ def open_file(file_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: file = file.read().strip() return file + def format_json_output(data): - return json.dumps(data, indent=4) + return json.dumps(data, indent=4) + engine = "gpt-4o-mini" -api_key = os.environ.get('OPENAI_API_KEY') +api_key = os.environ.get("OPENAI_API_KEY") openai_llm = OPENAI(model=engine, api_key=api_key) -domain_desc = open_file('data/domains/blocksworld.txt') -problem_desc = open_file("data/problems/blocksworld_p1.txt") +domain_desc = open_file("paper_reconstructions/nl2plan/prompts/blocksworld.txt") +problem_desc = open_file("paper_reconstructions/nl2plan/prompts/blocksworld_p1.txt") # open and create type extraction prompt builder class -role_desc = open_file('paper_reconstructions/nl2plan/prompts/type_extraction/role.txt') -tech_desc = open_file('paper_reconstructions/nl2plan/prompts/type_extraction/technique.txt') -task_desc = open_file('paper_reconstructions/nl2plan/prompts/type_extraction/task.txt') +role_desc = open_file("paper_reconstructions/nl2plan/prompts/type_extraction/role.txt") +tech_desc = open_file("paper_reconstructions/nl2plan/prompts/type_extraction/technique.txt") +task_desc = open_file("paper_reconstructions/nl2plan/prompts/type_extraction/task.txt") type_extraction_prompt = PromptBuilder(role=role_desc, technique=tech_desc, task=task_desc) # open and create type hierarchy prompt builder class -role_desc = open_file('paper_reconstructions/nl2plan/prompts/hierarchy_construction/role.txt') -tech_desc = open_file('paper_reconstructions/nl2plan/prompts/hierarchy_construction/technique.txt') -task_desc = open_file('paper_reconstructions/nl2plan/prompts/hierarchy_construction/task.txt') +role_desc = open_file("paper_reconstructions/nl2plan/prompts/hierarchy_construction/role.txt") +tech_desc = open_file("paper_reconstructions/nl2plan/prompts/hierarchy_construction/technique.txt") +task_desc = open_file("paper_reconstructions/nl2plan/prompts/hierarchy_construction/task.txt") type_hierarchy_prompt = PromptBuilder(role=role_desc, technique=tech_desc, task=task_desc) -# open and create NL action prompt builder class -role_desc = open_file('paper_reconstructions/nl2plan/prompts/action_extraction/role.txt') -tech_desc = open_file('paper_reconstructions/nl2plan/prompts/action_extraction/technique.txt') -task_desc = open_file('paper_reconstructions/nl2plan/prompts/action_extraction/task.txt') +# open and create NL action prompt builder class +role_desc = open_file("paper_reconstructions/nl2plan/prompts/action_extraction/role.txt") +tech_desc = open_file("paper_reconstructions/nl2plan/prompts/action_extraction/technique.txt") +task_desc = open_file("paper_reconstructions/nl2plan/prompts/action_extraction/task.txt") action_extraction_prompt = PromptBuilder(role=role_desc, technique=tech_desc, task=task_desc) # open and create PDDL action prompt builder class -role_desc = open_file('paper_reconstructions/nl2plan/prompts/action_construction/role.txt') -tech_desc = open_file('paper_reconstructions/nl2plan/prompts/action_construction/technique.txt') -task_desc = open_file('paper_reconstructions/nl2plan/prompts/action_construction/task.txt') +role_desc = open_file("paper_reconstructions/nl2plan/prompts/action_construction/role.txt") +tech_desc = open_file("paper_reconstructions/nl2plan/prompts/action_construction/technique.txt") +task_desc = open_file("paper_reconstructions/nl2plan/prompts/action_construction/task.txt") action_construction_prompt = PromptBuilder(role=role_desc, technique=tech_desc, task=task_desc) # open and create compact action prompt builder class -role_desc = open_file('paper_reconstructions/nl2plan/prompts/task_extraction/role.txt') -tech_desc = open_file('paper_reconstructions/nl2plan/prompts/task_extraction/technique.txt') -task_desc = open_file('paper_reconstructions/nl2plan/prompts/task_extraction/task.txt') +role_desc = open_file("paper_reconstructions/nl2plan/prompts/task_extraction/role.txt") +tech_desc = open_file("paper_reconstructions/nl2plan/prompts/task_extraction/technique.txt") +task_desc = open_file("paper_reconstructions/nl2plan/prompts/task_extraction/task.txt") task_extraction_prompt = PromptBuilder(role=role_desc, technique=tech_desc, task=task_desc) domain_builder = DomainBuilder() @@ -60,67 +62,93 @@ def format_json_output(data): planner = FastDownward() feedback_builder = FeedbackBuilder() -unsupported_keywords = ['object', 'pddl', 'lisp'] +unsupported_keywords = ["object", "pddl", "lisp"] + -def type_extraction(model: LLM, domain_desc: str, type_extraction_prompt: PromptBuilder) -> dict[str,str]: +def type_extraction( + model: LLM, domain_desc: str, type_extraction_prompt: PromptBuilder +) -> dict[str, str]: # STEP ONE: type extraction - types, response = domain_builder.extract_type(model, domain_desc, type_extraction_prompt.generate_prompt()) + types, response = domain_builder.extract_type( + model, domain_desc, type_extraction_prompt.generate_prompt() + ) - feedback_template = open_file('paper_reconstructions/nl2plan/prompts/type_extraction/feedback.txt') + feedback_template = open_file( + "paper_reconstructions/nl2plan/prompts/type_extraction/feedback.txt" + ) types, _ = feedback_builder.type_feedback( - model=model, - domain_desc=domain_desc, - feedback_template=feedback_template, - feedback_type="llm", - types=types, - llm_response=response) + model=model, + domain_desc=domain_desc, + feedback_template=feedback_template, + feedback_type="llm", + types=types, + llm_response=response, + ) print("Types:", format_json_output(types)) return types -def hierarchy_construction(model, domain_desc, type_hierarchy_prompt, types) -> dict[str,str]: + +def hierarchy_construction( + model, domain_desc, type_hierarchy_prompt, types +) -> dict[str, str]: # STEP TWO: type hierarchy extraction type_hierarchy, response = domain_builder.extract_type_hierarchy( - model=model, - domain_desc=domain_desc, - prompt_template=type_hierarchy_prompt.generate_prompt(), - types=types) + model=model, + domain_desc=domain_desc, + prompt_template=type_hierarchy_prompt.generate_prompt(), + types=types, + ) - feedback_template = open_file('paper_reconstructions/nl2plan/prompts/hierarchy_construction/feedback.txt') + feedback_template = open_file( + "paper_reconstructions/nl2plan/prompts/hierarchy_construction/feedback.txt" + ) type_hierarchy, _ = feedback_builder.type_hierarchy_feedback( - model=model, - domain_desc=domain_desc, - feedback_template=feedback_template, - feedback_type="llm", - type_hierarchy=type_hierarchy, - llm_response=response) + model=model, + domain_desc=domain_desc, + feedback_template=feedback_template, + feedback_type="llm", + type_hierarchy=type_hierarchy, + llm_response=response, + ) print("Type Hierarchy", format_json_output(type_hierarchy)) return type_hierarchy -def action_extraction(model, domain_desc, action_extraction_prompt, type_hierarchy) -> dict[str,str]: + +def action_extraction( + model, domain_desc, action_extraction_prompt, type_hierarchy +) -> dict[str, str]: # STEP THREE: action extraction nl_actions, response = domain_builder.extract_nl_actions( - model=model, - domain_desc=domain_desc, - prompt_template=action_extraction_prompt.generate_prompt(), - types=type_hierarchy) + model=model, + domain_desc=domain_desc, + prompt_template=action_extraction_prompt.generate_prompt(), + types=type_hierarchy, + ) - feedback_template = open_file('paper_reconstructions/nl2plan/prompts/action_extraction/feedback.txt') + feedback_template = open_file( + "paper_reconstructions/nl2plan/prompts/action_extraction/feedback.txt" + ) nl_actions, _ = feedback_builder.nl_action_feedback( - model=model, + model=model, domain_desc=domain_desc, - llm_response=response, - feedback_template=feedback_template, - feedback_type="llm", - nl_actions=nl_actions, - type_hierarchy=type_hierarchy) - - print("Natural Language Actions") - for i in nl_actions: print(i) + llm_response=response, + feedback_template=feedback_template, + feedback_type="llm", + nl_actions=nl_actions, + type_hierarchy=type_hierarchy, + ) + + print("Natural Language Actions") + for i in nl_actions: + print(i) return nl_actions -def action_construction(model, domain_desc, action_construction_prompt, nl_actions, type_hierarchy) -> tuple[list[Action], list[Predicate]]: + +def action_construction( + model, domain_desc, action_construction_prompt, nl_actions, type_hierarchy +) -> tuple[list[Action], list[Predicate]]: # STEP FOUR: action construction predicates = [] @@ -132,11 +160,17 @@ def action_construction(model, domain_desc, action_construction_prompt, nl_actio for action_name, action_desc in nl_actions.items(): - feedback_template = open_file('paper_reconstructions/nl2plan/prompts/action_construction/feedback.txt') - + feedback_template = open_file( + "paper_reconstructions/nl2plan/prompts/action_construction/feedback.txt" + ) + # retrieve rest of list - action_list = {a_name: a_desc for a_name, a_desc in nl_actions.items() if a_name != action_name} - + action_list = { + a_name: a_desc + for a_name, a_desc in nl_actions.items() + if a_name != action_name + } + action, new_predicates, llm_response = domain_builder.extract_pddl_action( model=model, domain_desc=domain_desc, @@ -145,26 +179,31 @@ def action_construction(model, domain_desc, action_construction_prompt, nl_actio action_desc=action_desc, action_list=action_list, predicates=predicates, - types=type_hierarchy + types=type_hierarchy, ) # perform syntax check on action model - is_valid, feedback_msg = syntax_validator.validate_usage_predicates(llm_response, predicates, type_hierarchy) + is_valid, feedback_msg = syntax_validator.validate_usage_predicates( + llm_response, predicates, type_hierarchy + ) # if there is syntax error, run through feedback mechanism to retrieve new action model if is_valid == False: - feedback_template += "\n\nThe following is a syntax error with your response:\n" + feedback_msg + feedback_template += ( + "\n\nThe following is a syntax error with your response:\n" + + feedback_msg + ) # RUN FEEDBACK action, new_predicates, _ = feedback_builder.pddl_action_feedback( - model=model, + model=model, domain_desc=domain_desc, - llm_response=llm_response, - feedback_template=feedback_template, + llm_response=llm_response, + feedback_template=feedback_template, feedback_type="llm", - action=action, - predicates=predicates, - types=types - ) + action=action, + predicates=predicates, + types=types, + ) actions.append(action) predicates.extend(new_predicates) @@ -178,10 +217,14 @@ def action_construction(model, domain_desc, action_construction_prompt, nl_actio return actions, predicates -def task_extraction(model, problem_desc, task_extraction_prompt, types, predicates, actions - ) -> tuple[dict[str,str], list[dict[str,str]], list[dict[str,str]]]: + +def task_extraction( + model, problem_desc, task_extraction_prompt, types, predicates, actions +) -> tuple[dict[str, str], list[dict[str, str]], list[dict[str, str]]]: # STEP FIVE: task extraction - feedback_template = open_file('paper_reconstructions/nl2plan/prompts/task_extraction/feedback.txt') + feedback_template = open_file( + "paper_reconstructions/nl2plan/prompts/task_extraction/feedback.txt" + ) objects, initial, goal, llm_response = task_builder.extract_task( model=model, @@ -189,16 +232,19 @@ def task_extraction(model, problem_desc, task_extraction_prompt, types, predicat prompt_template=task_extraction_prompt.generate_prompt(), types=types, predicates=predicates, - actions=actions - ) - + actions=actions, + ) + feedback_msgs = [] all_valid = True # List of validation checks validation_checks = [ (syntax_validator.validate_task_objects, (objects, types)), - (syntax_validator.validate_task_states, (initial, objects, predicates, "initial")), + ( + syntax_validator.validate_task_states, + (initial, objects, predicates, "initial"), + ), (syntax_validator.validate_task_states, (goal, objects, predicates, "goal")), ] @@ -211,19 +257,23 @@ def task_extraction(model, problem_desc, task_extraction_prompt, types, predicat # If any check fails, append feedback messages if not all_valid: - feedback_template += "\n\nThe following is a syntax error with your response:" + "\n".join(feedback_msgs) + feedback_template += ( + "\n\nThe following is a syntax error with your response:" + + "\n".join(feedback_msgs) + ) objects, initial, goal, _ = feedback_builder.task_feedback( - model=model, + model=model, problem_desc=problem_desc, - llm_response=llm_response, - feedback_template=feedback_template, - feedback_type="llm", - predicates=predicates, - types=types, - objects=objects, - initial=initial, - goal=goal) + llm_response=llm_response, + feedback_template=feedback_template, + feedback_type="llm", + predicates=predicates, + types=types, + objects=objects, + initial=initial, + goal=goal, + ) objects = task_builder.format_objects(objects) initial = task_builder.format_initial(initial) @@ -235,50 +285,83 @@ def task_extraction(model, problem_desc, task_extraction_prompt, types, predicat return objects, initial, goal + if __name__ == "__main__": - + # STEP ONE: type extraction types = type_extraction(openai_llm, domain_desc, type_extraction_prompt) print("END OF STEP ONE") # STEP TWO: hierarchy construction - type_hierarchy = hierarchy_construction(openai_llm, domain_desc, type_hierarchy_prompt, types) + type_hierarchy = hierarchy_construction( + openai_llm, domain_desc, type_hierarchy_prompt, types + ) print("END OF STEP TWO") # STEP THREE: action extraction - nl_actions = action_extraction(openai_llm, domain_desc, action_extraction_prompt, type_hierarchy) + nl_actions = action_extraction( + openai_llm, domain_desc, action_extraction_prompt, type_hierarchy + ) print("END OF STEP THREE") # STEP FOUR: action construction - actions, predicates = action_construction(openai_llm, domain_desc, action_construction_prompt, - nl_actions,type_hierarchy) + actions, predicates = action_construction( + openai_llm, domain_desc, action_construction_prompt, nl_actions, type_hierarchy + ) print("END OF STEP FOUR") - types = format_types(type_hierarchy) # retrieve types - types = prune_types(types=types, predicates=predicates, actions=actions) # discard types not in predicates / actions + duplicates - types = {name: description for name, description in types.items() if name not in unsupported_keywords} # remove unsupported words + types = format_types(type_hierarchy) # retrieve types + types = prune_types( + types=types, predicates=predicates, actions=actions + ) # discard types not in predicates / actions + duplicates + types = { + name: description + for name, description in types.items() + if name not in unsupported_keywords + } # remove unsupported words # STEP FIVE: task extraction - objects, initial, goal = task_extraction(openai_llm, problem_desc, - task_extraction_prompt, types, predicates, actions) - + objects, initial, goal = task_extraction( + openai_llm, problem_desc, task_extraction_prompt, types, predicates, actions + ) + print("END OF STEP FIVE") # format strings - predicate_str = "\n".join([pred["clean"].replace(":", " ; ") for pred in predicates]) + predicate_str = "\n".join( + [pred["clean"].replace(":", " ; ") for pred in predicates] + ) types_str = "\n".join(types) - requirements = [':strips',':typing',':equality',':negative-preconditions', - ':disjunctive-preconditions',':universal-preconditions',':conditional-effects'] - + requirements = [ + ":strips", + ":typing", + ":equality", + ":negative-preconditions", + ":disjunctive-preconditions", + ":universal-preconditions", + ":conditional-effects", + ] + # generate PDDL specifications - pddl_domain = domain_builder.generate_domain(domain="test_domain", requirements=requirements, - types=types_str, predicates=predicate_str, actions=actions) - pddl_problem = task_builder.generate_task(domain="test_domain", problem="test_problem", objects=objects, initial=initial, goal=goal) + pddl_domain = domain_builder.generate_domain( + domain="test_domain", + requirements=requirements, + types=types_str, + predicates=predicate_str, + actions=actions, + ) + pddl_problem = task_builder.generate_task( + domain="test_domain", + problem="test_problem", + objects=objects, + initial=initial, + goal=goal, + ) # write files domain_file = "paper_reconstructions/nl2plan/results/domain.pddl" @@ -287,6 +370,6 @@ def task_extraction(model, problem_desc, task_extraction_prompt, types, predicat problem_file = "paper_reconstructions/nl2plan/results/problem.pddl" with open(problem_file, "w") as f: f.write(pddl_problem) - + # run planner - planner.run_fast_downward(domain_file=domain_file, problem_file=problem_file) \ No newline at end of file + planner.run_fast_downward(domain_file=domain_file, problem_file=problem_file) diff --git a/paper_reconstructions/nl2plan/prompts/blocksworld.txt b/paper_reconstructions/nl2plan/prompts/blocksworld.txt new file mode 100644 index 0000000..e4b544f --- /dev/null +++ b/paper_reconstructions/nl2plan/prompts/blocksworld.txt @@ -0,0 +1,6 @@ +The robot has four actions: pickup, putdown, stack, and unstack. The domain assumes a world where there are a set of blocks that can be stacked on top of each other, an arm that can hold one block at a time, and a table where blocks can be placed. +The actions defined in this domain include: +pickup: allows the arm to pick up a block from the table if it is clear and the arm is empty. After the pickup action, the arm will be holding the block, and the block will no longer be on the table or clear. +putdown: allows the arm to put down a block on the table if it is holding a block. After the putdown action, the arm will be empty, and the block will be on the table and clear. +stack: allows the arm to stack a block on top of another block if the arm is holding the top block and the bottom block is clear. After the stack action, the arm will be empty, the top block will be on top of the bottom block, and the bottom block will no longer be clear. +unstack: allows the arm to unstack a block from on top of another block if the arm is empty and the top block is clear. After the unstack action, the arm will be holding the top block, the top block will no longer be on top of the bottom block, and the bottom block will be clear. \ No newline at end of file diff --git a/paper_reconstructions/nl2plan/prompts/blocksworld_p1.txt b/paper_reconstructions/nl2plan/prompts/blocksworld_p1.txt new file mode 100644 index 0000000..ee1d196 --- /dev/null +++ b/paper_reconstructions/nl2plan/prompts/blocksworld_p1.txt @@ -0,0 +1 @@ +There are four blocks currently. The blue block is on the red which is on the yellow. The yellow and the green are on the table. I want the red on top of the green. \ No newline at end of file diff --git a/paper_reconstructions/proc2pddl/proc2pddl.py b/paper_reconstructions/proc2pddl/proc2pddl.py index 8e91fa9..b2a269d 100644 --- a/paper_reconstructions/proc2pddl/proc2pddl.py +++ b/paper_reconstructions/proc2pddl/proc2pddl.py @@ -8,16 +8,19 @@ from l2p import * from l2p.utils.pddl_planner import FastDownward + def load_file(file_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: return file.read().strip() + def load_json(file_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: return json.load(file) + engine = "gpt-4o-mini" -api_key = os.environ.get('OPENAI_API_KEY') +api_key = os.environ.get("OPENAI_API_KEY") openai_llm = OPENAI(model=engine, api_key=api_key) domain_builder = DomainBuilder() @@ -27,35 +30,53 @@ def load_json(file_path): planner = FastDownward() # annotated original domain header -types = load_json('paper_reconstructions/proc2pddl/prompts/types.json') -predicates = load_json('paper_reconstructions/proc2pddl/prompts/predicates.json') -nl_actions = load_json('paper_reconstructions/proc2pddl/prompts/nl_actions.json') +types = load_json("paper_reconstructions/proc2pddl/prompts/types.json") +predicates = load_json("paper_reconstructions/proc2pddl/prompts/predicates.json") +nl_actions = load_json("paper_reconstructions/proc2pddl/prompts/nl_actions.json") if __name__ == "__main__": - - unsupported_keywords = ['object', 'pddl', 'lisp'] - + + unsupported_keywords = ["object", "pddl", "lisp"] + # retrieve wikihow text - domain_desc = load_file('paper_reconstructions/proc2pddl/prompts/wikihow.txt') + domain_desc = load_file("paper_reconstructions/proc2pddl/prompts/wikihow.txt") # ZPD prompt - role = load_file('paper_reconstructions/proc2pddl/prompts/zpd_prompt/role.txt') - technique = load_file('paper_reconstructions/proc2pddl/prompts/zpd_prompt/technique.txt') - example = load_file('paper_reconstructions/proc2pddl/prompts/zpd_prompt/example.txt') - task = "here are the actions I want:\n" + (str(nl_actions)) + "\n\nhere are the types I have:\n" + format_dict(types) \ - + "\n\nhere are the predicates I have:\n" + format_predicates(predicates) - ZPD_prompt = PromptBuilder(role=role, technique=technique, examples=[example], task=task) + role = load_file("paper_reconstructions/proc2pddl/prompts/zpd_prompt/role.txt") + technique = load_file( + "paper_reconstructions/proc2pddl/prompts/zpd_prompt/technique.txt" + ) + example = load_file( + "paper_reconstructions/proc2pddl/prompts/zpd_prompt/example.txt" + ) + task = ( + "here are the actions I want:\n" + + (str(nl_actions)) + + "\n\nhere are the types I have:\n" + + format_dict(types) + + "\n\nhere are the predicates I have:\n" + + format_predicates(predicates) + ) + ZPD_prompt = PromptBuilder( + role=role, technique=technique, examples=[example], task=task + ) # (1) query LLM for ZPD information action_descriptions = openai_llm.query(prompt=ZPD_prompt.generate_prompt()) - + # PDDL extraction prompt - role = load_file('paper_reconstructions/proc2pddl/prompts/pddl_translate_prompt/role.txt') - example = load_file('paper_reconstructions/proc2pddl/prompts/pddl_translate_prompt/example.txt') - task = load_file('paper_reconstructions/proc2pddl/prompts/pddl_translate_prompt/task.txt') + role = load_file( + "paper_reconstructions/proc2pddl/prompts/pddl_translate_prompt/role.txt" + ) + example = load_file( + "paper_reconstructions/proc2pddl/prompts/pddl_translate_prompt/example.txt" + ) + task = load_file( + "paper_reconstructions/proc2pddl/prompts/pddl_translate_prompt/task.txt" + ) task += "\n\nhere are the action descriptions to use:\n" + action_descriptions pddl_extract_prompt = PromptBuilder(role=role, examples=[example], task=task) - + # (2) extract PDDL requirements actions, _, llm_response = domain_builder.extract_pddl_actions( model=openai_llm, @@ -63,35 +84,48 @@ def load_json(file_path): prompt_template=pddl_extract_prompt.generate_prompt(), nl_actions=nl_actions, predicates=predicates, - types=types - ) + types=types, + ) + + types = format_types(types) # retrieve types + pruned_types = { + name: description + for name, description in types.items() + if name not in unsupported_keywords + } # remove unsupported words - types = format_types(types) # retrieve types - pruned_types = {name: description for name, description in types.items() if name not in unsupported_keywords} # remove unsupported words - # format strings predicate_str = "\n".join([pred["clean"] for pred in predicates]) types_str = "\n".join(pruned_types) - requirements = [':strips',':typing',':equality',':negative-preconditions',':disjunctive-preconditions',':universal-preconditions',':conditional-effects'] + requirements = [ + ":strips", + ":typing", + ":equality", + ":negative-preconditions", + ":disjunctive-preconditions", + ":universal-preconditions", + ":conditional-effects", + ] # generate domain pddl_domain = domain_builder.generate_domain( - domain="survive_deserted_island", + domain="survive_deserted_island", requirements=requirements, types=types_str, predicates=predicate_str, - actions=actions - ) + actions=actions, + ) domain_file = "paper_reconstructions/proc2pddl/results/domain.pddl" with open(domain_file, "w") as f: f.write(pddl_domain) print("PDDL domain:\n", pddl_domain) - - problem_file = load_file("paper_reconstructions/proc2pddl/results/problems/problem-catch_cook_fish.pddl") - + + problem_file = load_file( + "paper_reconstructions/proc2pddl/results/problems/problem-catch_cook_fish.pddl" + ) + # run planner planner.run_fast_downward(domain_file=domain_file, problem_file=problem_file) - \ No newline at end of file diff --git a/setup.py b/setup.py index 338c097..c0a45a0 100644 --- a/setup.py +++ b/setup.py @@ -12,11 +12,7 @@ long_description_content_type="text/markdown", author="Marcus Tantakoun", author_email="mtantakoun@gmail.com", - install_requires=[ - "retry", - "pddl", - "typing_extensions" - ], + install_requires=["retry", "pddl", "typing_extensions"], license="MIT", url="https://github.com/AI-Planning/l2p", classifiers=[ diff --git a/tests/__init__.py b/tests/__init__.py index fca057d..02bdc25 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1 @@ -from l2p import * \ No newline at end of file +from l2p import * diff --git a/tests/mock_llm.py b/tests/mock_llm.py index 394b022..265eeae 100644 --- a/tests/mock_llm.py +++ b/tests/mock_llm.py @@ -18,4 +18,4 @@ def reset_tokens(self): """ Placeholder for resetting tokens; not needed for testing. """ - pass \ No newline at end of file + pass diff --git a/tests/parse.py b/tests/parse.py index e334678..d60d3bb 100644 --- a/tests/parse.py +++ b/tests/parse.py @@ -2,6 +2,7 @@ from pddl import parse_domain, parse_problem import sys + def check_parse_domain(file_path): try: domain = parse_domain(file_path) @@ -13,6 +14,7 @@ def check_parse_domain(file_path): print("------------------") sys.exit(1) + def check_parse_problem(file_path): try: problem = parse_problem(file_path) @@ -24,13 +26,14 @@ def check_parse_problem(file_path): print("------------------") sys.exit(1) + if __name__ == "__main__": - - domain_file_path = 'paper_reconstructions/llm+dm/results/domain.pddl' - problem_file_path = 'data/problem_1.pddl' - + + domain_file_path = "paper_reconstructions/llm+dm/results/domain.pddl" + problem_file_path = "data/problem_1.pddl" + pddl_domain = check_parse_domain(domain_file_path) print("PDDL domain:\n", pddl_domain) - + with open(domain_file_path, "w") as f: - f.write(pddl_domain) \ No newline at end of file + f.write(pddl_domain) diff --git a/tests/test_domain_builder.py b/tests/test_domain_builder.py index 48e1f50..9507fc3 100644 --- a/tests/test_domain_builder.py +++ b/tests/test_domain_builder.py @@ -1,12 +1,13 @@ import unittest from l2p.domain_builder import DomainBuilder from l2p.utils.pddl_parser import * -from .mock_llm import MockLLM +from .mock_llm import MockLLM + class TestDomainBuilder(unittest.TestCase): def setUp(self): self.domain_builder = DomainBuilder() - self.domain_desc = 'Blocksworld is...' + self.domain_desc = "Blocksworld is..." self.prompt_template = "Domain: {domain_desc}\nTypes: {types}" self.types = None @@ -14,80 +15,85 @@ def test_extract_type(self): mock_llm_1 = MockLLM(['{"robot": "A machine capable of carrying out tasks."}']) mock_llm_2 = MockLLM(["This is not a valid dictionary format."]) mock_llm_3 = MockLLM([None]) - + types, llm_response = self.domain_builder.extract_type( model=mock_llm_1, domain_desc=self.domain_desc, prompt_template=self.prompt_template, - types=self.types + types=self.types, ) self.assertEqual(types, {"robot": "A machine capable of carrying out tasks."}) - self.assertEqual(llm_response, '{"robot": "A machine capable of carrying out tasks."}') - + self.assertEqual( + llm_response, '{"robot": "A machine capable of carrying out tasks."}' + ) + with self.assertRaises(RuntimeError) as context: types, llm_response = self.domain_builder.extract_type( model=mock_llm_2, domain_desc=self.domain_desc, prompt_template=self.prompt_template, - types=self.types + types=self.types, ) self.assertIn("Max retries exceeded", str(context.exception)) - + with self.assertRaises(RuntimeError) as context: types, llm_response = self.domain_builder.extract_type( model=mock_llm_3, domain_desc=self.domain_desc, prompt_template=self.prompt_template, - types=self.types + types=self.types, ) self.assertIn("Max retries exceeded", str(context.exception)) - + def test_extract_type_hierarchy(self): mock_model = MockLLM(["{'car': 'vehicle', 'truck': 'vehicle'}"]) - + domain_desc = "A domain about vehicles" prompt_template = "Extract types from: {domain_desc}. Current types: {types}." types = {"bike": "vehicle"} - + expected_hierarchy = {"car": "vehicle", "truck": "vehicle"} expected_response = "{'car': 'vehicle', 'truck': 'vehicle'}" - + result, response = self.domain_builder.extract_type_hierarchy( model=mock_model, domain_desc=domain_desc, prompt_template=prompt_template, - types=types + types=types, ) - + self.assertEqual(result, expected_hierarchy) self.assertEqual(response, expected_response) - + def test_extract_nl_actions(self): - mock_model = MockLLM(["{'drive': 'Move a vehicle', 'park': 'Stop a vehicle at a location'}"]) - + mock_model = MockLLM( + ["{'drive': 'Move a vehicle', 'park': 'Stop a vehicle at a location'}"] + ) + domain_desc = "Vehicle domain" prompt_template = "Extract actions for: {domain_desc}. Current actions: {nl_actions}. Current types: {types}." nl_actions = {"start": "Initiate a vehicle"} types = {"car": "vehicle"} - + expected_actions = { "drive": "Move a vehicle", - "park": "Stop a vehicle at a location" + "park": "Stop a vehicle at a location", } - expected_response = "{'drive': 'Move a vehicle', 'park': 'Stop a vehicle at a location'}" - + expected_response = ( + "{'drive': 'Move a vehicle', 'park': 'Stop a vehicle at a location'}" + ) + result, response = self.domain_builder.extract_nl_actions( model=mock_model, domain_desc=domain_desc, prompt_template=prompt_template, nl_actions=nl_actions, - types=types + types=types, ) - + self.assertEqual(result, expected_actions) self.assertEqual(response, expected_response) - - + def test_pddl_action(self): self.maxDiff = None llm_response = """ @@ -121,7 +127,7 @@ def test_pddl_action(self): ``` """ mock_model = MockLLM([llm_response]) - + domain_desc = "Vehicle domain" prompt_template = "Extract PDDL action for: {domain_desc}. Action: {action_name} with description {action_desc}." action_name = "drive" @@ -129,13 +135,21 @@ def test_pddl_action(self): action_list = {"park": "Stop a vehicle"} types = {"car": "vehicle"} predicates = [ - {"name": "at", "desc": "true if the object ?o (a vehicle or a worker) is at the location ?l", "parameters": ["?o - object", "?l - location"]}, - {"name": "connected", "desc": "true if a road exists between ?l1 and ?l2 allowing vehicle travel between them.", "parameters": ["?l1 - location", "?l2 - location"]}, + { + "name": "at", + "desc": "true if the object ?o (a vehicle or a worker) is at the location ?l", + "parameters": ["?o - object", "?l - location"], + }, + { + "name": "connected", + "desc": "true if a road exists between ?l1 and ?l2 allowing vehicle travel between them.", + "parameters": ["?l1 - location", "?l2 - location"], + }, ] - expected_predicates = predicates + expected_predicates = predicates expected_response = llm_response - + action, new_preds, response = self.domain_builder.extract_pddl_action( model=mock_model, domain_desc=domain_desc, @@ -144,31 +158,32 @@ def test_pddl_action(self): action_desc=action_desc, action_list=action_list, predicates=predicates, - types=types + types=types, ) - + # TODO: finish this comparing actual Action type to this response. - + print("\n", action) print("\n", new_preds) - + def test_extract_pddl_actions(self): pass - + def test_extract_parameters(self): pass - + def test_extract_preconditions(self): pass - + def test_extract_effects(self): pass - + def test_extract_predicates(self): pass - + def test_generate_domain(self): pass + if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/test_task_builder.py b/tests/test_task_builder.py index 12db133..4615c0c 100644 --- a/tests/test_task_builder.py +++ b/tests/test_task_builder.py @@ -1,42 +1,44 @@ import unittest from l2p.task_builder import TaskBuilder from l2p.utils.pddl_parser import * -from .mock_llm import MockLLM +from .mock_llm import MockLLM + class TestTaskBuilder(unittest.TestCase): def setUp(self): self.domain_builder = TaskBuilder() - self.domain_desc = 'Blocksworld is...' - + self.domain_desc = "Blocksworld is..." + def test_extract_objects(self): pass - + def test_extract_initial_state(self): pass - + def test_extract_object_state(self): pass - + def test_extract_task(self): pass - + def test_extract_nl_conditions(self): pass - + def test_generate_task(self): pass - + def test_format_action(self): pass - + def test_format_objects(self): pass - + def test_format_initial(self): pass - + def test_format_goal(self): pass + if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/usage/demonstration.py b/tests/usage/demonstration.py index a9a9a53..64abe4f 100644 --- a/tests/usage/demonstration.py +++ b/tests/usage/demonstration.py @@ -1,39 +1,39 @@ import os from l2p import * -builder = DomainBuilder() # create domain build class +builder = DomainBuilder() # create domain build class + + +def run_aba_alg( + model: LLM, action_model, domain_desc, hierarchy, prompt, max_iter: int = 2 +) -> tuple[list[Predicate], list[Action]]: -def run_aba_alg(model: LLM, action_model, domain_desc, - hierarchy, prompt, max_iter: int=2 - ) -> tuple[list[Predicate], list[Action]]: - actions = list(action_model.keys()) pred_list = [] - + for _ in range(max_iter): action_list = [] # iterate each action spec. + new predicates for _, action in enumerate(actions): if len(pred_list) == 0: - prompt = prompt.replace('{predicates}', - '\nNo predicate has been defined yet') + prompt = prompt.replace( + "{predicates}", "\nNo predicate has been defined yet" + ) else: res = "" for i, p in enumerate(pred_list): - res += f'\n{i + 1}. {p["raw"]}' - prompt = prompt.replace('{predicates}', res) + res += f'\n{i + 1}. {p["raw"]}' + prompt = prompt.replace("{predicates}", res) # extract pddl action and predicates - pddl_action, new_preds, response = ( - builder.extract_pddl_action( - model=model, - domain_desc=domain_desc, - prompt_template=prompt, - action_name=action, - action_desc=action_model[action]['desc'], - action_list=action_list, - predicates=pred_list, - types=hierarchy["hierarchy"] - ) + pddl_action, new_preds, response = builder.extract_pddl_action( + model=model, + domain_desc=domain_desc, + prompt_template=prompt, + action_name=action, + action_desc=action_model[action]["desc"], + action_list=action_list, + predicates=pred_list, + types=hierarchy["hierarchy"], ) # format + add extracted actions and predicates new_preds = parse_new_predicates(response) @@ -43,127 +43,135 @@ def run_aba_alg(model: LLM, action_model, domain_desc, return pred_list, action_list + def run_aba(): - + # retrieve prompt information - base_path='paper_reconstructions/llm+dm/prompts/' - action_model=load_file(f'{base_path}action_model.json') - domain_desc=load_file(f'{base_path}domain_desc.txt') - hier=load_file( - f'{base_path}hierarchy_requirements.json') - prompt=load_file(f'{base_path}pddl_prompt.txt') + base_path = "paper_reconstructions/llm+dm/prompts/" + action_model = load_file(f"{base_path}action_model.json") + domain_desc = load_file(f"{base_path}domain_desc.txt") + hier = load_file(f"{base_path}hierarchy_requirements.json") + prompt = load_file(f"{base_path}pddl_prompt.txt") # initialise LLM engine (OpenAI in this case) - api_key = os.environ.get('OPENAI_API_KEY') + api_key = os.environ.get("OPENAI_API_KEY") llm = OPENAI(model="gpt-4o-mini", api_key=api_key) # run "action-by-action" algorithm pred, action = run_aba_alg( - model=llm, + model=llm, action_model=action_model, domain_desc=domain_desc, hierarchy=hier, - prompt=prompt) + prompt=prompt, + ) + def run_predicates(): - + domain_builder = DomainBuilder() - - api_key = os.environ.get('OPENAI_API_KEY') + + api_key = os.environ.get("OPENAI_API_KEY") llm = OPENAI(model="gpt-4o-mini", api_key=api_key) - + # retrieve prompt information - base_path='tests/usage/prompts/domain/' - domain_desc = load_file(f'{base_path}blocksworld_domain.txt') - extract_predicates_prompt = load_file(f'{base_path}extract_predicates.txt') - types = load_file(f'{base_path}types.json') - action = load_file(f'{base_path}action.json') - + base_path = "tests/usage/prompts/domain/" + domain_desc = load_file(f"{base_path}blocksworld_domain.txt") + extract_predicates_prompt = load_file(f"{base_path}extract_predicates.txt") + types = load_file(f"{base_path}types.json") + action = load_file(f"{base_path}action.json") + # extract predicates via LLM predicates, llm_output = domain_builder.extract_predicates( model=llm, domain_desc=domain_desc, prompt_template=extract_predicates_prompt, types=types, - nl_actions={action['action_name']: action['action_desc']} - ) - + nl_actions={action["action_name"]: action["action_desc"]}, + ) + # format key info into PDDL strings - predicate_str = "\n".join([pred["clean"].replace(":", " ; ") for pred in predicates]) - + predicate_str = "\n".join( + [pred["clean"].replace(":", " ; ") for pred in predicates] + ) + print(f"PDDL domain predicates:\n{predicate_str}") - + return predicates - + + def run_task(): - + task_builder = TaskBuilder() - - api_key = os.environ.get('OPENAI_API_KEY') + + api_key = os.environ.get("OPENAI_API_KEY") llm = OPENAI(model="gpt-4o-mini", api_key=api_key) - + # load in assumptions - problem_desc = load_file(r'tests/usage/prompts/problem/blocksworld_problem.txt') - extract_task_prompt = load_file(r'tests/usage/prompts/problem/extract_task.txt') - types = load_file(r'tests/usage/prompts/domain/types.json') - predicates_json = load_file(r'tests/usage/prompts/domain/predicates.json') + problem_desc = load_file(r"tests/usage/prompts/problem/blocksworld_problem.txt") + extract_task_prompt = load_file(r"tests/usage/prompts/problem/extract_task.txt") + types = load_file(r"tests/usage/prompts/domain/types.json") + predicates_json = load_file(r"tests/usage/prompts/domain/predicates.json") predicates: List[Predicate] = [Predicate(**item) for item in predicates_json] - + # extract PDDL task specifications via LLM objects, initial_states, goal_states, llm_response = task_builder.extract_task( model=llm, problem_desc=problem_desc, prompt_template=extract_task_prompt, types=types, - predicates=predicates - ) - + predicates=predicates, + ) + # format key info into PDDL strings objects_str = task_builder.format_objects(objects) initial_str = task_builder.format_initial(initial_states) goal_str = task_builder.format_goal(goal_states) - + # generate task file pddl_problem = task_builder.generate_task( - domain="blocksworld", - problem="blocksworld_problem", - objects=objects_str, - initial=initial_str, - goal=goal_str) - + domain="blocksworld", + problem="blocksworld_problem", + objects=objects_str, + initial=initial_str, + goal=goal_str, + ) + print(f"### LLM OUTPUT:\n {pddl_problem}") - + print(llm_response) - + + def run_feedback(): - + feedback_builder = FeedbackBuilder() - - api_key = os.environ.get('OPENAI_API_KEY') + + api_key = os.environ.get("OPENAI_API_KEY") llm = OPENAI(model="gpt-4o-mini", api_key=api_key) - - problem_desc = load_file(r'tests/usage/prompts/problem/blocksworld_problem.txt') - types = load_file(r'tests/usage/prompts/domain/types.json') - feedback_template = load_file(r'tests/usage/prompts/problem/feedback.txt') - predicates_json = load_file(r'tests/usage/prompts/domain/predicates.json') + + problem_desc = load_file(r"tests/usage/prompts/problem/blocksworld_problem.txt") + types = load_file(r"tests/usage/prompts/domain/types.json") + feedback_template = load_file(r"tests/usage/prompts/problem/feedback.txt") + predicates_json = load_file(r"tests/usage/prompts/domain/predicates.json") predicates: List[Predicate] = [Predicate(**item) for item in predicates_json] - llm_response = load_file(r'tests/usage/prompts/domain/llm_output_task.txt') - + llm_response = load_file(r"tests/usage/prompts/domain/llm_output_task.txt") + objects, initial, goal, feedback_response = feedback_builder.task_feedback( - model=llm, - problem_desc=problem_desc, - feedback_template=feedback_template, - feedback_type="llm", + model=llm, + problem_desc=problem_desc, + feedback_template=feedback_template, + feedback_type="llm", predicates=predicates, - types=types, - llm_response=llm_response) + types=types, + llm_response=llm_response, + ) print("FEEDBACK:\n", feedback_response) + if __name__ == "__main__": - + # run_aba() # run_predicates() # run_task() run_feedback() - \ No newline at end of file diff --git a/tests/usage/usage.py b/tests/usage/usage.py index d802b2d..1e21b9b 100644 --- a/tests/usage/usage.py +++ b/tests/usage/usage.py @@ -2,34 +2,43 @@ from l2p import * from tests.parse import check_parse_domain + def load_file(file_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: return file.read().strip() + def load_json(file_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: return json.load(file) + domain_builder = DomainBuilder() # engine = "gpt-4o" engine = "gpt-4o-mini" # engine = "gpt-3.5-turbo-0125" -api_key = os.environ.get('OPENAI_API_KEY') +api_key = os.environ.get("OPENAI_API_KEY") openai_llm = OPENAI(model=engine, api_key=api_key) # load in assumptions -domain_desc = load_file('tests/usage/prompts/domain/blocksworld_domain.txt') -extract_predicates_prompt = load_file('tests/usage/prompts/domain/extract_predicates.txt') -extract_parameters_prompt = load_file('tests/usage/prompts/domain/extract_parameters.txt') -extract_preconditions_prompt = load_file('tests/usage/prompts/domain/extract_preconditions.txt') -extract_effects_prompt = load_file('tests/usage/prompts/domain/extract_effects.txt') -types = load_json('tests/usage/prompts/domain/types.json') -action = load_json('tests/usage/prompts/domain/action.json') -action_name = action['action_name'] -action_desc = action['action_desc'] - -unsupported_keywords = ['object', 'pddl', 'lisp'] +domain_desc = load_file("tests/usage/prompts/domain/blocksworld_domain.txt") +extract_predicates_prompt = load_file( + "tests/usage/prompts/domain/extract_predicates.txt" +) +extract_parameters_prompt = load_file( + "tests/usage/prompts/domain/extract_parameters.txt" +) +extract_preconditions_prompt = load_file( + "tests/usage/prompts/domain/extract_preconditions.txt" +) +extract_effects_prompt = load_file("tests/usage/prompts/domain/extract_effects.txt") +types = load_json("tests/usage/prompts/domain/types.json") +action = load_json("tests/usage/prompts/domain/action.json") +action_name = action["action_name"] +action_desc = action["action_desc"] + +unsupported_keywords = ["object", "pddl", "lisp"] # extract predicates predicates, llm_output = domain_builder.extract_predicates( @@ -37,8 +46,8 @@ def load_json(file_path): domain_desc=domain_desc, prompt_template=extract_predicates_prompt, types=types, - nl_actions={action_name:action_desc} - ) + nl_actions={action_name: action_desc}, +) # extract parameters params, params_raw, llm_output = domain_builder.extract_parameters( @@ -47,8 +56,8 @@ def load_json(file_path): prompt_template=extract_parameters_prompt, action_name=action_name, action_desc=action_desc, - types=types - ) + types=types, +) # extract preconditions preconditions, new_predicates, llm_output = domain_builder.extract_preconditions( @@ -58,10 +67,10 @@ def load_json(file_path): action_name=action_name, action_desc=action_desc, params=params_raw, - predicates=predicates - ) + predicates=predicates, +) -predicates.extend(new_predicates) # add new predicates +predicates.extend(new_predicates) # add new predicates # extract preconditions effects, new_predicates, llm_output = domain_builder.extract_effects( @@ -72,47 +81,58 @@ def load_json(file_path): action_desc=action_desc, params=params_raw, precondition=preconditions, - predicates=predicates - ) + predicates=predicates, +) -predicates.extend(new_predicates) # add new predicates +predicates.extend(new_predicates) # add new predicates # assemble action model action = { - 'name': action_name, - 'parameters': params, - 'preconditions': preconditions, - 'effects': effects - } + "name": action_name, + "parameters": params, + "preconditions": preconditions, + "effects": effects, +} # discard predicates not found in action models + duplicates predicates = prune_predicates(predicates=predicates, actions=[action]) # format types and remove unsupported words types = format_types(types) -types = {name: description for name, description in types.items() if name not in unsupported_keywords} +types = { + name: description + for name, description in types.items() + if name not in unsupported_keywords +} # format key info into strings -predicate_str = '\n'.join([pred['clean'].replace(':', ' ; ') for pred in predicates]) -types_str = '\n'.join(types) +predicate_str = "\n".join([pred["clean"].replace(":", " ; ") for pred in predicates]) +types_str = "\n".join(types) # generate PDDL domain pddl_domain = domain_builder.generate_domain( - domain='blocksworld_domain', - requirements=[':strips',':typing',':equality',':negative-preconditions', - ':disjunctive-preconditions',':universal-preconditions',':conditional-effects'], + domain="blocksworld_domain", + requirements=[ + ":strips", + ":typing", + ":equality", + ":negative-preconditions", + ":disjunctive-preconditions", + ":universal-preconditions", + ":conditional-effects", + ], types=types_str, predicates=predicate_str, - actions=[action] - ) + actions=[action], +) -domain_file_path = 'tests/usage/results/domain.pddl' +domain_file_path = "tests/usage/results/domain.pddl" with open(domain_file_path, "w") as f: - f.write(pddl_domain) + f.write(pddl_domain) pddl_domain = check_parse_domain(domain_file_path) print("PDDL domain:\n", pddl_domain) with open(domain_file_path, "w") as f: - f.write(pddl_domain) \ No newline at end of file + f.write(pddl_domain) From 9b620cf1419b897a5ab5fb8a2c0c4c7197edc8b1 Mon Sep 17 00:00:00 2001 From: Marcus Tantakoun Date: Sat, 14 Dec 2024 12:18:36 -0500 Subject: [PATCH 03/10] clean up some unneccesary imports --- l2p/llm_builder.py | 2 +- l2p/utils/pddl_parser.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/l2p/llm_builder.py b/l2p/llm_builder.py index 39a924b..8685d52 100644 --- a/l2p/llm_builder.py +++ b/l2p/llm_builder.py @@ -4,7 +4,7 @@ It also offers extension to OpenAI and Huggingface, but can generalise to any third-party LLM store. """ -import os, logging, argparse, functools +import logging, functools from retry import retry from abc import ABC, abstractmethod from typing_extensions import override diff --git a/l2p/utils/pddl_parser.py b/l2p/utils/pddl_parser.py index dc25711..a204bdb 100644 --- a/l2p/utils/pddl_parser.py +++ b/l2p/utils/pddl_parser.py @@ -233,10 +233,6 @@ def parse_initial(llm_response: str) -> list[dict[str, str]]: """ state_head = extract_heading(llm_response, "INITIAL") state_raw = combine_blocks(state_head) - print("STATE RAW") - print(state_raw) - print("END STATE RAW") - state_clean = clear_comments(state_raw) states = [] From 098fc8cc9a90ee694d96f168af47b9c74204e9a2 Mon Sep 17 00:00:00 2001 From: Marcus Tantakoun Date: Sat, 21 Dec 2024 00:53:13 -0500 Subject: [PATCH 04/10] implemented more test cases --- .gitignore | 1 + l2p/domain_builder.py | 40 ++-- l2p/task_builder.py | 1 + tests/mock_llm.py | 2 +- tests/test_domain_builder.py | 216 ++++++------------ .../test_extract_effects/01.txt | 2 + .../test_extract_effects/02.txt | 2 + .../test_extract_effects/03.txt | 2 + .../test_extract_nl_actions/01.txt | 2 + .../test_extract_nl_actions/02.txt | 2 + .../test_extract_nl_actions/03.txt | 2 + .../test_extract_parameters/01.txt | 2 + .../test_extract_parameters/02.txt | 2 + .../test_extract_parameters/03.txt | 2 + .../test_extract_pddl_action/01.txt | 2 + .../test_extract_pddl_action/02.txt | 2 + .../test_extract_pddl_action/03.txt | 2 + .../test_extract_pddl_actions/01.txt | 2 + .../test_extract_pddl_actions/02.txt | 2 + .../test_extract_pddl_actions/03.txt | 2 + .../test_extract_preconditions/01.txt | 2 + .../test_extract_preconditions/02.txt | 2 + .../test_extract_preconditions/03.txt | 2 + .../test_extract_predicates/01.txt | 2 + .../test_extract_predicates/02.txt | 2 + .../test_extract_predicates/03.txt | 2 + .../test_extract_type/01.txt | 35 +++ .../test_extract_type/02.txt | 1 + .../test_extract_type/03.txt | 2 + .../test_extract_type_hierarchy/01.txt | 120 ++++++++++ .../test_extract_type_hierarchy/02.txt | 1 + .../test_extract_type_hierarchy/03.txt | 2 + .../test_extract_goal_state/01.txt | 2 + .../test_extract_goal_state/02.txt | 2 + .../test_extract_goal_state/03.txt | 2 + .../test_extract_initial_state/01.txt | 2 + .../test_extract_initial_state/02.txt | 2 + .../test_extract_initial_state/03.txt | 2 + .../test_extract_objects/01.txt | 2 + .../test_extract_objects/02.txt | 2 + .../test_extract_objects/03.txt | 2 + .../test_extract_task/01.txt | 2 + .../test_extract_task/02.txt | 2 + .../test_extract_task/03.txt | 2 + tests/test_task_builder.py | 8 +- 45 files changed, 333 insertions(+), 162 deletions(-) create mode 100644 tests/test_prompts/test_domain_builder/test_extract_effects/01.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_effects/02.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_effects/03.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_nl_actions/01.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_nl_actions/02.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_nl_actions/03.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_parameters/01.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_parameters/02.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_parameters/03.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_pddl_action/01.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_pddl_action/02.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_pddl_action/03.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_pddl_actions/01.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_pddl_actions/02.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_pddl_actions/03.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_preconditions/01.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_preconditions/02.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_preconditions/03.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_predicates/01.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_predicates/02.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_predicates/03.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_type/01.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_type/02.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_type/03.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_type_hierarchy/01.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_type_hierarchy/02.txt create mode 100644 tests/test_prompts/test_domain_builder/test_extract_type_hierarchy/03.txt create mode 100644 tests/test_prompts/test_task_builder/test_extract_goal_state/01.txt create mode 100644 tests/test_prompts/test_task_builder/test_extract_goal_state/02.txt create mode 100644 tests/test_prompts/test_task_builder/test_extract_goal_state/03.txt create mode 100644 tests/test_prompts/test_task_builder/test_extract_initial_state/01.txt create mode 100644 tests/test_prompts/test_task_builder/test_extract_initial_state/02.txt create mode 100644 tests/test_prompts/test_task_builder/test_extract_initial_state/03.txt create mode 100644 tests/test_prompts/test_task_builder/test_extract_objects/01.txt create mode 100644 tests/test_prompts/test_task_builder/test_extract_objects/02.txt create mode 100644 tests/test_prompts/test_task_builder/test_extract_objects/03.txt create mode 100644 tests/test_prompts/test_task_builder/test_extract_task/01.txt create mode 100644 tests/test_prompts/test_task_builder/test_extract_task/02.txt create mode 100644 tests/test_prompts/test_task_builder/test_extract_task/03.txt diff --git a/.gitignore b/.gitignore index 3fe66e8..6eb970a 100644 --- a/.gitignore +++ b/.gitignore @@ -161,6 +161,7 @@ cython_debug/ # directories to ignore tests/main.py +tests/parse.py data/ tests/tests.py diff --git a/l2p/domain_builder.py b/l2p/domain_builder.py index e7c2772..8424b65 100644 --- a/l2p/domain_builder.py +++ b/l2p/domain_builder.py @@ -79,10 +79,10 @@ def extract_type( if types is not None: return types, llm_response - print(f"Attempt {attempt + 1}/{max_retries}: Failed to extract types.") except Exception as e: print( - f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + f"Error encountered during attempt {attempt + 1}/{max_retries}: {e}. " + f"LLM Output: \n\n{llm_response if 'llm_response' in locals() else 'None'}\n\n Retrying..." ) time.sleep(1) # add a delay before retrying @@ -128,12 +128,14 @@ def extract_type_hierarchy( # extract respective types from response type_hierarchy = convert_to_dict(llm_response=llm_response) - - return type_hierarchy, llm_response + + if type_hierarchy is not None: + return type_hierarchy, llm_response except Exception as e: print( - f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + f"Error encountered during attempt {attempt + 1}/{max_retries}: {e}. " + f"LLM Output: \n\n{llm_response if 'llm_response' in locals() else 'None'}\n\n Retrying..." ) time.sleep(1) # add a delay before retrying @@ -189,12 +191,14 @@ def extract_nl_actions( # extract respective types from response nl_actions = convert_to_dict(llm_response=llm_response) - - return nl_actions, llm_response + + if nl_actions is not None: + return nl_actions, llm_response except Exception as e: print( - f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + f"Error encountered during attempt {attempt + 1}/{max_retries}: {e}. " + f"LLM Output: \n\n{llm_response if 'llm_response' in locals() else 'None'}\n\n Retrying..." ) time.sleep(1) # add a delay before retrying @@ -264,12 +268,14 @@ def extract_pddl_action( llm_response=llm_response, action_name=action_name ) new_predicates = parse_new_predicates(llm_response) - - return action, new_predicates, llm_response + + if action is not None and new_predicates is not None: + return action, new_predicates, llm_response except Exception as e: print( - f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + f"Error encountered during attempt {attempt + 1}/{max_retries}: {e}. " + f"LLM Output: \n\n{llm_response if 'llm_response' in locals() else 'None'}\n\n Retrying..." ) time.sleep(1) # add a delay before retrying @@ -409,7 +415,8 @@ def extract_parameters( except Exception as e: print( - f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + f"Error encountered during attempt {attempt + 1}/{max_retries}: {e}. " + f"LLM Output: \n\n{llm_response if 'llm_response' in locals() else 'None'}\n\n Retrying..." ) time.sleep(1) # add a delay before retrying @@ -478,7 +485,8 @@ def extract_preconditions( except Exception as e: print( - f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + f"Error encountered during attempt {attempt + 1}/{max_retries}: {e}. " + f"LLM Output: \n\n{llm_response if 'llm_response' in locals() else 'None'}\n\n Retrying..." ) time.sleep(1) # add a delay before retrying @@ -550,7 +558,8 @@ def extract_effects( except Exception as e: print( - f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + f"Error encountered during attempt {attempt + 1}/{max_retries}: {e}. " + f"LLM Output: \n\n{llm_response if 'llm_response' in locals() else 'None'}\n\n Retrying..." ) time.sleep(1) # add a delay before retrying @@ -614,7 +623,8 @@ def extract_predicates( except Exception as e: print( - f"Error encountered: {e}. Retrying {attempt + 1}/{max_retries}..." + f"Error encountered during attempt {attempt + 1}/{max_retries}: {e}. " + f"LLM Output: \n\n{llm_response if 'llm_response' in locals() else 'None'}\n\n Retrying..." ) time.sleep(1) # add a delay before retrying diff --git a/l2p/task_builder.py b/l2p/task_builder.py index 6e4136c..f6b6546 100644 --- a/l2p/task_builder.py +++ b/l2p/task_builder.py @@ -313,6 +313,7 @@ def extract_task( raise RuntimeError("Max retries exceeded. Failed to extract task.") + # NOTE: This function is experimental and may be subject to change in future versions. @require_llm def extract_nl_conditions( self, diff --git a/tests/mock_llm.py b/tests/mock_llm.py index 265eeae..52f6ff8 100644 --- a/tests/mock_llm.py +++ b/tests/mock_llm.py @@ -6,7 +6,7 @@ def __init__(self, responses): self.responses = responses self.current_index = 0 - def query(self, prompt): + def query(self, prompt: str = None): """ Simulates the LLM query response. """ diff --git a/tests/test_domain_builder.py b/tests/test_domain_builder.py index 9507fc3..19b4743 100644 --- a/tests/test_domain_builder.py +++ b/tests/test_domain_builder.py @@ -1,170 +1,102 @@ import unittest from l2p.domain_builder import DomainBuilder -from l2p.utils.pddl_parser import * +from l2p.utils import * from .mock_llm import MockLLM class TestDomainBuilder(unittest.TestCase): def setUp(self): self.domain_builder = DomainBuilder() - self.domain_desc = "Blocksworld is..." - self.prompt_template = "Domain: {domain_desc}\nTypes: {types}" - self.types = None def test_extract_type(self): - mock_llm_1 = MockLLM(['{"robot": "A machine capable of carrying out tasks."}']) - mock_llm_2 = MockLLM(["This is not a valid dictionary format."]) - mock_llm_3 = MockLLM([None]) + mock_llm_1 = MockLLM([load_file("tests/test_prompts/test_extract_type/01.txt")]) + mock_llm_2 = MockLLM([load_file("tests/test_prompts/test_extract_type/02.txt")]) + mock_llm_3 = MockLLM([load_file("tests/test_prompts/test_extract_type/03.txt")]) - types, llm_response = self.domain_builder.extract_type( + types, _ = self.domain_builder.extract_type( model=mock_llm_1, - domain_desc=self.domain_desc, - prompt_template=self.prompt_template, - types=self.types, - ) - self.assertEqual(types, {"robot": "A machine capable of carrying out tasks."}) - self.assertEqual( - llm_response, '{"robot": "A machine capable of carrying out tasks."}' + domain_desc="Blocksworld is a...", + prompt_template="Prompt template placeholder" ) + self.assertEqual(types, { + "object": "Object is always root, everything is an object", + "children": [ + {"arm": "mechanical arm that picks up and stacks blocks on other blocks or table.", "children": []}, + {"block": "colored block that can be stacked or stacked on other blocks or table.", "children": []}, + {"table": "surface where the blocks can be placed on top of.", "children": []} + ] + }) with self.assertRaises(RuntimeError) as context: - types, llm_response = self.domain_builder.extract_type( + types, _ = self.domain_builder.extract_type( model=mock_llm_2, - domain_desc=self.domain_desc, - prompt_template=self.prompt_template, - types=self.types, - ) - self.assertIn("Max retries exceeded", str(context.exception)) - - with self.assertRaises(RuntimeError) as context: - types, llm_response = self.domain_builder.extract_type( - model=mock_llm_3, - domain_desc=self.domain_desc, - prompt_template=self.prompt_template, - types=self.types, + domain_desc="Blocksworld is a...", + prompt_template="Prompt template placeholder" ) self.assertIn("Max retries exceeded", str(context.exception)) def test_extract_type_hierarchy(self): - mock_model = MockLLM(["{'car': 'vehicle', 'truck': 'vehicle'}"]) - - domain_desc = "A domain about vehicles" - prompt_template = "Extract types from: {domain_desc}. Current types: {types}." - types = {"bike": "vehicle"} - - expected_hierarchy = {"car": "vehicle", "truck": "vehicle"} - expected_response = "{'car': 'vehicle', 'truck': 'vehicle'}" - - result, response = self.domain_builder.extract_type_hierarchy( - model=mock_model, - domain_desc=domain_desc, - prompt_template=prompt_template, - types=types, - ) - - self.assertEqual(result, expected_hierarchy) - self.assertEqual(response, expected_response) - - def test_extract_nl_actions(self): - mock_model = MockLLM( - ["{'drive': 'Move a vehicle', 'park': 'Stop a vehicle at a location'}"] - ) - - domain_desc = "Vehicle domain" - prompt_template = "Extract actions for: {domain_desc}. Current actions: {nl_actions}. Current types: {types}." - nl_actions = {"start": "Initiate a vehicle"} - types = {"car": "vehicle"} - - expected_actions = { - "drive": "Move a vehicle", - "park": "Stop a vehicle at a location", + mock_llm_1 = MockLLM([load_file("tests/test_prompts/test_extract_type_hierarchy/01.txt")]) + mock_llm_2 = MockLLM([load_file("tests/test_prompts/test_extract_type_hierarchy/02.txt")]) + mock_llm_3 = MockLLM([load_file("tests/test_prompts/test_extract_type_hierarchy/03.txt")]) + + expected_hierarchy = { + "object": "Object is always root, everything is an object", + "children": [ + { + "worker": "A type of object consisting of humans who do things.", + "children": [ + {"administrator": "A type of worker.", "children": []}, + {"general_worker": "A type of worker.", "children": []} + ] + }, + {"order": "A type of object consisting of instructions.", "children": []}, + {"vehicle": "A type of object consisting of vehicles.", "children": []}, + { + "house_component": "A type of object consisting of the components of a house.", + "children": [ + {"wall": "A type of house_component.", "children": []}, + {"floor": "A type of house_component.", "children": []}, + {"roof": "A type of house_component.", "children": []} + ] + }, + { + "location": "A type of object consisting of places which can be visited.", + "children": [ + { + "house": "A type of location. ", + "children": [ + {"mansion": "A type of house.", "children": []}, + {"library": "A type of house.", "children": []} + ] + }, + {"depot": "A type of location.", "children": []} + ] + } + ] } - expected_response = ( - "{'drive': 'Move a vehicle', 'park': 'Stop a vehicle at a location'}" - ) - - result, response = self.domain_builder.extract_nl_actions( - model=mock_model, - domain_desc=domain_desc, - prompt_template=prompt_template, - nl_actions=nl_actions, - types=types, - ) - self.assertEqual(result, expected_actions) - self.assertEqual(response, expected_response) - - def test_pddl_action(self): - self.maxDiff = None - llm_response = """ - ### Action Parameters - ``` - - ?v - vehicle: The vehicle travelling - - ?from - location: The location travelling from - - ?to - location: The location travelling to - ``` - - ### Action Preconditions - ``` - (and - (at ?v ?from) ; The vehicle is at the starting location - (or (connected ?from ?to) (connected ?to ?from)) ; A road exists between the locations - ) - ``` - - ### Action Effects - ``` - (and - (not (at ?v ?from)) ; ?v is no longer at ?from - (at ?v ?to) ; ?v is now instead at ?to - ) - ``` - - ### New Predicates - ``` - - (at ?o - object ?l - location): true if the object ?o (a vehicle or a worker) is at the location ?l - - (connected ?l1 - location ?l2 - location): true if a road exists between ?l1 and ?l2 allowing vehicle travel between them. - ``` - """ - mock_model = MockLLM([llm_response]) - - domain_desc = "Vehicle domain" - prompt_template = "Extract PDDL action for: {domain_desc}. Action: {action_name} with description {action_desc}." - action_name = "drive" - action_desc = "Move a vehicle between locations." - action_list = {"park": "Stop a vehicle"} - types = {"car": "vehicle"} - predicates = [ - { - "name": "at", - "desc": "true if the object ?o (a vehicle or a worker) is at the location ?l", - "parameters": ["?o - object", "?l - location"], - }, - { - "name": "connected", - "desc": "true if a road exists between ?l1 and ?l2 allowing vehicle travel between them.", - "parameters": ["?l1 - location", "?l2 - location"], - }, - ] - - expected_predicates = predicates - expected_response = llm_response - - action, new_preds, response = self.domain_builder.extract_pddl_action( - model=mock_model, - domain_desc=domain_desc, - prompt_template=prompt_template, - action_name=action_name, - action_desc=action_desc, - action_list=action_list, - predicates=predicates, - types=types, + type_hierarchy, _ = self.domain_builder.extract_type_hierarchy( + model=mock_llm_1, + domain_desc="HouseConstruction is...", + prompt_template="Prompt template placeholder" ) + self.assertEqual(type_hierarchy, expected_hierarchy) + + with self.assertRaises(RuntimeError) as context: + type_hierarchy, _ = self.domain_builder.extract_type_hierarchy( + model=mock_llm_2, + domain_desc="HouseConstruction is...", + prompt_template="Prompt template placeholder" + ) + self.assertIn("Max retries exceeded", str(context.exception)) - # TODO: finish this comparing actual Action type to this response. + # TODO: implement rest of test domain builder functions + def test_extract_nl_actions(self): + pass - print("\n", action) - print("\n", new_preds) + def test_extract_pddl_action(self): + pass def test_extract_pddl_actions(self): pass diff --git a/tests/test_prompts/test_domain_builder/test_extract_effects/01.txt b/tests/test_prompts/test_domain_builder/test_extract_effects/01.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_effects/01.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_effects/02.txt b/tests/test_prompts/test_domain_builder/test_extract_effects/02.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_effects/02.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_effects/03.txt b/tests/test_prompts/test_domain_builder/test_extract_effects/03.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_effects/03.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_nl_actions/01.txt b/tests/test_prompts/test_domain_builder/test_extract_nl_actions/01.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_nl_actions/01.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_nl_actions/02.txt b/tests/test_prompts/test_domain_builder/test_extract_nl_actions/02.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_nl_actions/02.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_nl_actions/03.txt b/tests/test_prompts/test_domain_builder/test_extract_nl_actions/03.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_nl_actions/03.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_parameters/01.txt b/tests/test_prompts/test_domain_builder/test_extract_parameters/01.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_parameters/01.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_parameters/02.txt b/tests/test_prompts/test_domain_builder/test_extract_parameters/02.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_parameters/02.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_parameters/03.txt b/tests/test_prompts/test_domain_builder/test_extract_parameters/03.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_parameters/03.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_pddl_action/01.txt b/tests/test_prompts/test_domain_builder/test_extract_pddl_action/01.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_pddl_action/01.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_pddl_action/02.txt b/tests/test_prompts/test_domain_builder/test_extract_pddl_action/02.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_pddl_action/02.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_pddl_action/03.txt b/tests/test_prompts/test_domain_builder/test_extract_pddl_action/03.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_pddl_action/03.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_pddl_actions/01.txt b/tests/test_prompts/test_domain_builder/test_extract_pddl_actions/01.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_pddl_actions/01.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_pddl_actions/02.txt b/tests/test_prompts/test_domain_builder/test_extract_pddl_actions/02.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_pddl_actions/02.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_pddl_actions/03.txt b/tests/test_prompts/test_domain_builder/test_extract_pddl_actions/03.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_pddl_actions/03.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_preconditions/01.txt b/tests/test_prompts/test_domain_builder/test_extract_preconditions/01.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_preconditions/01.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_preconditions/02.txt b/tests/test_prompts/test_domain_builder/test_extract_preconditions/02.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_preconditions/02.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_preconditions/03.txt b/tests/test_prompts/test_domain_builder/test_extract_preconditions/03.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_preconditions/03.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_predicates/01.txt b/tests/test_prompts/test_domain_builder/test_extract_predicates/01.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_predicates/01.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_predicates/02.txt b/tests/test_prompts/test_domain_builder/test_extract_predicates/02.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_predicates/02.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_predicates/03.txt b/tests/test_prompts/test_domain_builder/test_extract_predicates/03.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_predicates/03.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_type/01.txt b/tests/test_prompts/test_domain_builder/test_extract_type/01.txt new file mode 100644 index 0000000..183a569 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_type/01.txt @@ -0,0 +1,35 @@ +LLM Chain of Thought Reasoning (example) +1. Understand the Domain Entities: + In the Blocksworld domain, the primary objects involved are: + - A mechanical arm that manipulates the blocks. + - Blocks that can be stacked on other blocks or placed on a table. + - A table that serves as the base where blocks can be placed. + +2. Determine Relationships Between Entities: + - The mechanical arm is used to interact with blocks. + - Blocks can exist in two states: stacked on another block or placed on the table. + - The table acts as the root surface for blocks but does not stack on anything else. + +3. Establish Hierarchy in JSON Representation: + - Start with a root entity, which encapsulates all the components (in this case, "Object"). + - Add child entities (mechanical arm, block, table) under the root. + - For each child, add a short description and any children if hierarchical relationships exist. + +4. Create JSON Output: + - Translate the entities and relationships into a structured JSON format. + - Each component is represented as an object, with "children" to handle potential nested relationships. + +5. Verification: + - Ensure the JSON structure adheres to the intended hierarchy and is properly formatted for parsing. + +## OUTPUT +``` +{ + "object": "Object is always root, everything is an object", + "children": [ + {"arm": "mechanical arm that picks up and stacks blocks on other blocks or table.", "children": []}, + {"block": "colored block that can be stacked or stacked on other blocks or table.", "children": []}, + {"table": "surface where the blocks can be placed on top of.", "children": []} + ] +} +``` \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_type/02.txt b/tests/test_prompts/test_domain_builder/test_extract_type/02.txt new file mode 100644 index 0000000..c01a4c5 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_type/02.txt @@ -0,0 +1 @@ +This is not a valid dictionary format. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_type/03.txt b/tests/test_prompts/test_domain_builder/test_extract_type/03.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_type/03.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_type_hierarchy/01.txt b/tests/test_prompts/test_domain_builder/test_extract_type_hierarchy/01.txt new file mode 100644 index 0000000..3cb1ab9 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_type_hierarchy/01.txt @@ -0,0 +1,120 @@ +The following is an in-context example following the CoT technique. + +Given domain: +The AI agent is here a scheduling system for a house construction company with administrators who can create orders, workers who perform them and vehicles to transport things. + +Given types: +- administrator: Workers who can create orders for other workers +- wall: The exterior of a house. Requires a floor to be built first +- order: Instructions to general_workers to perform something. Created by administrators +- general_worker: Workers who can perform a variety of tasks, if there is an order for it +- vehicle: Used to transport equipment +- house: What is constructed. Can be finalised when all components are built +- house_component: The parts which make up houses +- floor: The bottom of a house. Can be built wherever +- depot: Where material is stored and vehicles can be loaded +- roof: The top of a house. Requires the walls to be built first +- location: Places where workers and machines can be +- mansion: A large type of house +- library: A building filled with books + +----------------------------------------------------- + +1. Reasoning +The "administrator" and "general_worker" are both types of human workers, so we should probably create a "worker" meta-type class for those. +Furthermore, there are several different types of "house_component": "floor", "roof", and "wall". However, these aren't a subtype of "house" since a "floor" is not a "house" for example. +On the other hand, there are actually two subtypes of "house": "mansion" and "library". Both of these are types of houses. +Additionally, "house" and "depot" are both types of locations which can be visited. So, they're subtypes of "location". +"vehicle" is not a subtype or meta-type to any of the others. Same thing holds for "order". + +2. Hierarchy +So, all in all the hierarchy becomes: + +- object: Object is always root, everything is an object + - worker: A type of object consisting of humans who do things. + - administrator: A type of worker. + - general_worker: A type of worker. + - order: A type of object consisting of instructions. + - vehicle: A type of object consisting of vehicles. + - house_component: A type of object consisting of the components of a house. + - wall: A type of house_component. + - floor: A type of house_component. + - roof: A type of house_component. + - location: A type of object consisting of places which can be visited. + - house: A type of location. + - mansion: A type of house. + - library: A type of house. + - depot: A type of location. + +3. Feedback + 1: Is any child NOT a subtype of its parent? + No. This is correct. + + 2: Is any subtype NOT a child of its meta-type? + Yes. Depot is a type of location, and as such it should be a subtype and child of location. This has to be corrected. + + 3: Are any new types needed for organisation? + Yes. There should be a meta-type called "construction_equipment" which includes both "house_component" and "vehicle" since these are used to build with. + + My advice is that you perform the following concrete changes: + - Move the "depot" type to be a subtype of "location". + - Add a "construction_equipment" meta-type with "house_component" and "vehicle" as subtypes. + +4. Response: +The feedback is partially correct. Moving the "depot" type is valid and should be done. However, adding "construction_equipment" would group objects which behave too differently. + +5. Re-iterate and convert to nested Python dictionary: +``` +- object: Object is always root, everything is an object + - worker: A type of object consisting of humans who do things. + - administrator: A type of worker. + - general_worker: A type of worker. + - order: A type of object consisting of instructions. + - vehicle: A type of object consisting of vehicles. + - house_component: A type of object consisting of the components of a house. + - wall: A type of house_component. + - floor: A type of house_component. + - roof: A type of house_component. + - location: A type of object consisting of places which can be visited. + - house: A type of location. + - mansion: A type of house. + - library: A type of house. + - depot: A type of location. +``` + +## OUTPUT +{ + "object": "Object is always root, everything is an object", + "children": [ + { + "worker": "A type of object consisting of humans who do things.", + "children": [ + {"administrator": "A type of worker.", "children": []}, + {"general_worker": "A type of worker.", "children": []} + ] + }, + {"order": "A type of object consisting of instructions.", "children": []}, + {"vehicle": "A type of object consisting of vehicles.", "children": []}, + { + "house_component": "A type of object consisting of the components of a house.", + "children": [ + {"wall": "A type of house_component.", "children": []}, + {"floor": "A type of house_component.", "children": []}, + {"roof": "A type of house_component.", "children": []} + ] + }, + { + "location": "A type of object consisting of places which can be visited.", + "children": [ + { + "house": "A type of location. ", + "children": [ + {"mansion": "A type of house.", "children": []}, + {"library": "A type of house.", "children": []} + ] + }, + {"depot": "A type of location.", "children": []} + ] + } + ] +} diff --git a/tests/test_prompts/test_domain_builder/test_extract_type_hierarchy/02.txt b/tests/test_prompts/test_domain_builder/test_extract_type_hierarchy/02.txt new file mode 100644 index 0000000..c01a4c5 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_type_hierarchy/02.txt @@ -0,0 +1 @@ +This is not a valid dictionary format. \ No newline at end of file diff --git a/tests/test_prompts/test_domain_builder/test_extract_type_hierarchy/03.txt b/tests/test_prompts/test_domain_builder/test_extract_type_hierarchy/03.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_domain_builder/test_extract_type_hierarchy/03.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_task_builder/test_extract_goal_state/01.txt b/tests/test_prompts/test_task_builder/test_extract_goal_state/01.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_task_builder/test_extract_goal_state/01.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_task_builder/test_extract_goal_state/02.txt b/tests/test_prompts/test_task_builder/test_extract_goal_state/02.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_task_builder/test_extract_goal_state/02.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_task_builder/test_extract_goal_state/03.txt b/tests/test_prompts/test_task_builder/test_extract_goal_state/03.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_task_builder/test_extract_goal_state/03.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_task_builder/test_extract_initial_state/01.txt b/tests/test_prompts/test_task_builder/test_extract_initial_state/01.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_task_builder/test_extract_initial_state/01.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_task_builder/test_extract_initial_state/02.txt b/tests/test_prompts/test_task_builder/test_extract_initial_state/02.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_task_builder/test_extract_initial_state/02.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_task_builder/test_extract_initial_state/03.txt b/tests/test_prompts/test_task_builder/test_extract_initial_state/03.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_task_builder/test_extract_initial_state/03.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_task_builder/test_extract_objects/01.txt b/tests/test_prompts/test_task_builder/test_extract_objects/01.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_task_builder/test_extract_objects/01.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_task_builder/test_extract_objects/02.txt b/tests/test_prompts/test_task_builder/test_extract_objects/02.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_task_builder/test_extract_objects/02.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_task_builder/test_extract_objects/03.txt b/tests/test_prompts/test_task_builder/test_extract_objects/03.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_task_builder/test_extract_objects/03.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_task_builder/test_extract_task/01.txt b/tests/test_prompts/test_task_builder/test_extract_task/01.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_task_builder/test_extract_task/01.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_task_builder/test_extract_task/02.txt b/tests/test_prompts/test_task_builder/test_extract_task/02.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_task_builder/test_extract_task/02.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_prompts/test_task_builder/test_extract_task/03.txt b/tests/test_prompts/test_task_builder/test_extract_task/03.txt new file mode 100644 index 0000000..93e1c88 --- /dev/null +++ b/tests/test_prompts/test_task_builder/test_extract_task/03.txt @@ -0,0 +1,2 @@ +# Placeholder: This file is currently empty and serves as a test output for the LLM. +# Content will be implemented in future updates. \ No newline at end of file diff --git a/tests/test_task_builder.py b/tests/test_task_builder.py index 4615c0c..24ed1ec 100644 --- a/tests/test_task_builder.py +++ b/tests/test_task_builder.py @@ -1,21 +1,21 @@ import unittest from l2p.task_builder import TaskBuilder -from l2p.utils.pddl_parser import * +from l2p.utils import * from .mock_llm import MockLLM class TestTaskBuilder(unittest.TestCase): def setUp(self): - self.domain_builder = TaskBuilder() - self.domain_desc = "Blocksworld is..." + self.task_builder = TaskBuilder() + # TODO: implement test task builder functions def test_extract_objects(self): pass def test_extract_initial_state(self): pass - def test_extract_object_state(self): + def test_extract_goal_state(self): pass def test_extract_task(self): From 40e6a0660a2dfe39f67aa979131b710bfd1b2a8a Mon Sep 17 00:00:00 2001 From: Marcus Tantakoun Date: Sat, 21 Dec 2024 15:57:29 -0500 Subject: [PATCH 05/10] updated docs for new pip install and types page --- README.md | 2 +- docs/conf.py | 1 + docs/getting_started.rst | 8 +- docs/index.rst | 10 +- docs/paper_recreations.rst | 2 +- docs/templates.rst | 129 +++++++++++-------- docs/types.rst | 233 +++++++++++++++++++++++++++++++++++ l2p/feedback_builder.py | 2 +- l2p/task_builder.py | 11 -- l2p/utils/pddl_types.py | 2 +- tests/usage/demonstration.py | 11 +- 11 files changed, 334 insertions(+), 77 deletions(-) create mode 100644 docs/types.rst diff --git a/README.md b/README.md index d98fb6a..92ac085 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ print("FEEDBACK:\n", feedback_response) ## Installation and Setup -Currently, this repo has been tested for Python 3.11.10 +Currently, this repo has been tested for Python 3.11.10 but should be fine to install newer versions. You can set up a Python environment using either [Conda](https://conda.io) or [venv](https://docs.python.org/3/library/venv.html) and install the dependencies via the following steps. diff --git a/docs/conf.py b/docs/conf.py index 03201c5..40832f8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,6 +31,7 @@ "sphinx.ext.githubpages", "sphinx.ext.napoleon", "myst_parser", + "sphinx_copybutton", ] myst_enable_extensions = [ diff --git a/docs/getting_started.rst b/docs/getting_started.rst index fce896d..83ba1bc 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -5,12 +5,14 @@ Installing ---------- L2P can be installed with pip:: - pip install l2p + pip install lang2pddl + +**NOT** `pip install l2p` Using L2P ------------- -First things first, import the L2P library:: +First things first, import the whole L2P library, or necessary modules (see :doc:`l2p`):: from l2p import * @@ -73,7 +75,7 @@ Build LLM feedback components using the ``FeedbackBuilder`` class. This is an ex feedback_type = "llm" ) -Below are actual usage examples. This is the general setup to build domain predicates: +Below are actual runnable usage examples. This is the general setup to build domain predicates: .. code-block:: python :linenos: diff --git a/docs/index.rst b/docs/index.rst index 327eada..6d14b4a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,19 +3,21 @@ Welcome to Language-to-Plan (L2P)! .. toctree:: :maxdepth: 2 + :titlesonly: :caption: Contents: getting_started l2p paper_recreations templates + types What is L2P? ------------ -This library is a collection of tools for PDDL model generation extracted from natural language driven by large language models. This library is an expansion from the survey paper **Leveraging Large Language Models for Automated Planning and Model Construction: A Survey** which can be found `Here `_ (currently under review process). +This library is a collection of tools for PDDL model generation extracted from natural language driven by large language models. This library is an expansion from the survey paper **Leveraging Large Language Models for Automated Planning and Model Construction: A Survey** (currently under review process). L2P is an offline, NL to PDDL system that supports domain-agnostic planning. It does this via creating an intermediate PDDL representation of the domain and task, which can then be solved by a classical planner. -Our GitHub can be found `here `_. +Our GitHub can be found `here `_. Features -------- @@ -28,7 +30,9 @@ Installation Install ``l2p`` by running:: - pip install l2p + pip install lang2pddl + +**NOT** `pip install l2p` Usage ----- diff --git a/docs/paper_recreations.rst b/docs/paper_recreations.rst index 91ddb1a..461255e 100644 --- a/docs/paper_recreations.rst +++ b/docs/paper_recreations.rst @@ -46,7 +46,7 @@ Below is L2P code reconstruction of "action-by-action algorithm" from `"Leveragi Current Model Construction Works -------------------------------- -This section provides a taxonomy of research within Model Construction. For more detailed overview, visit our `paper `_. +This section provides a taxonomy of research within Model Construction from the survey paper **Leveraging Large Language Models for Automated Planning and Model Construction: A Survey** (currently under review process). Task Translation Frameworks ------------------------------- diff --git a/docs/templates.rst b/docs/templates.rst index db5809b..0ca360a 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -2,98 +2,123 @@ Templates ================ It is **highly** recommended to use the base templates to properly extract LLM output into the designated Python formats from these methods. -Below are some examples of the base prompt structure that should be used in this library with your customised prompt. More details of each methods' prompt structure is found in **l2p/data/prompt_templates** +Below are some examples of the base prompt structure that should be used in this library with your customized prompt using the `PromptBuilder` class. More details of each methods' prompt structure is found in **l2p/templates**. Domain Extraction Prompts Example ------------------------------------------------------- -This is an example found in `l2p/data/prompt_templates/action_construction/extract_action` +This is an example using `l2p/templates/domain_templates/extract_pddl_action.txt` -**Role**: :: +.. code-block:: python + :linenos: - End your final answers underneath the headers: '### Action Parameters,' '### Action Preconditions,' '### Action Effects,' and '### New Predicates' with ''' ''' comment blocks in PDDL as so: + from l2p.prompt_builder import PromptBuilder + from l2p.utils import load_file - ### Action Parameters - ``` - - ?v - vehicle: The vehicle travelling - ``` + # LOAD BASE FORMAT TEMPLATE + template_path = "templates/domain_templates/extract_pddl_action.txt" + base_template = load_file(template_path) - ### Action Preconditions - ``` - (and - (at ?v ?from) ; The vehicle is at the starting location + role_desc = ( + "You are a PDDL action constructor. Your job is to take the task given in natural language and convert it into the following format:\n\n" + f"{base_template}" # INSERT BASE TEMPLATE ) - ``` - - ### Action Effects - ``` - (and - (not (at ?v ?from)) ; ?v is no longer at ?from - ) - ``` - - ### New Predicates - ``` - - (at ?o - object ?l - location): true if the object ?o (a vehicle or a worker) is at the location ?l - ``` - -**Task**: :: - - Here is the task: - + + tech_desc = """ + You should follow a Chain of Thought (CoT) process for constructing a PDDL action before outputting the final answer: + + 1. Construct action parameters and create necessary predicates to produce action preconditions in PDDL + 2. Construct necessary predicates to produce action effects in PDDL + 3. Check for inconsistencies and/or requirements and state the errors if there are any. If there are errors, generate a suggestion response (i.e. deleting, modifying, adding types) + 4. Re-iterate parameters, preconditions, effects, and predicates + """ + + task_desc = f""" ## Domain {domain_desc} - ## Types + ## Available types {types} - ## Future actions to be used later: + ## Future actions to be implemented later {action_list} - ## Action name + ## Action name (to implement now) {action_name} - ## Action description + ## Action description (corresponding action description) {action_desc} - # Available predicates + ## Available predicates {predicates} + """ + + action_construction_prompt = PromptBuilder(role=role_desc, technique=tech_desc, task=task_desc) + print(action_construction_prompt.generate_prompt()) -Task Extraction Prompts Example ---------------------------------------------------- -This is an example found in `l2p/data/prompt_templates/task_extraction/extract_task` +The following is the output: :: + + [ROLE]: You are a PDDL action constructor. Your job is to take the task given in natural language and convert it into the following format: -**Role**: :: + End your final answers underneath the headers: '### Action Parameters,' '### Action Preconditions,' '### Action Effects,' and '### New Predicates' with ''' ''' comment blocks in PDDL as so: - Do not, under any circumstance, output the answers in PDDL format. Final answer must be in the following format at the end: - ## OBJECTS + ### Action Parameters ``` - truck1 - truck + - ?t - type: 'parameter_description' ``` - ## INITIAL + ### Action Preconditions ``` - (at truck1 chicago_depot): truck1 is at the chicago_depot + (and + (predicate_name ?t1 ?t2) ; COMMENT DESCRIPTION + ) ``` - ## GOAL + ### Action Effects ``` - (AND ; all the following should be done - (finalised house1) ; house 1 is done + (and + (predicate_name ?t1 ?t2) ; COMMENT DESCRIPTION ) ``` -**Task**: :: + ### New Predicates + ``` + - (predicate_name ?t1 - type_1 ?t2 - type_2): 'predicate_description' + ``` + + If there are no new predicates created, keep an empty space enclosed ``` ``` with the '### New Predicates' header. + + ------------------------------------------------ + [TECHNIQUE]: + You should follow a Chain of Thought (CoT) process for constructing a PDDL action before outputting the final answer: + + 1. Construct action parameters and create necessary predicates to produce action preconditions in PDDL + 2. Construct necessary predicates to produce action effects in PDDL + 3. Check for inconsistencies and/or requirements and state the errors if there are any. If there are errors, generate a suggestion response (i.e. deleting, modifying, adding types) + 4. Re-iterate parameters, preconditions, effects, and predicates + + ------------------------------------------------ + [TASK]: + Here is the task to solve: ## Domain {domain_desc} - ## Types + ## Available types {types} - ## Predicates + ## Future actions to be implemented later + {action_list} + + ## Action name (to implement now) + {action_name} + + ## Action description (corresponding action description) + {action_desc} + + ## Available predicates {predicates} - ## Problem description - {problem_desc} \ No newline at end of file + +Users have the flexibility to customize all aspects of their prompts, with the exception of the provided base template. While users can include few-shot examples to guide the LLM, the base template must remain intact during inference. \ No newline at end of file diff --git a/docs/types.rst b/docs/types.rst new file mode 100644 index 0000000..d9e5da3 --- /dev/null +++ b/docs/types.rst @@ -0,0 +1,233 @@ +Data Types +================ +L2P serves as a intermediary between Large Language Models (LLMs) and the construction of reliable planning components in PDDL. To ensure consistency and reliability in planning workflows, L2P translates LLM-generated outputs into well-defined Python types. These types form the foundation for creating, manipulating, and validating planning components, enabling easy integration of LLM capabilities in PDDL. You can find more information on PDDL `here `_. + +Currently, L2P is currently at work to integrate more advanced PDDL aspects. Below you will find the types and respective Python format: + +Types +------------------------------------------------------- +**Types** are formatted as a Python dictionary: `dict[str,str]` where the first string represents the PDDL type name, and the second string provides a natural language description of the type. + +**Example One (Basic types)**: :: + + { + "type_1": "description", + "type_2": "description", + "type_3": "description", + } + +**Example Two (Nested types)**: :: + + { + "parent_type_1": "description", + "children": [ + { + "child_type_1": "description", + "children": [ + {"child_child_type_1": "description", "children": []}, + {"child_child_type_2": "description", "children": []} + ] + } + ] + } + + +Predicates +------------------------------------------------------- +A **Predicate** is a class defined as a `TypeDict` in Python. Each key specifies a property of the predicate: + +- **name** (*str*): The name of the predicate. +- **desc** (*Optional[str]*): An optional natural language description of the predicate. +- **raw** (*str*): The raw representation of the predicate as defined in PDDL. +- **params** (*ParameterList*): A list of parameters associated with the predicate. +- **clean** (*str*): A cleaned or standardized version of the predicate for easier processing or display. + +Where **ParameterList** is: :: + + ParameterList = NewType("ParameterList", OrderedDict[str, str]) # {'?param_name': 'param_type'} OR OrderedDict([('?param_name', 'param_type')]) + +For example, **extract_predicates()** takes the LLM output: :: + + ### New Predicates + ``` + - (on_top ?b1 - block ?b2 - block): true if the block ?b1 is on top of the block ?b + ``` + +And converts it into this: :: + + predicate: Predicate = {'name': 'on_top', + 'desc': 'true if the block ?b1 is on top of the block ?b2', + 'raw': '(on_top ?b1 - block ?b2 - block): true if the block ?b1 is on top of the block ?b', + 'params': OrderedDict([('?b1', 'block'), ('?b2', 'block')]), + 'clean': '(on_top ?b1 - block ?b2 - block): true if the block ?b1 is on top of the block ?b2'} + +Action +------------------------------------------------------- +An **Action** is a class defined as a `TypeDict` in Python. Each key specifies a property of the action schema: + +- **name** (*str*): The name of the action +- **raw** (*str*): The raw representation of the action as defined in PDDL. +- **params** (*ParameterList*): A list of parameters associated with the action. +- **preconditions** (*str*): +- **effects** (*str*): + +For example, **extract_pddl_action()** takes the LLM output: :: + + ### Action Parameters + ``` + - ?b1 - block: The block being stacked on top + - ?b2 - block: The block being stacked upon + - ?a - arm: The arm performing the stacking action + ``` + + ### Action Preconditions + ``` + (and + (holding ?a ?b1) ; The arm is holding the top block + (clear ?b2) ; The bottom block is clear + ) + ``` + + ### Action Effects + ``` + (and + (not (holding ?a ?b1)) ; The arm is no longer holding the top block + (on ?b1 ?b2) ; The top block is now on the bottom block + (not (clear ?b2)) ; The bottom block is no longer clear + ) + ``` + + ### New Predicates + ``` + - (holding ?a - arm ?b - block): true if the arm ?a is holding the block ?b + - (clear ?b - block): true if the block ?b is clear and can be stacked upon + - (on ?b1 - block ?b2 - block): true if the block ?b1 is on top of the block ?b2 + ``` + +And converts it into this: :: + + action: Action = {'name': 'stack', + 'params': OrderedDict([('?b1', 'block'), ('?b2', 'block'), ('?a', 'arm')]), + 'preconditions': '(and\n (holding ?a ?b1) ; The arm is holding the top block\n (clear ?b2) ; The bottom block is clear\n)', + 'effects': '(and\n (not (holding ?a ?b1)) ; The arm is no longer holding the top block\n (on ?b1 ?b2) ; The top block is now on the bottom block\n (not (clear ?b2)) ; The bottom block is no longer clear\n)'} + +Action Parameters +------------------------------------------------------- +**Action Parameters** are formatted as `OrderedDict` type. + +For example, **extract_parameters()** takes the LLM output: :: + + ### Action Parameters + ``` + - ?top - block: The block being stacked on top + - ?bottom - block: The block being stacked upon + - ?a - arm: The arm performing the stacking action + ``` + +And converts it into this: :: + + params: ParameterList = OrderedDict([('?a', 'arm'), ('?top', 'block'), ('?bottom', 'block')]) + +Action Preconditions +------------------------------------------------------- +**Action Preconditions** are formatted as Python string type. + +For example, **extract_preconditions()** takes the LLM output: :: + + ### Action Preconditions + ``` + (and + (holding ?arm ?top) ; The arm is holding the top block + (clear ?bottom) ; The bottom block is clear + ) + ``` + +And converts it into this: :: + + preconditions: str = '(and\n (holding ?arm ?top) ; The arm is holding the top block\n (clear ?bottom) ; The bottom block is clear\n)' + +Action Effects +------------------------------------------------------- +**Action Effects** are formatted as Python string type. + +For example, **extract_effects()** takes the LLM output: :: + + ### Action Effects + ``` + (and + (not (holding ?arm ?top)) ; The arm is no longer holding the top block + (on ?top ?bottom) ; The top block is now on the bottom block + (not (clear ?bottom)) ; The bottom block is no longer clear + ) + ``` + +And converts it into this: :: + + effects: str = '(and\n (not (holding ?arm ?top)) ; The arm is no longer holding the top block\n (on ?top ?bottom) ; The top block is now on the bottom block\n (not (clear ?bottom)) ; The bottom block is no longer clear\n)'} + + + +Task Objects +------------------------------------------------------- +**Objects** are formatted as Python `dict[str,str] # {'name': 'description'}` + +For example, **extract_objects()** takes the LLM output: :: + + ## OBJECTS + ``` + blue_block - object + red_block - object + yellow_block - object + green_block - object + ``` + +And converts it into this: :: + + objects: dict[str,str] = {'blue_block': 'object', 'red_block': 'object', 'yellow_block': 'object', 'green_block': 'object'} + +Task Initial States +------------------------------------------------------- +**Initial States** are formatted as Python `list[dict[str,str]] # essentially [{predicate,params,neg}]` + +For example, **extract_initial_state()** takes the LLM output: :: + + ## INITIAL + ``` + (on_top blue_block red_block): blue_block is on top of red_block + (on_top red_block yellow_block): red_block is on top of yellow_block + (on_table yellow_block): yellow_block is on the table + (on_table green_block): green_block is on the table + (clear yellow_block): yellow_block is clear + (clear green_block): green_block is clear + (not clear red_block): red_block is not clear + ``` + +And converts it into this: :: + + initial: list[dict[str,str]] = [{'name': 'on_top', 'params': ['blue_block', 'red_block'], 'neg': False}, + {'name': 'on_top', 'params': ['red_block', 'yellow_block'], 'neg': False}, + {'name': 'on_table', 'params': ['yellow_block'], 'neg': False}, + {'name': 'on_table', 'params': ['green_block'], 'neg': False}, + {'name': 'clear', 'params': ['yellow_block'], 'neg': False}, + {'name': 'clear', 'params': ['green_block'], 'neg': False}, + {'name': 'clear', 'params': ['red_block'], 'neg': True}, + {'name': 'AND', 'params': [], 'neg': False}, + {'name': 'on_top', 'params': ['red_block', 'green_block'], 'neg': False}] + + +Task Goal States +------------------------------------------------------- +**Goal States** are formatted as Python `list[dict[str,str]] # essentially [{predicate,params,neg}]` + +For example, **extract_goal_state()** takes the LLM output: :: + + ## GOAL + ``` + (AND ; all the following should be done + (on_top red_block green_block) ; red block is on top of green block + ) + ``` + +And converts it into this: :: + + goal: list[dict[str,str]] = [{'name': 'on_top', 'params': ['red_block', 'green_block']}] \ No newline at end of file diff --git a/l2p/feedback_builder.py b/l2p/feedback_builder.py index e5b2675..7785c2a 100644 --- a/l2p/feedback_builder.py +++ b/l2p/feedback_builder.py @@ -187,7 +187,7 @@ def pddl_action_feedback( ) param_str = ( ", ".join( - [f"{name} - {type}" for name, type in action["parameters"].items()] + [f"{name} - {type}" for name, type in action["params"].items()] ) if action else "No parameters provided" diff --git a/l2p/task_builder.py b/l2p/task_builder.py index f6b6546..348b6ef 100644 --- a/l2p/task_builder.py +++ b/l2p/task_builder.py @@ -287,21 +287,10 @@ def extract_task( llm_response = model.query(prompt=prompt_template) - print(llm_response) - print("END OF LLM RESPONSE") - # extract respective types from response objects = parse_objects(llm_response) - print(objects) - print("----------") - initial = parse_initial(llm_response) - print(initial) - print("----------") - goal = parse_goal(llm_response) - print(goal) - print("----------") return objects, initial, goal, llm_response diff --git a/l2p/utils/pddl_types.py b/l2p/utils/pddl_types.py index 2f04322..5b687ad 100644 --- a/l2p/utils/pddl_types.py +++ b/l2p/utils/pddl_types.py @@ -23,7 +23,7 @@ class Predicate(TypedDict): class Action(TypedDict): name: str raw: str - parameters: ParameterList + params: ParameterList preconditions: str effects: str diff --git a/tests/usage/demonstration.py b/tests/usage/demonstration.py index 64abe4f..fb86cac 100644 --- a/tests/usage/demonstration.py +++ b/tests/usage/demonstration.py @@ -122,6 +122,11 @@ def run_task(): types=types, predicates=predicates, ) + + print("OBJECTS:", objects) + print("INITIAL:", initial_states) + print("GOAL:", goal_states) + print("LLM OUTPUT:", llm_response) # format key info into PDDL strings objects_str = task_builder.format_objects(objects) @@ -137,9 +142,7 @@ def run_task(): goal=goal_str, ) - print(f"### LLM OUTPUT:\n {pddl_problem}") - - print(llm_response) + print(f"### PDDL PROBLEM:\n {pddl_problem}") def run_feedback(): @@ -174,4 +177,4 @@ def run_feedback(): # run_aba() # run_predicates() # run_task() - run_feedback() + run_task() From 5d15afd4242578f5002ba3eb7d42dff9c333f511 Mon Sep 17 00:00:00 2001 From: Marcus Tantakoun Date: Sat, 21 Dec 2024 16:03:29 -0500 Subject: [PATCH 06/10] quick fix pip install information --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 6d14b4a..a87f76b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,7 +17,7 @@ What is L2P? This library is a collection of tools for PDDL model generation extracted from natural language driven by large language models. This library is an expansion from the survey paper **Leveraging Large Language Models for Automated Planning and Model Construction: A Survey** (currently under review process). L2P is an offline, NL to PDDL system that supports domain-agnostic planning. It does this via creating an intermediate PDDL representation of the domain and task, which can then be solved by a classical planner. -Our GitHub can be found `here `_. +Our GitHub can be found `here `_. L2P PyPI can be found `here `_. Features -------- From 2aa1f5cfc2c4ac6a3d2cb0950508ccb50d2390aa Mon Sep 17 00:00:00 2001 From: Marcus Tantakoun Date: Sun, 22 Dec 2024 10:53:12 -0500 Subject: [PATCH 07/10] used black formatting tool to cleanup --- l2p/domain_builder.py | 6 +- l2p/feedback_builder.py | 4 +- paper_reconstructions/nl2plan/nl2plan.py | 64 +++++++++++++++----- tests/test_domain_builder.py | 75 +++++++++++++++--------- tests/usage/demonstration.py | 2 +- 5 files changed, 101 insertions(+), 50 deletions(-) diff --git a/l2p/domain_builder.py b/l2p/domain_builder.py index 8424b65..7c6257d 100644 --- a/l2p/domain_builder.py +++ b/l2p/domain_builder.py @@ -128,7 +128,7 @@ def extract_type_hierarchy( # extract respective types from response type_hierarchy = convert_to_dict(llm_response=llm_response) - + if type_hierarchy is not None: return type_hierarchy, llm_response @@ -191,7 +191,7 @@ def extract_nl_actions( # extract respective types from response nl_actions = convert_to_dict(llm_response=llm_response) - + if nl_actions is not None: return nl_actions, llm_response @@ -268,7 +268,7 @@ def extract_pddl_action( llm_response=llm_response, action_name=action_name ) new_predicates = parse_new_predicates(llm_response) - + if action is not None and new_predicates is not None: return action, new_predicates, llm_response diff --git a/l2p/feedback_builder.py b/l2p/feedback_builder.py index 7785c2a..9bcc48f 100644 --- a/l2p/feedback_builder.py +++ b/l2p/feedback_builder.py @@ -186,9 +186,7 @@ def pddl_action_feedback( format_predicates(predicates) if predicates else "No predicates provided." ) param_str = ( - ", ".join( - [f"{name} - {type}" for name, type in action["params"].items()] - ) + ", ".join([f"{name} - {type}" for name, type in action["params"].items()]) if action else "No parameters provided" ) diff --git a/paper_reconstructions/nl2plan/nl2plan.py b/paper_reconstructions/nl2plan/nl2plan.py index 3fc66f0..1d9fb88 100644 --- a/paper_reconstructions/nl2plan/nl2plan.py +++ b/paper_reconstructions/nl2plan/nl2plan.py @@ -28,33 +28,65 @@ def format_json_output(data): # open and create type extraction prompt builder class role_desc = open_file("paper_reconstructions/nl2plan/prompts/type_extraction/role.txt") -tech_desc = open_file("paper_reconstructions/nl2plan/prompts/type_extraction/technique.txt") +tech_desc = open_file( + "paper_reconstructions/nl2plan/prompts/type_extraction/technique.txt" +) task_desc = open_file("paper_reconstructions/nl2plan/prompts/type_extraction/task.txt") -type_extraction_prompt = PromptBuilder(role=role_desc, technique=tech_desc, task=task_desc) +type_extraction_prompt = PromptBuilder( + role=role_desc, technique=tech_desc, task=task_desc +) # open and create type hierarchy prompt builder class -role_desc = open_file("paper_reconstructions/nl2plan/prompts/hierarchy_construction/role.txt") -tech_desc = open_file("paper_reconstructions/nl2plan/prompts/hierarchy_construction/technique.txt") -task_desc = open_file("paper_reconstructions/nl2plan/prompts/hierarchy_construction/task.txt") -type_hierarchy_prompt = PromptBuilder(role=role_desc, technique=tech_desc, task=task_desc) +role_desc = open_file( + "paper_reconstructions/nl2plan/prompts/hierarchy_construction/role.txt" +) +tech_desc = open_file( + "paper_reconstructions/nl2plan/prompts/hierarchy_construction/technique.txt" +) +task_desc = open_file( + "paper_reconstructions/nl2plan/prompts/hierarchy_construction/task.txt" +) +type_hierarchy_prompt = PromptBuilder( + role=role_desc, technique=tech_desc, task=task_desc +) # open and create NL action prompt builder class -role_desc = open_file("paper_reconstructions/nl2plan/prompts/action_extraction/role.txt") -tech_desc = open_file("paper_reconstructions/nl2plan/prompts/action_extraction/technique.txt") -task_desc = open_file("paper_reconstructions/nl2plan/prompts/action_extraction/task.txt") -action_extraction_prompt = PromptBuilder(role=role_desc, technique=tech_desc, task=task_desc) +role_desc = open_file( + "paper_reconstructions/nl2plan/prompts/action_extraction/role.txt" +) +tech_desc = open_file( + "paper_reconstructions/nl2plan/prompts/action_extraction/technique.txt" +) +task_desc = open_file( + "paper_reconstructions/nl2plan/prompts/action_extraction/task.txt" +) +action_extraction_prompt = PromptBuilder( + role=role_desc, technique=tech_desc, task=task_desc +) # open and create PDDL action prompt builder class -role_desc = open_file("paper_reconstructions/nl2plan/prompts/action_construction/role.txt") -tech_desc = open_file("paper_reconstructions/nl2plan/prompts/action_construction/technique.txt") -task_desc = open_file("paper_reconstructions/nl2plan/prompts/action_construction/task.txt") -action_construction_prompt = PromptBuilder(role=role_desc, technique=tech_desc, task=task_desc) +role_desc = open_file( + "paper_reconstructions/nl2plan/prompts/action_construction/role.txt" +) +tech_desc = open_file( + "paper_reconstructions/nl2plan/prompts/action_construction/technique.txt" +) +task_desc = open_file( + "paper_reconstructions/nl2plan/prompts/action_construction/task.txt" +) +action_construction_prompt = PromptBuilder( + role=role_desc, technique=tech_desc, task=task_desc +) # open and create compact action prompt builder class role_desc = open_file("paper_reconstructions/nl2plan/prompts/task_extraction/role.txt") -tech_desc = open_file("paper_reconstructions/nl2plan/prompts/task_extraction/technique.txt") +tech_desc = open_file( + "paper_reconstructions/nl2plan/prompts/task_extraction/technique.txt" +) task_desc = open_file("paper_reconstructions/nl2plan/prompts/task_extraction/task.txt") -task_extraction_prompt = PromptBuilder(role=role_desc, technique=tech_desc, task=task_desc) +task_extraction_prompt = PromptBuilder( + role=role_desc, technique=tech_desc, task=task_desc +) domain_builder = DomainBuilder() task_builder = TaskBuilder() diff --git a/tests/test_domain_builder.py b/tests/test_domain_builder.py index 19b4743..ceba52f 100644 --- a/tests/test_domain_builder.py +++ b/tests/test_domain_builder.py @@ -16,29 +16,47 @@ def test_extract_type(self): types, _ = self.domain_builder.extract_type( model=mock_llm_1, domain_desc="Blocksworld is a...", - prompt_template="Prompt template placeholder" + prompt_template="Prompt template placeholder", + ) + self.assertEqual( + types, + { + "object": "Object is always root, everything is an object", + "children": [ + { + "arm": "mechanical arm that picks up and stacks blocks on other blocks or table.", + "children": [], + }, + { + "block": "colored block that can be stacked or stacked on other blocks or table.", + "children": [], + }, + { + "table": "surface where the blocks can be placed on top of.", + "children": [], + }, + ], + }, ) - self.assertEqual(types, { - "object": "Object is always root, everything is an object", - "children": [ - {"arm": "mechanical arm that picks up and stacks blocks on other blocks or table.", "children": []}, - {"block": "colored block that can be stacked or stacked on other blocks or table.", "children": []}, - {"table": "surface where the blocks can be placed on top of.", "children": []} - ] - }) with self.assertRaises(RuntimeError) as context: types, _ = self.domain_builder.extract_type( model=mock_llm_2, domain_desc="Blocksworld is a...", - prompt_template="Prompt template placeholder" + prompt_template="Prompt template placeholder", ) self.assertIn("Max retries exceeded", str(context.exception)) def test_extract_type_hierarchy(self): - mock_llm_1 = MockLLM([load_file("tests/test_prompts/test_extract_type_hierarchy/01.txt")]) - mock_llm_2 = MockLLM([load_file("tests/test_prompts/test_extract_type_hierarchy/02.txt")]) - mock_llm_3 = MockLLM([load_file("tests/test_prompts/test_extract_type_hierarchy/03.txt")]) + mock_llm_1 = MockLLM( + [load_file("tests/test_prompts/test_extract_type_hierarchy/01.txt")] + ) + mock_llm_2 = MockLLM( + [load_file("tests/test_prompts/test_extract_type_hierarchy/02.txt")] + ) + mock_llm_3 = MockLLM( + [load_file("tests/test_prompts/test_extract_type_hierarchy/03.txt")] + ) expected_hierarchy = { "object": "Object is always root, everything is an object", @@ -47,18 +65,21 @@ def test_extract_type_hierarchy(self): "worker": "A type of object consisting of humans who do things.", "children": [ {"administrator": "A type of worker.", "children": []}, - {"general_worker": "A type of worker.", "children": []} - ] + {"general_worker": "A type of worker.", "children": []}, + ], + }, + { + "order": "A type of object consisting of instructions.", + "children": [], }, - {"order": "A type of object consisting of instructions.", "children": []}, {"vehicle": "A type of object consisting of vehicles.", "children": []}, { "house_component": "A type of object consisting of the components of a house.", "children": [ {"wall": "A type of house_component.", "children": []}, {"floor": "A type of house_component.", "children": []}, - {"roof": "A type of house_component.", "children": []} - ] + {"roof": "A type of house_component.", "children": []}, + ], }, { "location": "A type of object consisting of places which can be visited.", @@ -67,27 +88,27 @@ def test_extract_type_hierarchy(self): "house": "A type of location. ", "children": [ {"mansion": "A type of house.", "children": []}, - {"library": "A type of house.", "children": []} - ] + {"library": "A type of house.", "children": []}, + ], }, - {"depot": "A type of location.", "children": []} - ] - } - ] + {"depot": "A type of location.", "children": []}, + ], + }, + ], } type_hierarchy, _ = self.domain_builder.extract_type_hierarchy( model=mock_llm_1, domain_desc="HouseConstruction is...", - prompt_template="Prompt template placeholder" + prompt_template="Prompt template placeholder", ) self.assertEqual(type_hierarchy, expected_hierarchy) - + with self.assertRaises(RuntimeError) as context: type_hierarchy, _ = self.domain_builder.extract_type_hierarchy( model=mock_llm_2, domain_desc="HouseConstruction is...", - prompt_template="Prompt template placeholder" + prompt_template="Prompt template placeholder", ) self.assertIn("Max retries exceeded", str(context.exception)) diff --git a/tests/usage/demonstration.py b/tests/usage/demonstration.py index fb86cac..6493a5e 100644 --- a/tests/usage/demonstration.py +++ b/tests/usage/demonstration.py @@ -122,7 +122,7 @@ def run_task(): types=types, predicates=predicates, ) - + print("OBJECTS:", objects) print("INITIAL:", initial_states) print("GOAL:", goal_states) From 05d15ee2d2281c4cb8970c33a6ddd45b72b5e363 Mon Sep 17 00:00:00 2001 From: Marcus Tantakoun Date: Sun, 22 Dec 2024 10:58:26 -0500 Subject: [PATCH 08/10] removed unwanted file for the github --- tests/parse.py | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 tests/parse.py diff --git a/tests/parse.py b/tests/parse.py deleted file mode 100644 index d60d3bb..0000000 --- a/tests/parse.py +++ /dev/null @@ -1,39 +0,0 @@ -from pddl.formatter import domain_to_string, problem_to_string -from pddl import parse_domain, parse_problem -import sys - - -def check_parse_domain(file_path): - try: - domain = parse_domain(file_path) - pddl_domain = domain_to_string(domain) - return pddl_domain - except Exception as e: - print("------------------") - print(f"Error parsing domain: {e}", file=sys.stderr) - print("------------------") - sys.exit(1) - - -def check_parse_problem(file_path): - try: - problem = parse_problem(file_path) - pddl_problem = problem_to_string(problem) - return pddl_problem - except Exception as e: - print("------------------") - print(f"Error parsing domain: {e}", file=sys.stderr) - print("------------------") - sys.exit(1) - - -if __name__ == "__main__": - - domain_file_path = "paper_reconstructions/llm+dm/results/domain.pddl" - problem_file_path = "data/problem_1.pddl" - - pddl_domain = check_parse_domain(domain_file_path) - print("PDDL domain:\n", pddl_domain) - - with open(domain_file_path, "w") as f: - f.write(pddl_domain) From 558e97551d510b19f06117d008415ea12eba7db5 Mon Sep 17 00:00:00 2001 From: Marcus Tantakoun Date: Sun, 22 Dec 2024 11:04:36 -0500 Subject: [PATCH 09/10] fixed file import error in tests/test_domain_builder.py --- tests/test_domain_builder.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_domain_builder.py b/tests/test_domain_builder.py index ceba52f..cc46bd9 100644 --- a/tests/test_domain_builder.py +++ b/tests/test_domain_builder.py @@ -9,9 +9,9 @@ def setUp(self): self.domain_builder = DomainBuilder() def test_extract_type(self): - mock_llm_1 = MockLLM([load_file("tests/test_prompts/test_extract_type/01.txt")]) - mock_llm_2 = MockLLM([load_file("tests/test_prompts/test_extract_type/02.txt")]) - mock_llm_3 = MockLLM([load_file("tests/test_prompts/test_extract_type/03.txt")]) + mock_llm_1 = MockLLM([load_file("tests/test_prompts/test_domain_builder/test_extract_type/01.txt")]) + mock_llm_2 = MockLLM([load_file("tests/test_prompts/test_domain_builder/test_extract_type/02.txt")]) + mock_llm_3 = MockLLM([load_file("tests/test_prompts/test_domain_builder/test_extract_type/03.txt")]) types, _ = self.domain_builder.extract_type( model=mock_llm_1, @@ -49,13 +49,13 @@ def test_extract_type(self): def test_extract_type_hierarchy(self): mock_llm_1 = MockLLM( - [load_file("tests/test_prompts/test_extract_type_hierarchy/01.txt")] + [load_file("tests/test_prompts/test_domain_builder/test_extract_type_hierarchy/01.txt")] ) mock_llm_2 = MockLLM( - [load_file("tests/test_prompts/test_extract_type_hierarchy/02.txt")] + [load_file("tests/test_prompts/test_domain_builder/test_extract_type_hierarchy/02.txt")] ) mock_llm_3 = MockLLM( - [load_file("tests/test_prompts/test_extract_type_hierarchy/03.txt")] + [load_file("tests/test_prompts/test_domain_builder/test_extract_type_hierarchy/03.txt")] ) expected_hierarchy = { From 86ec62210eea5f4b9cd93251a53c9ef674cef939 Mon Sep 17 00:00:00 2001 From: Marcus Tantakoun Date: Mon, 23 Dec 2024 13:59:06 -0500 Subject: [PATCH 10/10] cleaned FastDownward planning module, retested paper reconstructions, fixed multiple bugs concerning .utils --- docs/l2p.rst | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/l2p.rst b/docs/l2p.rst index 8bf112a..cd53800 100644 --- a/docs/l2p.rst +++ b/docs/l2p.rst @@ -2,6 +2,13 @@ L2P ================ Below are the in-depth usage of L2P. It is **highly** recommended to use the base template found in :doc:`templates` to properly extract LLM output into the designated Python formats from these methods. +PromptBuilder +------------- +.. autoclass:: l2p.PromptBuilder + :members: + :undoc-members: + :inherited-members: + DomainBuilder ------------- .. autoclass:: l2p.DomainBuilder @@ -23,9 +30,34 @@ FeedbackBuilder :undoc-members: :inherited-members: -PromptBuilder -------------- -.. autoclass:: l2p.PromptBuilder +Utils +----- +The `utils` package contains several helper modules for working with PDDL and L2P processes. + +PDDL Parser +~~~~~~~~~~~ +.. automodule:: l2p.utils.pddl_parser + :members: + :undoc-members: + :inherited-members: + +PDDL Planner +~~~~~~~~~~~~ +.. automodule:: l2p.utils.pddl_planner + :members: + :undoc-members: + :inherited-members: + +PDDL Types +~~~~~~~~~~ +.. automodule:: l2p.utils.pddl_types + :members: + :undoc-members: + :inherited-members: + +PDDL Validator +~~~~~~~~~~~~~~ +.. automodule:: l2p.utils.pddl_validator :members: :undoc-members: :inherited-members: \ No newline at end of file