diff --git a/lunary/__init__.py b/lunary/__init__.py index 367c754..912ef00 100644 --- a/lunary/__init__.py +++ b/lunary/__init__.py @@ -118,7 +118,6 @@ def track_event( app_id=None, api_url=None, callback_queue=None, - is_openai=False, ): try: config = get_config() @@ -1853,6 +1852,48 @@ def get_dataset(slug: str, app_id: str | None = None, api_url: str | None = None except Exception as e: raise DatasetError(f"Error fetching dataset: {str(e)}") + +def score(run_id: str, label: str, value: int | float | str | bool, comment: str | None, app_id: str | None = None, api_url: str | None = None): + """ + Scores a run based on the provided label, value, and optional comment. + + Parameters: + run_id (str): Unique run identifier. + label (str): Evaluation label. + value (int | float | str | bool): Evaluation value. + comment (str, optional): Evaluation comment. + + Raises: + ScoringError: If scoring fails. + """ + try: + config = get_config() + token = app_id or config.app_id + api_url = api_url or config.api_url + + url = f"{api_url}/v1/runs/{run_id}/score" + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + + data = { + "label": label, + "value": value, + **({"comment": comment} if comment else {}), + } + + response = requests.patch(url, headers=headers, json=data, verify=config.ssl_verify) + + if response.status_code == 500: + error_message = response.json().get("message", "Unknown error") + raise ScoringError(f"Scoring failed: {error_message}") + elif not response.ok: + raise ScoringError(f"Error scoring run: {response.status_code} - {response.text}") + + except Exception as e: + raise EvaluationError(f"Error scoring run: {str(e)}") + def evaluate( checklist, @@ -1923,6 +1964,8 @@ def evaluate( except Exception as e: raise EvaluationError(f"Error evaluating result: {str(e)}") + +# TODO: use the endpoint, not track_event def track_feedback(run_id: str, feedback: Dict[str, Any] | Any): """ Tracks feedback for a given run ID, validating the provided feedback. diff --git a/lunary/exceptions.py b/lunary/exceptions.py index 905138f..752b698 100644 --- a/lunary/exceptions.py +++ b/lunary/exceptions.py @@ -20,4 +20,8 @@ class ThreadError(LunaryError): class FeedbackError(LunaryError): """Raised when there's any error with feedback operations""" + pass + +class ScoringError(LunaryError): + """Raised when there's any error with scoring operations""" pass \ No newline at end of file diff --git a/lunary/thread.py b/lunary/thread.py index 365b5a9..d1a18db 100644 --- a/lunary/thread.py +++ b/lunary/thread.py @@ -1,5 +1,6 @@ import uuid from typing import List, TypedDict +from .config import get_config class Message(TypedDict, total=False): @@ -19,20 +20,22 @@ def __init__( id: str | None = None, tags: List[str] | None = None, app_id: str | None = None, + api_url: str | None = None, ): self.id = id or str(uuid.uuid4()) self.user_id = user_id self.user_props = user_props self.tags = tags - self.track_event = track_event + self._track_event = track_event self.app_id = app_id + self.api_url = api_url def track_message( self, message: Message, user_id=None, user_props=None, feedback=None ) -> str: run_id = message.get("id", str(uuid.uuid4())) - self.track_event( + self._track_event( "thread", "chat", run_id=run_id, @@ -45,3 +48,26 @@ def track_message( app_id=self.app_id, ) return run_id + + def track_event(self, event_name: str, user_id: str | None = None, user_props: dict[str, any] | None = None, metadata: dict[str, any] | None = None): + """ + Track a custom event in the thread. + + Parameters: + - event_name (str): The name of the event. + - user_id (str): The user ID associated with the event. + - metadata (dict): A dictionary of metadata associated with the event. + """ + + #TODO: remove event like start and end + self._track_event( + "thread", + "custom-event", + name=event_name, + run_id=str(uuid.uuid4()), + user_id=user_id or self.user_id, + user_props=user_props or self.user_props, + parent_run_id=self.id, + thread_tags=self.tags, + app_id=self.app_id + ) diff --git a/pyproject.toml b/pyproject.toml index b94f640..835ab64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "lunary" -version = "1.1.12" +version = "1.1.13" description = "Observability, analytics and evaluations for AI agents and chatbots." authors = ["lunary "] readme = "README.md"