feat: add score and thread.track_event

lunary/__init__.py

@@ -118,7 +118,6 @@ def track_event(
     app_id=None,
     api_url=None,
     callback_queue=None,
-    is_openai=False,
 ):
     try:
         config = get_config()
@@ -1853,6 +1852,48 @@ def get_dataset(slug: str, app_id: str | None = None, api_url: str | None = None
 
     except Exception as e:
         raise DatasetError(f"Error fetching dataset: {str(e)}")
+
+def score(run_id: str, label: str, value: int | float | str | bool, comment: str | None, app_id: str | None = None, api_url: str | None = None):
+    """
+    Scores a run based on the provided label, value, and optional comment.
+
+    Parameters:
+        run_id (str): Unique run identifier.
+        label (str): Evaluation label.
+        value (int | float | str | bool): Evaluation value.
+        comment (str, optional): Evaluation comment.
+
+    Raises:
+        ScoringError: If scoring fails.
+    """
+    try:
+        config = get_config()
+        token = app_id or config.app_id
+        api_url = api_url or config.api_url
+
+        url = f"{api_url}/v1/runs/{run_id}/score"
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+        }
+
+        data = {
+            "label": label,
+            "value": value,
+            **({"comment": comment} if comment else {}),
+        }
+
+        response = requests.patch(url, headers=headers, json=data, verify=config.ssl_verify)
+
+        if response.status_code == 500:
+            error_message = response.json().get("message", "Unknown error")
+            raise ScoringError(f"Scoring failed: {error_message}")
+        elif not response.ok:
+            raise ScoringError(f"Error scoring run: {response.status_code} - {response.text}")
+
+    except Exception as e:
+        raise EvaluationError(f"Error scoring run: {str(e)}")
+
 
 def evaluate(
     checklist,
@@ -1923,6 +1964,8 @@ def evaluate(
     except Exception as e:
         raise EvaluationError(f"Error evaluating result: {str(e)}")
 
+
+# TODO: use the endpoint, not track_event
 def track_feedback(run_id: str, feedback: Dict[str, Any] | Any):
     """
     Tracks feedback for a given run ID, validating the provided feedback.

lunary/exceptions.py

@@ -20,4 +20,8 @@ class ThreadError(LunaryError):
 
 class FeedbackError(LunaryError):
     """Raised when there's any error with feedback operations"""
+    pass
+
+class ScoringError(LunaryError):
+    """Raised when there's any error with scoring operations"""
     pass

lunary/thread.py

@@ -1,5 +1,6 @@
 import uuid
 from typing import List, TypedDict
+from .config import get_config
 
 
 class Message(TypedDict, total=False):
@@ -19,20 +20,22 @@ def __init__(
         id: str | None = None,
         tags: List[str] | None = None,
         app_id: str | None = None,
+        api_url: str | None = None,
     ):
         self.id = id or str(uuid.uuid4())
         self.user_id = user_id
         self.user_props = user_props
         self.tags = tags
-        self.track_event = track_event
+        self._track_event = track_event
         self.app_id = app_id
+        self.api_url = api_url
 
     def track_message(
         self, message: Message, user_id=None, user_props=None, feedback=None
     ) -> str:
         run_id = message.get("id", str(uuid.uuid4()))
 
-        self.track_event(
+        self._track_event(
             "thread",
             "chat",
             run_id=run_id,
@@ -45,3 +48,26 @@ def track_message(
             app_id=self.app_id,
         )
         return run_id
+
+    def track_event(self, event_name: str, user_id: str | None = None, user_props: dict[str, any] | None = None, metadata: dict[str, any] | None = None):
+        """
+        Track a custom event in the thread.
+
+        Parameters:
+        - event_name (str): The name of the event.
+        - user_id (str): The user ID associated with the event.
+        - metadata (dict): A dictionary of metadata associated with the event.
+        """
+
+        #TODO: remove event like start and end
+        self._track_event(
+            "thread",
+            "custom-event",
+            name=event_name,
+            run_id=str(uuid.uuid4()),
+            user_id=user_id or self.user_id,
+            user_props=user_props or self.user_props,
+            parent_run_id=self.id,
+            thread_tags=self.tags,
+            app_id=self.app_id
+        )

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lunary"
-version = "1.1.12"
+version = "1.1.13"
 description = "Observability, analytics and evaluations for AI agents and chatbots."
 authors = ["lunary <hello@lunary.ai>"]
 readme = "README.md"