fix: cached llm cost (#732)

lunary-ai · Feb 4, 2025 · 98eca6d · 98eca6d
1 parent 7d4f61e
commit 98eca6d
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 5 deletions.
diff --git a/packages/backend/src/api/v1/runs/ingest.ts b/packages/backend/src/api/v1/runs/ingest.ts
@@ -299,7 +299,7 @@ async function registerRunEvent(
         promptTokens: tokensUsage?.prompt,
         completionTokens: tokensUsage?.completion,
         name: runData?.name,
-        duration: +timestamp - +runData?.createdAt,
+        duration: new Date(timestamp) - new Date(runData?.createdAt),
         projectId,
       });
     }

diff --git a/packages/backend/src/utils/calcCost.ts b/packages/backend/src/utils/calcCost.ts
@@ -185,7 +185,7 @@ export function calcRunCostLegacy(run: any) {
 }
 
 export async function calcRunCost(run: any) {
-  if (run.duration && run.duration < 0.01 * 1000) return null; // cached llm calls
+  if (!run.duration || run.duration < 0.01 * 1000) return null; // cached llm calls
   if (run.type !== "llm" || !run.name) return null;
 
   // Look at table model_mapping, in this logic:

diff --git a/packages/frontend/components/blocks/DurationBadge.tsx b/packages/frontend/components/blocks/DurationBadge.tsx
@@ -11,7 +11,7 @@ export default function DurationBadge({
     ? new Date(endedAt).getTime() - new Date(createdAt).getTime()
     : NaN;
 
-  if (cached) {
+  if (cached || duration < 0.01 * 1000) {
     return (
       <Badge
         variant="light"

diff --git a/packages/frontend/utils/datatable.tsx b/packages/frontend/utils/datatable.tsx
@@ -42,6 +42,17 @@ export function durationColumn(unit = "s"): ColumnDef<any> {
     enableSorting: true,
     cell: (props) => {
       const value = props?.getValue() || 0;
+      const duration =
+        new Date(props.row.original.endedAt) -
+        new Date(props.row.original.createdAt);
+      const metadata = props.row.original.metadata?.cache;
+
+      console.log(duration);
+      const isCached = metadata?.cached || duration < 0.01 * 1000;
+
+      if (isCached) {
+        return "Cached";
+      }
 
       if (value === 0) {
         return "0.00s";

diff --git a/packages/shared/schemas/run.ts b/packages/shared/schemas/run.ts
@@ -12,9 +12,9 @@ export type Feedback = z.infer<typeof Feedback>;
 export const Score = z.object({
   label: z.string(),
   value: z.union([z.number(), z.string(), z.boolean()]),
-  comment: z.string().nullable(),
+  comment: z.string().nullable().optional(),
 });
-export type Score = z.infer<typeof Scor>;
+export type Score = z.infer<typeof Score>;
 
 export interface Run {
   id: string;