fix: metadata keys query (#724)

lunary-ai · Jan 26, 2025 · 8f63982 · 8f63982
1 parent 99985bb
commit 8f63982
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 15 deletions.
diff --git a/packages/backend/src/api/v1/filters.ts b/packages/backend/src/api/v1/filters.ts
@@ -45,12 +45,11 @@ filters.get("/metadata", async (ctx: Context) => {
 
   const rows = await sql`
     select distinct
-      jsonb_object_keys(metadata) as key
+      key
     from
-      run
+      metadata_cache
     where
-      project_id = ${projectId} 
-      and metadata is not null
+      project_id = ${projectId}
     order by
       key;
   `;

diff --git a/packages/backend/src/create-indexes.ts b/packages/backend/src/create-indexes.ts
@@ -1,5 +1,6 @@
 import sql from "./utils/db";
 
+// TODO: rename to "async migration" or something like this
 export async function createIndexes() {
   await sql`select pg_advisory_lock(123456789)`;
 
@@ -11,7 +12,7 @@ export async function createIndexes() {
         operation,
         statement
       from 
-        _db_migration_index
+        _db_migration_async
        where 
         status in ('pending', 'failed')
        order 
@@ -24,7 +25,7 @@ export async function createIndexes() {
       try {
         await sql`
           update 
-            _db_migration_index
+            _db_migration_async
           set 
             status = 'in-progress'
            where 
@@ -33,6 +34,7 @@ export async function createIndexes() {
 
         await sql.unsafe(statement);
 
+        // TODO: rename to "create_index" and "drop_index"
         if (operation === "create") {
           const [validCheck] = await sql`
           select 
@@ -49,7 +51,7 @@ export async function createIndexes() {
           if (validCheck) {
             await sql`
             update 
-              _db_migration_index
+              _db_migration_async
             set 
               status = 'done'
             where 
@@ -60,7 +62,7 @@ export async function createIndexes() {
             await sql.unsafe(`drop index if exists ${name}`);
             await sql`
             update 
-              _db_migration_index
+              _db_migration_async
             set 
               status = 'failed'
             where 
@@ -83,7 +85,7 @@ export async function createIndexes() {
           if (!stillExists) {
             await sql`
               update 
-                _db_migration_index
+                _db_migration_async
               set 
                 status = 'done'
               where 
@@ -93,21 +95,33 @@ export async function createIndexes() {
           } else {
             await sql`
               update 
-                _db_migration_index
+                _db_migration_async
               set 
                 status = 'failed'
               where 
                 id = ${id}
         `;
             console.warn(`Index drop "${name}" failed; index still exists.`);
           }
+        } else if (operation === "create-materialized-view") {
+          await sql`
+            update 
+              _db_migration_async
+            set 
+              status = 'done'
+            where 
+              id = ${id}
+          `;
+          console.log(
+            `Materialized view migration "${name}" completed successfully.`,
+          );
         }
       } catch (err) {
         console.error(`Index migration "${name}" errored:`, err);
-        await sql.unsafe(`drop index if exists ${name}`);
+        await sql.unsafe(`drop index if exists ${name}`); // TODO: only drop if it was an index, not a materialized views
         await sql`
           update 
-            _db_migration_index
+            _db_migration_async
           set 
             status = 'failed'
           where 

diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts
@@ -17,7 +17,7 @@ import ratelimit from "./utils/ratelimit";
 import { initSentry, requestHandler, tracingMiddleWare } from "./utils/sentry";
 import licenseMiddleware from "./utils/license";
 import config from "./utils/config";
-import { startMaterializedViewRefreshJob } from "./jobs/materializedViews";
+import { startMaterializedViewRefreshJob } from "./jobs/materialized-views";
 import { createIndexes } from "./create-indexes";
 
 checkDbConnection();

diff --git a/...ges/backend/src/jobs/materializedViews.ts → ...es/backend/src/jobs/materialized-views.ts b/...ges/backend/src/jobs/materializedViews.ts → ...es/backend/src/jobs/materialized-views.ts
@@ -2,11 +2,12 @@ import sql from "../utils/db";
 import { sleep } from "../utils/misc";
 
 export async function startMaterializedViewRefreshJob() {
+  // TODO: locks
   if (process.env.DISABLE_MATERIALIZED_VIEW_REFRESH) {
     return;
   }
   try {
-    const views = ["run_parent_feedback_cache"];
+    const views = ["run_parent_feedback_cache", "metadata_cache"];
 
     while (true) {
       for (const view of views) {
@@ -18,7 +19,7 @@ export async function startMaterializedViewRefreshJob() {
         );
       }
 
-      await sleep(1000);
+      await sleep(60000);
     }
   } catch (error) {
     console.error(error);

diff --git a/packages/db/0060.sql b/packages/db/0060.sql
@@ -0,0 +1,22 @@
+alter table _db_migration_index rename to _db_migration_async;
+
+insert into _db_migration_async (name, operation, statement) values
+('metadata_cache', 'create-materialized-view', 
+'create materialized view metadata_cache as  
+select 
+    run.project_id,
+    run.type,
+    jsonb_object_keys(run.metadata) as key,
+    now() as refreshed_at
+from 
+    run
+group by 
+    run.project_id, 
+    run.type,
+    jsonb_object_keys(run.metadata);'
+),
+('metadata_cache_project_id_idx', 'create', 'create index concurrently if not exists metadata_cache_project_id_idx on metadata_cache(project_id)'),
+('metadata_cache_project_id_type_key_idx', 'create', 'create unique index concurrently if not exists metadata_cache_project_id_type_key_idx on metadata_cache (project_id, type, key)');
+
+
+