feat: show a link to the Dev UI on running evals

genkit-tools/cli/src/commands/eval-flow.ts

@@ -35,6 +35,7 @@ import {
   loadInferenceDatasetFile,
   logger,
 } from '@genkit-ai/tools-common/utils';
+import * as clc from 'colorette';
 import { Command } from 'commander';
 import { runWithManager } from '../utils/manager-utils';
 
@@ -167,9 +168,18 @@ export const evalFlow = new Command('eval:flow')
           await exportFn(evalRun, options.output);
         }
 
-        console.log(
-          `Succesfully ran evaluation, with evalId: ${evalRun.key.evalRunId}`
-        );
+        const toolsInfo = manager.getMostRecentDevUI();
+        if (toolsInfo) {
+          logger.info(
+            clc.green(
+              `\nView the evaluation results at: ${toolsInfo.url}/evaluate/${evalRun.key.evalRunId}`
+            )
+          );
+        } else {
+          logger.info(
+            `Succesfully ran evaluation, with evalId: ${evalRun.key.evalRunId}`
+          );
+        }
       });
     }
   );

genkit-tools/cli/src/commands/eval-run.ts

@@ -27,6 +27,7 @@ import {
   loadEvaluationDatasetFile,
   logger,
 } from '@genkit-ai/tools-common/utils';
+import * as clc from 'colorette';
 import { Command } from 'commander';
 import { runWithManager } from '../utils/manager-utils';
 
@@ -113,8 +114,17 @@ export const evalRun = new Command('eval:run')
         await exportFn(evalRun, options.output);
       }
 
-      console.log(
-        `Succesfully ran evaluation, with evalId: ${evalRun.key.evalRunId}`
-      );
+      const toolsInfo = manager.getMostRecentDevUI();
+      if (toolsInfo) {
+        logger.info(
+          clc.green(
+            `\nView the evaluation results at: ${toolsInfo.url}/evaluate/${evalRun.key.evalRunId}`
+          )
+        );
+      } else {
+        logger.info(
+          `Succesfully ran evaluation, with evalId: ${evalRun.key.evalRunId}`
+        );
+      }
     });
   });

genkit-tools/cli/src/commands/ui-start.ts

@@ -15,7 +15,9 @@
  */
 
 import {
+  DevToolsInfo,
   findServersDir,
+  isValidDevToolsInfo,
   logger,
   waitUntilHealthy,
 } from '@genkit-ai/tools-common/utils';
@@ -33,24 +35,6 @@ interface StartOptions {
   open?: boolean;
 }
 
-export interface DevToolsInfo {
-  /** URL of the dev tools server. */
-  url: string;
-  /** Timestamp of when the dev tools server was started. */
-  timestamp: string;
-}
-
-/**
- * Checks if the provided data is a valid dev tools server state file.
- */
-export function isValidDevToolsInfo(data: any): data is DevToolsInfo {
-  return (
-    typeof data === 'object' &&
-    typeof data.url === 'string' &&
-    typeof data.timestamp === 'string'
-  );
-}
-
 /** Command to start the Genkit Developer UI. */
 export const uiStart = new Command('ui:start')
   .description(

genkit-tools/cli/src/commands/ui-stop.ts

@@ -15,7 +15,9 @@
  */
 
 import {
+  DevToolsInfo,
   findServersDir,
+  isValidDevToolsInfo,
   logger,
   waitUntilUnresponsive,
 } from '@genkit-ai/tools-common/utils';
@@ -24,7 +26,6 @@ import * as clc from 'colorette';
 import { Command } from 'commander';
 import fs from 'fs/promises';
 import path from 'path';
-import { DevToolsInfo, isValidDevToolsInfo } from './ui-start';
 
 /** Command to stop the Genkit Developer UI. */
 export const uiStop = new Command('ui:stop')

genkit-tools/common/src/manager/manager.ts

@@ -29,9 +29,13 @@ import { GenkitErrorData } from '../types/error';
 import { TraceData } from '../types/trace';
 import { logger } from '../utils/logger';
 import {
+  DevToolsInfo,
   checkServerHealth,
   findRuntimesDir,
+  findServersDir,
+  isValidDevToolsInfo,
   projectNameFromGenkitFilePath,
+  removeToolsInfoFile,
   retriable,
 } from '../utils/utils';
 import {
@@ -54,6 +58,7 @@ interface RuntimeManagerOptions {
 
 export class RuntimeManager {
   private filenameToRuntimeMap: Record<string, RuntimeInfo> = {};
+  private filenameToDevUiMap: Record<string, DevToolsInfo> = {};
   private idToFileMap: Record<string, string> = {};
   private eventEmitter = new EventEmitter();
 
@@ -71,6 +76,7 @@ export class RuntimeManager {
       options.manageHealth ?? true
     );
     await manager.setupRuntimesWatcher();
+    await manager.setupDevUiWatcher();
     if (manager.manageHealth) {
       setInterval(
         async () => await manager.performHealthChecks(),
@@ -112,6 +118,18 @@ export class RuntimeManager {
         );
   }
 
+  /**
+   * Gets the Dev UI that was started most recently.
+   */
+  getMostRecentDevUI(): DevToolsInfo | undefined {
+    const toolsInfo = Object.values(this.filenameToDevUiMap);
+    return toolsInfo.length === 0
+      ? undefined
+      : toolsInfo.reduce((a, b) =>
+          new Date(a.timestamp) > new Date(b.timestamp) ? a : b
+        );
+  }
+
   /**
    * Subscribe to changes to the available runtimes. e.g.) whenever a new
    * runtime is added or removed.
@@ -326,6 +344,73 @@ export class RuntimeManager {
     }
   }
 
+  /**
+   * Sets up a watcher for the servers directory.
+   */
+  private async setupDevUiWatcher() {
+    try {
+      const serversDir = await findServersDir();
+      await fs.mkdir(serversDir, { recursive: true });
+      const watcher = chokidar.watch(serversDir, {
+        persistent: true,
+        ignoreInitial: false,
+      });
+      watcher.on('add', (filePath) => this.handleNewDevUi(filePath));
+      if (this.manageHealth) {
+        watcher.on('unlink', (filePath) => this.handleRemovedDevUi(filePath));
+      }
+      // eagerly check existing Dev UI on first load.
+      for (const toolsInfo of await fs.readdir(serversDir)) {
+        await this.handleNewDevUi(path.resolve(serversDir, toolsInfo));
+      }
+    } catch (error) {
+      logger.error('Failed to set up tools server watcher:', error);
+    }
+  }
+
+  /**
+   * Handles a new Dev UI file.
+   */
+  private async handleNewDevUi(filePath: string) {
+    try {
+      const { content, toolsInfo } = await retriable(
+        async () => {
+          const content = await fs.readFile(filePath, 'utf-8');
+          const toolsInfo = JSON.parse(content) as DevToolsInfo;
+          return { content, toolsInfo };
+        },
+        { maxRetries: 10, delayMs: 500 }
+      );
+
+      if (isValidDevToolsInfo(toolsInfo)) {
+        const fileName = path.basename(filePath);
+        if (await checkServerHealth(toolsInfo.url)) {
+          this.filenameToDevUiMap[fileName] = toolsInfo;
+        } else {
+          logger.debug('Found an unhealthy tools config file', fileName);
+          await removeToolsInfoFile(fileName);
+        }
+      } else {
+        logger.error(`Unexpected file in the servers directory: ${content}`);
+      }
+    } catch (error) {
+      logger.info('Error reading tools config', error);
+      return undefined;
+    }
+  }
+
+  /**
+   * Handles a removed Dev UI file.
+   */
+  private handleRemovedDevUi(filePath: string) {
+    const fileName = path.basename(filePath);
+    if (fileName in this.filenameToDevUiMap) {
+      const toolsInfo = this.filenameToDevUiMap[fileName];
+      delete this.filenameToDevUiMap[fileName];
+      logger.debug(`Removed Dev UI with url ${toolsInfo.url}.`);
+    }
+  }
+
   /**
    * Handles a new runtime file.
    */

genkit-tools/common/src/server/server.ts

@@ -23,6 +23,7 @@ import os from 'os';
 import path from 'path';
 import { GenkitToolsError } from '../manager';
 import { RuntimeManager } from '../manager/manager';
+import { writeToolsInfoFile } from '../utils';
 import { logger } from '../utils/logger';
 import { toolsPackage } from '../utils/package';
 import { downloadAndExtractUiAssets } from '../utils/ui-assets';
@@ -162,9 +163,10 @@ export function startServer(manager: RuntimeManager, port: number) {
   };
   app.use(errorHandler);
 
-  server = app.listen(port, () => {
+  server = app.listen(port, async () => {
     const uiUrl = 'http://localhost:' + port;
     logger.info(`${clc.green(clc.bold('Genkit Developer UI:'))} ${uiUrl}`);
+    await writeToolsInfoFile(uiUrl);
   });
 
   return new Promise<void>((resolve) => {

genkit-tools/common/src/utils/utils.ts

@@ -17,6 +17,14 @@
 import * as fs from 'fs/promises';
 import * as path from 'path';
 import { Runtime } from '../manager/types';
+import { logger } from './logger';
+
+export interface DevToolsInfo {
+  /** URL of the dev tools server. */
+  url: string;
+  /** Timestamp of when the dev tools server was started. */
+  timestamp: string;
+}
 
 /**
  * Finds the project root by looking for a `package.json` file.
@@ -215,3 +223,47 @@ export async function retriable<T>(
     attempt++;
   }
 }
+
+/**
+ * Checks if the provided data is a valid dev tools server state file.
+ */
+export function isValidDevToolsInfo(data: any): data is DevToolsInfo {
+  return (
+    typeof data === 'object' &&
+    typeof data.url === 'string' &&
+    typeof data.timestamp === 'string'
+  );
+}
+
+/**
+ * Writes the toolsInfo file to the project root
+ */
+export async function writeToolsInfoFile(url: string, projectRoot?: string) {
+  const serversDir = await findServersDir(projectRoot);
+  const toolsJsonPath = path.join(serversDir, `tools-${process.pid}.json`);
+  try {
+    const serverInfo = {
+      url,
+      timestamp: new Date().toISOString(),
+    } as DevToolsInfo;
+    await fs.mkdir(serversDir, { recursive: true });
+    await fs.writeFile(toolsJsonPath, JSON.stringify(serverInfo, null, 2));
+    logger.debug(`Tools Info file written: ${toolsJsonPath}`);
+  } catch (error) {
+    logger.info('Error writing tools config', error);
+  }
+}
+
+/**
+ * Removes the toolsInfo file.
+ */
+export async function removeToolsInfoFile(fileName: string) {
+  try {
+    const serversDir = await findServersDir();
+    const filePath = path.join(serversDir, fileName);
+    await fs.unlink(filePath);
+    logger.debug(`Removed unhealthy toolsInfo file ${fileName} from manager.`);
+  } catch (error) {
+    logger.debug(`Failed to delete toolsInfo file: ${error}`);
+  }
+}