diff --git a/.env.example b/.env.example index d6159887944..f8ba70f3095 100644 --- a/.env.example +++ b/.env.example @@ -240,12 +240,12 @@ FLOW_ENDPOINT_URL= # Default: https://mainnet.onflow.org INTERNET_COMPUTER_PRIVATE_KEY= INTERNET_COMPUTER_ADDRESS= -# Github -GITHUB_API_TOKEN= # from github developer portal # Aptos APTOS_PRIVATE_KEY= # Aptos private key APTOS_NETWORK= # must be one of mainnet, testnet +# Github +GITHUB_API_TOKEN= # from github developer portal # AWS S3 Configuration Settings for File Upload AWS_ACCESS_KEY_ID= diff --git a/packages/plugin-github/src/plugins/createCommit.ts b/packages/plugin-github/src/plugins/createCommit.ts index 1331a305e6e..958203833e4 100644 --- a/packages/plugin-github/src/plugins/createCommit.ts +++ b/packages/plugin-github/src/plugins/createCommit.ts @@ -17,6 +17,11 @@ import { isCreateCommitContent, } from "../types"; import { commitAndPushChanges, getRepoPath, writeFiles } from "../utils"; +import { sourceCodeProvider } from "../providers/sourceCode"; +import { testFilesProvider } from "../providers/testFiles"; +import { workflowFilesProvider } from "../providers/workflowFiles"; +import { documentationFilesProvider } from "../providers/documentationFiles"; +import { releasesProvider } from "../providers/releases"; export const createCommitAction: Action = { name: "CREATE_COMMIT", @@ -195,10 +200,16 @@ export const createCommitAction: Action = { }; export const githubCreateCommitPlugin: Plugin = { - name: "githubCreateCommitPlugin", + name: "githubCreateCommit", description: - "Integration with GitHub for commiting changes to the repository", + "Integration with GitHub for committing changes to the repository", actions: [createCommitAction], evaluators: [], - providers: [], + providers: [ + sourceCodeProvider, + testFilesProvider, + workflowFilesProvider, + documentationFilesProvider, + releasesProvider, + ], }; diff --git a/packages/plugin-github/src/plugins/createMemoriesFromFiles.ts b/packages/plugin-github/src/plugins/createMemoriesFromFiles.ts index 921a1469176..a80fc39c473 100644 --- a/packages/plugin-github/src/plugins/createMemoriesFromFiles.ts +++ b/packages/plugin-github/src/plugins/createMemoriesFromFiles.ts @@ -21,6 +21,11 @@ import { isCreateMemoriesFromFilesContent, } from "../types"; import { getRepoPath, retrieveFiles } from "../utils"; +import { sourceCodeProvider } from "../providers/sourceCode"; +import { testFilesProvider } from "../providers/testFiles"; +import { workflowFilesProvider } from "../providers/workflowFiles"; +import { documentationFilesProvider } from "../providers/documentationFiles"; +import { releasesProvider } from "../providers/releases"; export async function addFilesToMemory( runtime: IAgentRuntime, @@ -320,5 +325,11 @@ export const githubCreateMemorizeFromFilesPlugin: Plugin = { description: "Integration with GitHub for creating memories from files", actions: [createMemoriesFromFilesAction], evaluators: [], - providers: [], + providers: [ + sourceCodeProvider, + testFilesProvider, + workflowFilesProvider, + documentationFilesProvider, + releasesProvider, + ], }; diff --git a/packages/plugin-github/src/plugins/createPullRequest.ts b/packages/plugin-github/src/plugins/createPullRequest.ts index e36ae66a32d..622a9953276 100644 --- a/packages/plugin-github/src/plugins/createPullRequest.ts +++ b/packages/plugin-github/src/plugins/createPullRequest.ts @@ -23,6 +23,11 @@ import { getRepoPath, writeFiles, } from "../utils"; +import { sourceCodeProvider } from "../providers/sourceCode"; +import { testFilesProvider } from "../providers/testFiles"; +import { workflowFilesProvider } from "../providers/workflowFiles"; +import { documentationFilesProvider } from "../providers/documentationFiles"; +import { releasesProvider } from "../providers/releases"; export const createPullRequestAction: Action = { name: "CREATE_PULL_REQUEST", @@ -242,5 +247,11 @@ export const githubCreatePullRequestPlugin: Plugin = { description: "Integration with GitHub for creating a pull request", actions: [createPullRequestAction], evaluators: [], - providers: [], + providers: [ + sourceCodeProvider, + testFilesProvider, + workflowFilesProvider, + documentationFilesProvider, + releasesProvider, + ], }; diff --git a/packages/plugin-github/src/plugins/initializeRepository.ts b/packages/plugin-github/src/plugins/initializeRepository.ts index bd802cfa633..e9c6b96500b 100644 --- a/packages/plugin-github/src/plugins/initializeRepository.ts +++ b/packages/plugin-github/src/plugins/initializeRepository.ts @@ -22,6 +22,11 @@ import { createReposDirectory, getRepoPath, } from "../utils"; +import { sourceCodeProvider } from "../providers/sourceCode"; +import { testFilesProvider } from "../providers/testFiles"; +import { workflowFilesProvider } from "../providers/workflowFiles"; +import { documentationFilesProvider } from "../providers/documentationFiles"; +import { releasesProvider } from "../providers/releases"; export const initializeRepositoryAction: Action = { name: "INITIALIZE_REPOSITORY", @@ -252,5 +257,11 @@ export const githubInitializePlugin: Plugin = { description: "Integration with GitHub for initializing the repository", actions: [initializeRepositoryAction], evaluators: [], - providers: [], + providers: [ + sourceCodeProvider, + testFilesProvider, + workflowFilesProvider, + documentationFilesProvider, + releasesProvider, + ], }; diff --git a/packages/plugin-github/src/providers/documentationFiles.ts b/packages/plugin-github/src/providers/documentationFiles.ts new file mode 100644 index 00000000000..a70c4ad5606 --- /dev/null +++ b/packages/plugin-github/src/providers/documentationFiles.ts @@ -0,0 +1,14 @@ +import { Provider } from "@ai16z/eliza"; +import { fetchFiles } from "../utils/githubProviderUtil"; + +export const documentationFilesProvider: Provider = { + get: async (runtime, message, state) => { + return fetchFiles( + runtime, + message, + state, + "documentation files", + (githubService) => githubService.getDocumentation() + ); + }, +}; diff --git a/packages/plugin-github/src/providers/releases.ts b/packages/plugin-github/src/providers/releases.ts new file mode 100644 index 00000000000..1f0b3ae9f97 --- /dev/null +++ b/packages/plugin-github/src/providers/releases.ts @@ -0,0 +1,17 @@ +import { Provider } from "@ai16z/eliza"; +import { fetchFiles } from "../utils/githubProviderUtil"; +import { GitHubService } from "../services/github"; + +export const releasesProvider: Provider = { + get: async (runtime, message, state) => { + return fetchFiles( + runtime, + message, + state, + "releases", + (githubService) => githubService.getReleases(), + (release) => release, + async (githubService, path) => path + ); + }, +}; diff --git a/packages/plugin-github/src/providers/sourceCode.ts b/packages/plugin-github/src/providers/sourceCode.ts new file mode 100644 index 00000000000..8624dc8103c --- /dev/null +++ b/packages/plugin-github/src/providers/sourceCode.ts @@ -0,0 +1,14 @@ +import { Provider } from "@ai16z/eliza"; +import { fetchFiles } from "../utils/githubProviderUtil"; + +export const sourceCodeProvider: Provider = { + get: async (runtime, message, state) => { + return fetchFiles( + runtime, + message, + state, + "source code", + (githubService) => githubService.getSourceFiles("") + ); + }, +}; diff --git a/packages/plugin-github/src/providers/testFiles.ts b/packages/plugin-github/src/providers/testFiles.ts new file mode 100644 index 00000000000..20871cb5a23 --- /dev/null +++ b/packages/plugin-github/src/providers/testFiles.ts @@ -0,0 +1,15 @@ +import { Provider } from "@ai16z/eliza"; +import { fetchFiles } from "../utils/githubProviderUtil"; + +export const testFilesProvider: Provider = { + get: async (runtime, message, state) => { + const testPath = (state?.testPath as string) || ""; // Optional test directory path + return fetchFiles( + runtime, + message, + state, + "test files", + (githubService) => githubService.getTestFiles(testPath) + ); + }, +}; diff --git a/packages/plugin-github/src/providers/workflowFiles.ts b/packages/plugin-github/src/providers/workflowFiles.ts new file mode 100644 index 00000000000..142bbe131b9 --- /dev/null +++ b/packages/plugin-github/src/providers/workflowFiles.ts @@ -0,0 +1,15 @@ +import { Provider } from "@ai16z/eliza"; +import { fetchFiles } from "../utils/githubProviderUtil"; + +export const workflowFilesProvider: Provider = { + get: async (runtime, message, state) => { + return fetchFiles( + runtime, + message, + state, + "workflow files", + (githubService) => githubService.getWorkflows(), + (workflow) => workflow.path + ); + }, +}; diff --git a/packages/plugin-github/src/services/github.ts b/packages/plugin-github/src/services/github.ts new file mode 100644 index 00000000000..5cb33c7a12d --- /dev/null +++ b/packages/plugin-github/src/services/github.ts @@ -0,0 +1,150 @@ +import { Octokit, RestEndpointMethodTypes } from "@octokit/rest"; +import { elizaLogger } from "@ai16z/eliza"; + +interface GitHubConfig { + owner: string; + repo: string; + auth: string; +} + +export class GitHubService { + private octokit: Octokit; + private config: GitHubConfig; + + constructor(config: GitHubConfig) { + this.config = config; + this.octokit = new Octokit({ auth: config.auth }); + } + + // Scenario 1 & 2: Get file contents for code analysis + async getFileContents(path: string): Promise { + try { + const response = await this.octokit.repos.getContent({ + owner: this.config.owner, + repo: this.config.repo, + path, + }); + + // GitHub API returns content as base64 + if ("content" in response.data && !Array.isArray(response.data)) { + return Buffer.from(response.data.content, "base64").toString(); + } + throw new Error("Unable to get file contents"); + } catch (error) { + elizaLogger.error(`Error getting file contents: ${error}`); + throw error; + } + } + + // Scenario 3: Get test files + async getTestFiles(testPath: string): Promise { + try { + const response = await this.octokit.repos.getContent({ + owner: this.config.owner, + repo: this.config.repo, + path: testPath, + }); + + if (Array.isArray(response.data)) { + return response.data + .filter( + (file) => + file.type === "file" && file.name.includes("test") + ) + .map((file) => file.path); + } + return []; + } catch (error) { + elizaLogger.error(`Error getting test files: ${error}`); + throw error; + } + } + + // Scenario 4: Get workflow files + async getWorkflows(): Promise< + RestEndpointMethodTypes["actions"]["listRepoWorkflows"]["response"]["data"]["workflows"] + > { + try { + const response = await this.octokit.actions.listRepoWorkflows({ + owner: this.config.owner, + repo: this.config.repo, + }); + + return response.data.workflows; + } catch (error) { + elizaLogger.error(`Error getting workflows: ${error}`); + throw error; + } + } + + // Scenario 5: Get documentation files + async getDocumentation(docPath: string = ""): Promise { + try { + const response = await this.octokit.repos.getContent({ + owner: this.config.owner, + repo: this.config.repo, + path: docPath, + }); + + if (Array.isArray(response.data)) { + return response.data + .filter( + (file) => + file.type === "file" && + (file.name.toLowerCase().includes("readme") || + file.name.toLowerCase().includes("docs") || + file.path.includes(".md")) + ) + .map((file) => file.path); + } + return []; + } catch (error) { + elizaLogger.error(`Error getting documentation: ${error}`); + throw error; + } + } + + // Scenario 6: Get releases and changelogs + async getReleases(): Promise< + RestEndpointMethodTypes["repos"]["listReleases"]["response"]["data"] + > { + try { + const response = await this.octokit.repos.listReleases({ + owner: this.config.owner, + repo: this.config.repo, + }); + + return response.data; + } catch (error) { + elizaLogger.error(`Error getting releases: ${error}`); + throw error; + } + } + + // Scenario 7: Get source files for refactoring analysis + async getSourceFiles(sourcePath: string): Promise { + try { + const response = await this.octokit.repos.getContent({ + owner: this.config.owner, + repo: this.config.repo, + path: sourcePath, + }); + + if (Array.isArray(response.data)) { + return response.data + .filter( + (file) => + file.type === "file" && + !file.name.toLowerCase().includes("test") + ) + .map((file) => file.path); + } + return []; + } catch (error) { + elizaLogger.error(`Error getting source files: ${error}`); + throw error; + } + } +} + +export { GitHubConfig }; diff --git a/packages/plugin-github/src/templates.ts b/packages/plugin-github/src/templates.ts index e4f9abbf36c..507a0223476 100644 --- a/packages/plugin-github/src/templates.ts +++ b/packages/plugin-github/src/templates.ts @@ -90,3 +90,23 @@ Provide the commit details in the following JSON format: Here are the recent user messages for context: {{recentMessages}} `; + +export const fetchFilesTemplate = ` +Extract the details for fetching files from the GitHub repository: +- **owner** (string): The owner of the GitHub repository (e.g., "octocat") +- **repo** (string): The name of the GitHub repository (e.g., "hello-world") +- **branch** (string): The branch of the GitHub repository (e.g., "main") + +Provide the repository details in the following JSON format: + +\`\`\`json +{ + "owner": "", + "repo": "", + "branch": "" +} +\`\`\` + +Here are the recent user messages for context: +{{recentMessages}} +`; diff --git a/packages/plugin-github/src/types.ts b/packages/plugin-github/src/types.ts index c0fc293e608..a49aa0d4b66 100644 --- a/packages/plugin-github/src/types.ts +++ b/packages/plugin-github/src/types.ts @@ -1,3 +1,4 @@ +import { elizaLogger } from "@ai16z/eliza"; import { z } from "zod"; export const InitializeSchema = z.object({ @@ -18,7 +19,7 @@ export const isInitializeContent = ( if (InitializeSchema.safeParse(object).success) { return true; } - console.error("Invalid content: ", object); + elizaLogger.error("Invalid content: ", object); return false; }; @@ -40,7 +41,7 @@ export const isCreateMemoriesFromFilesContent = ( if (CreateMemoriesFromFilesSchema.safeParse(object).success) { return true; } - console.error("Invalid content: ", object); + elizaLogger.error("Invalid content: ", object); return false; }; @@ -70,7 +71,7 @@ export const isCreatePullRequestContent = ( if (CreatePullRequestSchema.safeParse(object).success) { return true; } - console.error("Invalid content: ", object); + elizaLogger.error("Invalid content: ", object); return false; }; @@ -94,6 +95,28 @@ export const isCreateCommitContent = ( if (CreateCommitSchema.safeParse(object).success) { return true; } - console.error("Invalid content: ", object); + elizaLogger.error("Invalid content: ", object); + return false; +}; + +export const FetchFilesSchema = z.object({ + owner: z.string().min(1, "GitHub owner is required"), + repo: z.string().min(1, "GitHub repo is required"), + branch: z.string().min(1, "GitHub branch is required"), +}); + +export interface FetchFilesContent { + owner: string; + repo: string; + branch: string; +} + +export const isFetchFilesContent = ( + object: any +): object is FetchFilesContent => { + if (FetchFilesSchema.safeParse(object).success) { + return true; + } + elizaLogger.error("Invalid content: ", object); return false; }; diff --git a/packages/plugin-github/src/utils/githubProviderUtil.ts b/packages/plugin-github/src/utils/githubProviderUtil.ts new file mode 100644 index 00000000000..3e3f6e77561 --- /dev/null +++ b/packages/plugin-github/src/utils/githubProviderUtil.ts @@ -0,0 +1,107 @@ +import { + composeContext, + generateObjectV2, + elizaLogger, + IAgentRuntime, + Memory, + State, + ModelClass, +} from "@ai16z/eliza"; +import { GitHubService } from "../services/github"; +import { + FetchFilesContent, + FetchFilesSchema, + isFetchFilesContent, +} from "../types"; +import { fetchFilesTemplate } from "../templates"; + +export async function fetchFiles( + runtime: IAgentRuntime, + message: Memory, + state: State, + description: string, + fetchFunction: (githubService: GitHubService) => Promise, + formatPath: (path: any) => string = (path) => path, + getContentFunction: ( + githubService: GitHubService, + item: any + ) => Promise = (service, item) => service.getFileContents(item) +) { + try { + elizaLogger.log("Composing state for message:", message); + if (!state) { + state = (await runtime.composeState(message)) as State; + } else { + state = await runtime.updateRecentMessageState(state); + } + + const context = composeContext({ + state, + template: fetchFilesTemplate, + }); + + const details = await generateObjectV2({ + runtime, + context, + modelClass: ModelClass.SMALL, + schema: FetchFilesSchema, + }); + + if (!isFetchFilesContent(details.object)) { + elizaLogger.error("Invalid content:", details.object); + throw new Error("Invalid content"); + } + + const content = details.object as FetchFilesContent; + + const owner = content.owner; + const repo = content.repo; + const branch = content.branch; + + elizaLogger.info( + `Fetching ${description} from GitHub ${owner}/${repo} on branch ${branch}` + ); + + if (!owner || !repo || !branch) { + elizaLogger.warn( + `Missing repository details in state for ${description}` + ); + return { files: [], repository: null }; + } + + // Initialize GitHub service + const githubService = new GitHubService({ + auth: runtime.getSetting("GITHUB_API_TOKEN"), + owner, + repo, + }); + + // Fetch file paths using the provided function + const filePaths = await fetchFunction(githubService); + + // Get contents for each file + const fileContents = await Promise.all( + filePaths.map(async (path) => { + path = formatPath(path); + const content = await getContentFunction(githubService, path); + return { path, content }; + }) + ); + + elizaLogger.info( + `Retrieved ${fileContents.length} files from ${owner}/${repo} for ${description}` + ); + + return { + files: fileContents, + repository: { + owner, + repo, + branch, + }, + }; + } catch (error) { + elizaLogger.error(`Error in fetchFiles for ${description}:`, error); + return { files: [], repository: null }; + } +}