From aadde5a33cc2f4c0bd6583b4da2d60f83a0c6879 Mon Sep 17 00:00:00 2001 From: Guillaume Bourdat Date: Wed, 5 Mar 2025 16:51:39 +0100 Subject: [PATCH 1/6] #291 : import data from GitLab --- .vscode/launch.json | 11 ++ api/package.json | 1 + api/src/core/adapters/GitLab/api/project.ts | 54 ++++++++++ api/src/core/adapters/GitLab/api/utils.ts | 30 ++++++ .../createPgSoftwareExternalDataRepository.ts | 1 + .../kysely/createPgSoftwareRepository.ts | 6 +- .../adapters/dbApi/kysely/kysely.database.ts | 7 ++ .../1741091782480_add-repo-metadata.ts | 9 ++ .../dbApi/kysely/pgDbApi.integration.test.ts | 1 + .../core/adapters/fetchExternalData.test.ts | 3 + .../hal/getHalSoftwareExternalData.ts | 34 +++++- api/src/core/ports/GetSoftwareExternalData.ts | 7 ++ api/src/tools/repoAnalyser.test.ts | 100 ++++++++++++++++++ api/src/tools/repoAnalyser.ts | 31 ++++++ 14 files changed, 293 insertions(+), 2 deletions(-) create mode 100644 api/src/core/adapters/GitLab/api/project.ts create mode 100644 api/src/core/adapters/GitLab/api/utils.ts create mode 100644 api/src/core/adapters/dbApi/kysely/migrations/1741091782480_add-repo-metadata.ts create mode 100644 api/src/tools/repoAnalyser.test.ts create mode 100644 api/src/tools/repoAnalyser.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index 5f8b27c2..7e23b33c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -35,6 +35,17 @@ "run", "update" ], + }, + { + "type": "node", + "request": "launch", + "name": "Fullcheck", + "runtimeExecutable": "yarn", + "cwd": "${workspaceFolder}", + "runtimeArgs": [ + "run", + "fullcheck" + ], } ] } \ No newline at end of file diff --git a/api/package.json b/api/package.json index 4a145fa1..09a665d9 100644 --- a/api/package.json +++ b/api/package.json @@ -70,6 +70,7 @@ "vitest": "^1.2.2" }, "dependencies": { + "@gitbeaker/core": "^42.1.0", "@octokit/graphql": "^7.0.2", "@trpc/server": "^10.18.0", "@types/pg": "^8.11.6", diff --git a/api/src/core/adapters/GitLab/api/project.ts b/api/src/core/adapters/GitLab/api/project.ts new file mode 100644 index 00000000..a1cff1f4 --- /dev/null +++ b/api/src/core/adapters/GitLab/api/project.ts @@ -0,0 +1,54 @@ +import { CommitSchema, IssueSchema, MergeRequestSchema } from "@gitbeaker/core"; +import { repoUrlToAPIUrl } from "./utils"; + +const getApiCallTakeFirst = async (url: string): Promise => { + const res = await fetch(url, { + signal: AbortSignal.timeout(10000) + }).catch(err => { + console.error(url, err); + }); + + if (!res) { + return undefined; + } + if (res.status === 404) { + console.error("Ressource not available"); + return undefined; + } + if (res.status === 403) { + console.info(`You don't seems to be allowed on ${url}`); + return undefined; + } + + const result: T[] = await res.json(); + + return result[0]; +}; + +const getLastClosedIssue = async (projectUrl: string) => { + return getApiCallTakeFirst(`${projectUrl}/issues?sort=desc&state=closed`); +}; + +const getLastCommit = async (projectUrl: string) => { + return getApiCallTakeFirst(`${projectUrl}/repository/commits?sort=desc`); +}; + +const getLastMergeRequest = async (projectUrl: string) => { + return getApiCallTakeFirst(`${projectUrl}/merge_requests?state=closed&sort=desc`); +}; + +export const projectEndpointMaker = (repoUrl: string | URL) => { + const apiProjectEndpoint = repoUrlToAPIUrl(repoUrl); + + return { + issues: { + getLastClosedIssue: () => getLastClosedIssue(apiProjectEndpoint) + }, + commits: { + getLastCommit: () => getLastCommit(apiProjectEndpoint) + }, + mergeRequests: { + getLast: () => getLastMergeRequest(apiProjectEndpoint) + } + }; +}; diff --git a/api/src/core/adapters/GitLab/api/utils.ts b/api/src/core/adapters/GitLab/api/utils.ts new file mode 100644 index 00000000..17c4bbd0 --- /dev/null +++ b/api/src/core/adapters/GitLab/api/utils.ts @@ -0,0 +1,30 @@ +export const repoUrlToAPIUrl = (projectUrl: string | URL): string => { + let url = projectUrl; + + if (typeof url === "string") { + // Case git+ at the beging + if (url.startsWith("git+")) url = url.substring(4); + + // Case ssh protocol + if (url.startsWith("git@")) url = url.replace(":", "/").replace("git@", "https://"); + + // Case .git at the end + if (url.endsWith(".git")) url = url.slice(0, -4); + } + + const urlObj = typeof projectUrl === "string" ? URL.parse(url) : projectUrl; + + if (!urlObj) { + throw new Error("Bad URL"); + } + + const base = urlObj.origin; + + let projectPath = urlObj.pathname.substring(1); + if (projectPath.includes("/-/")) projectPath = projectPath.split("-")[0]; + // Case / at the end + if (projectPath.endsWith("/")) projectPath = projectPath.slice(0, -1); + projectPath = projectPath.replaceAll("/", "%2F"); + + return `${base}/api/v4/projects/${projectPath}`; +}; diff --git a/api/src/core/adapters/dbApi/kysely/createPgSoftwareExternalDataRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgSoftwareExternalDataRepository.ts index 84b79905..4d57c7a7 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgSoftwareExternalDataRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgSoftwareExternalDataRepository.ts @@ -13,6 +13,7 @@ export const createPgSoftwareExternalDataRepository = (db: Kysely): So programmingLanguages: JSON.stringify(softwareExternalData.programmingLanguages), referencePublications: JSON.stringify(softwareExternalData.referencePublications), identifiers: JSON.stringify(softwareExternalData.identifiers), + repoMetadata: JSON.stringify(softwareExternalData.repoMetadata), description: JSON.stringify(softwareExternalData.description) }; diff --git a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts index d5ce9960..d08ec5f5 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts @@ -189,6 +189,7 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi programmingLanguages: softwareExternalData?.programmingLanguages ?? [], referencePublications: softwareExternalData?.referencePublications, identifiers: softwareExternalData?.identifiers, + repoMetadata: softwareExternalData?.repoMetadata, applicationCategories: software.categories.concat( softwareExternalData?.applicationCategories ?? [] ), @@ -292,7 +293,8 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi categories: undefined, // merged in applicationCategories, set to undefined to remove it programmingLanguages: softwareExternalData?.programmingLanguages ?? [], referencePublications: softwareExternalData?.referencePublications, - identifiers: softwareExternalData?.identifiers + identifiers: softwareExternalData?.identifiers, + repoMetadata: softwareExternalData?.repoMetadata }); } ); @@ -427,6 +429,7 @@ const makeGetSoftwareBuilder = (db: Kysely) => applicationCategories: ref("ext.applicationCategories"), referencePublications: ref("ext.referencePublications"), identifiers: ref("ext.identifiers"), + repoMetadata: ref("ext.repoMetadata"), keywords: ref("ext.keywords"), softwareVersion: ref("ext.softwareVersion"), publicationTime: ref("ext.publicationTime") @@ -561,6 +564,7 @@ const makeGetSoftwareById = programmingLanguages: softwareExternalData?.programmingLanguages ?? [], referencePublications: softwareExternalData?.referencePublications, identifiers: softwareExternalData?.identifiers, + repoMetadata: softwareExternalData?.repoMetadata, applicationCategories: filterDuplicate( software.categories.concat(softwareExternalData?.applicationCategories ?? []) ), diff --git a/api/src/core/adapters/dbApi/kysely/kysely.database.ts b/api/src/core/adapters/dbApi/kysely/kysely.database.ts index dd16c77e..d8a4d0bc 100644 --- a/api/src/core/adapters/dbApi/kysely/kysely.database.ts +++ b/api/src/core/adapters/dbApi/kysely/kysely.database.ts @@ -97,6 +97,13 @@ type SoftwareExternalDatasTable = { referencePublications: JSONColumnType | null; publicationTime: Date | null; identifiers: JSONColumnType | null; + repoMetadata: JSONColumnType<{ + healthCheck?: { + lastCommit?: number; + lastClosedIssue?: number; + lastClosedIssuePullRequest?: number; + }; + }> | null; }; type SoftwareType = diff --git a/api/src/core/adapters/dbApi/kysely/migrations/1741091782480_add-repo-metadata.ts b/api/src/core/adapters/dbApi/kysely/migrations/1741091782480_add-repo-metadata.ts new file mode 100644 index 00000000..278b0209 --- /dev/null +++ b/api/src/core/adapters/dbApi/kysely/migrations/1741091782480_add-repo-metadata.ts @@ -0,0 +1,9 @@ +import { Kysely } from "kysely"; + +export async function up(db: Kysely): Promise { + await db.schema.alterTable("software_external_datas").addColumn("repoMetadata", "jsonb").execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.alterTable("software_external_datas").dropColumn("repoMetadata").execute(); +} diff --git a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts index 2e90e863..fa633daf 100644 --- a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts +++ b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts @@ -446,6 +446,7 @@ describe("pgDbApi", () => { applicationCategories: JSON.stringify(softExtData.applicationCategories), programmingLanguages: JSON.stringify(softExtData.programmingLanguages), identifiers: JSON.stringify(softExtData.identifiers), + repoMetadata: JSON.stringify(softExtData.repoMetadata), referencePublications: JSON.stringify(softExtData.referencePublications) })) ) diff --git a/api/src/core/adapters/fetchExternalData.test.ts b/api/src/core/adapters/fetchExternalData.test.ts index b1a53a7d..4ecc3886 100644 --- a/api/src/core/adapters/fetchExternalData.test.ts +++ b/api/src/core/adapters/fetchExternalData.test.ts @@ -187,6 +187,7 @@ describe("fetches software extra data (from different providers)", () => { programmingLanguages: [], referencePublications: null, identifiers: [], + repoMetadata: null, softwareVersion: "5.0.1", publicationTime: new Date("2022-04-12T00:00:00.000Z") }, @@ -216,6 +217,7 @@ describe("fetches software extra data (from different providers)", () => { programmingLanguages: ["JavaScript"], referencePublications: null, identifiers: [], + repoMetadata: null, softwareVersion: expect.any(String), publicationTime: expect.any(Date) } @@ -271,6 +273,7 @@ describe("fetches software extra data (from different providers)", () => { websiteUrl: "https://httpd.apache.org/", referencePublications: null, identifiers: [], + repoMetadata: null, programmingLanguages: ["C"], softwareVersion: "2.5.0-alpha", publicationTime: new Date("2017-11-08T00:00:00.000Z") diff --git a/api/src/core/adapters/hal/getHalSoftwareExternalData.ts b/api/src/core/adapters/hal/getHalSoftwareExternalData.ts index eb3c5c39..e1ead67e 100644 --- a/api/src/core/adapters/hal/getHalSoftwareExternalData.ts +++ b/api/src/core/adapters/hal/getHalSoftwareExternalData.ts @@ -5,6 +5,8 @@ import { halAPIGateway } from "./HalAPI"; import { HalFetchError } from "./HalAPI/type"; import { SILL } from "../../../types/SILL"; import { HAL } from "./types/HAL"; +import { repoAnalyser, RepoType } from "../../../tools/repoAnalyser"; +import { projectEndpointMaker } from "../GitLab/api/project"; const buildParentOrganizationTree = async ( structureIdArray: number[] | string[] | undefined @@ -215,6 +217,35 @@ export const getHalSoftwareExternalData: GetSoftwareExternalData = memoize( } }) ?? []; + const repoType = await repoAnalyser(halRawSoftware?.softCodeRepository_s?.[0]); + + const getRepoMetadata = async (repoType: RepoType | undefined) => { + switch (repoType) { + case "GitLab": + const apiProject = projectEndpointMaker(halRawSoftware?.softCodeRepository_s?.[0]); + const lastCommit = await apiProject.commits.getLastCommit(); + const lastIssue = await apiProject.issues.getLastClosedIssue(); + const lastMergeRequest = await apiProject.mergeRequests.getLast(); + return { + healthCheck: { + lastCommit: lastCommit ? new Date(lastCommit.created_at) : undefined, + lastClosedIssue: + lastIssue && lastIssue.closed_at ? new Date(lastIssue.closed_at) : undefined, + lastClosedIssuePullRequest: lastMergeRequest + ? new Date(lastMergeRequest.updated_at) + : undefined + } + }; + case "GitHub": + return undefined; + case undefined: + return undefined; + default: + repoType satisfies never; + return undefined; + } + }; + return { externalId: halRawSoftware.docid, externalDataOrigin: "HAL", @@ -242,7 +273,8 @@ export const getHalSoftwareExternalData: GetSoftwareExternalData = memoize( ? new Date(halRawSoftware?.releasedDate_tdate) : undefined, referencePublications: codeMetaToReferencePublication(codemetaSoftware.referencePublication), - identifiers: identifiers + identifiers: identifiers, + repoMetadata: await getRepoMetadata(repoType) }; }, { diff --git a/api/src/core/ports/GetSoftwareExternalData.ts b/api/src/core/ports/GetSoftwareExternalData.ts index 67794428..e3594d49 100644 --- a/api/src/core/ports/GetSoftwareExternalData.ts +++ b/api/src/core/ports/GetSoftwareExternalData.ts @@ -33,6 +33,13 @@ export type SoftwareExternalData = { publicationTime: Date; referencePublications: SILL.ScholarlyArticle[]; identifiers: SILL.Identification[]; + repoMetadata?: { + healthCheck?: { + lastCommit?: Date; + lastClosedIssue?: Date; + lastClosedIssuePullRequest?: Date; + }; + }; }>; export type SimilarSoftwareExternalData = Pick< diff --git a/api/src/tools/repoAnalyser.test.ts b/api/src/tools/repoAnalyser.test.ts new file mode 100644 index 00000000..f277c12c --- /dev/null +++ b/api/src/tools/repoAnalyser.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect, vi } from "vitest"; +import { repoAnalyser } from "./repoAnalyser"; + +describe("repoAnalyser", () => { + it("should return undefined if url is undefined", async () => { + const result = await repoAnalyser(undefined); + expect(result).toBeUndefined(); + }); + + it('should return "GitHub" for GitHub URLs', async () => { + const result = await repoAnalyser("https://github.com/"); + expect(result).toBe("GitHub"); + }); + + it('should return "GitHub" for GitHub URLs', async () => { + const result = await repoAnalyser("git+https://github.com/agorajs/agora-gml.git"); + expect(result).toBe("GitHub"); + }); + + it('should return "GitLab" for GitLab URLs', async () => { + const mockFetch = vi.fn().mockResolvedValue({ + headers: new Headers({ + "x-gitlab-meta": "true" + }) + }); + + global.fetch = mockFetch; + + const result = await repoAnalyser("https://gitlab.com/"); + expect(result).toBe("GitLab"); + }); + + it('should return "GitLab" for gite.lirmm.fr URLs', async () => { + const result = await repoAnalyser("https://gite.lirmm.fr/doccy/RedOak"); + expect(result).toBe("GitLab"); + }); + + it('should return "GitLab" for gricad-gitlab URLs', async () => { + const result = await repoAnalyser("https://gricad-gitlab.univ-grenoble-alpes.fr/kraifo/ailign"); + expect(result).toBe("GitLab"); + }); + + it('should return "GitLab" for gitlab.lis-lab.fr/ URLs', async () => { + const result = await repoAnalyser("https://gitlab.lis-lab.fr/dev/mincoverpetri"); + expect(result).toBe("GitLab"); + }); + + it('should return "GitLab" for inria gitlab URLs', async () => { + const result = await repoAnalyser("https://forgemia.inra.fr/lisc/easyabc"); + expect(result).toBe("GitLab"); + }); + + it('should return "GitLab" for gricad-gitlab URLs', async () => { + const result = await repoAnalyser("https://gricad-gitlab.univ-grenoble-alpes.fr/kraifo/ailign"); + expect(result).toBe("GitLab"); + }); + + it('should return "GitLab" for gricad-gitlab URLs', async () => { + const result = await repoAnalyser("https://gricad-gitlab.univ-grenoble-alpes.fr/kraifo/ailign"); + expect(result).toBe("GitLab"); + }); + + it("should return undefined for unknown URLs", async () => { + const mockFetch = vi.fn().mockResolvedValue({ + headers: new Headers() + }); + + global.fetch = mockFetch; + + const result = await repoAnalyser("https://unknown.com/"); + expect(result).toBeUndefined(); + }); + + it("should return undefined for pari.math.u-bordeaux.fr URLs", async () => { + const result = await repoAnalyser("https://pari.math.u-bordeaux.fr/git/pari.git"); + expect(result).toBeUndefined(); + }); + + it("should return undefined for codeberg URLs", async () => { + const result = await repoAnalyser("https://codeberg.org/tesselle/aion"); + expect(result).toBeUndefined(); + }); + + it("should return undefined for CRAN.R-project.org URLs", async () => { + const result = await repoAnalyser("https://CRAN.R-project.org/package=glober"); + expect(result).toBeUndefined(); + }); + + it("should return undefined for Zenodo DOI URLs", async () => { + const result = await repoAnalyser("https://doi.org/10.5281/zenodo.11069161"); + expect(result).toBeUndefined(); + }); + + it("should return undefined for who.rocq.inria.fr URLs", async () => { + const result = await repoAnalyser( + "https://who.rocq.inria.fr/Jean-Charles.Gilbert/modulopt/optimization-routines/n1cv2/n1cv2.html" + ); + expect(result).toBeUndefined(); + }); +}); diff --git a/api/src/tools/repoAnalyser.ts b/api/src/tools/repoAnalyser.ts new file mode 100644 index 00000000..d451b007 --- /dev/null +++ b/api/src/tools/repoAnalyser.ts @@ -0,0 +1,31 @@ +export type RepoType = "GitHub" | "GitLab"; + +export const repoAnalyser = async (url: string | URL | undefined): Promise => { + if (!url) return undefined; + + const urlObj = typeof url === "string" ? URL.parse(url.substring(0, 4) === "git+" ? url.substring(4) : url) : url; + + if (!urlObj) { + return undefined; + } + + const test = urlObj.origin; + console.log(test); + + if (urlObj.origin === "https://github.com") { + return "GitHub"; + } + + const urlToGitLab = `${urlObj.origin}/api/v4/metadata`; + const res = await fetch(urlToGitLab, { + signal: AbortSignal.timeout(10000) + }).catch(err => { + console.error(url, err); + }); + + if (res && res.headers && res.headers.has("x-gitlab-meta")) { + return "GitLab"; + } + + return undefined; +}; From 6ea0ca1f6a74b629108b369073a06a2c9d1cc2a4 Mon Sep 17 00:00:00 2001 From: Guillaume Bourdat Date: Thu, 6 Mar 2025 11:29:12 +0100 Subject: [PATCH 2/6] #291 : update type and implement score poc --- .../kysely/createPgSoftwareRepository.ts | 20 ++++++++++++++++--- .../adapters/dbApi/kysely/kysely.database.ts | 8 +------- .../hal/getHalSoftwareExternalData.ts | 6 +++--- api/src/core/ports/GetSoftwareExternalData.ts | 6 +++--- .../core/usecases/readWriteSillData/types.ts | 1 + api/src/types/SILL.ts | 10 ++++++++++ 6 files changed, 35 insertions(+), 16 deletions(-) diff --git a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts index d08ec5f5..468a17e0 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts @@ -6,6 +6,7 @@ import { ParentSoftwareExternalData } from "../../../ports/GetSoftwareExternalDa import { Software } from "../../../usecases/readWriteSillData"; import { Database } from "./kysely.database"; import { stripNullOrUndefinedValues, jsonBuildObject } from "./kysely.utils"; +import { SILL } from "../../../../types/SILL"; const dateParser = (str: string | Date | undefined | null) => { if (str && typeof str === "string") { @@ -17,6 +18,19 @@ const dateParser = (str: string | Date | undefined | null) => { } }; +const computeRepoMetadata = (repoMetadata: SILL.RepoMetadata | undefined | null): SILL.RepoMetadata | undefined => { + const newMedata = repoMetadata; + if (!newMedata || !newMedata.healthCheck) return undefined; + + let score = 0; + if (repoMetadata.healthCheck?.lastClosedIssue) score += 1; + if (repoMetadata.healthCheck?.lastClosedIssuePullRequest) score += 1; + if (repoMetadata.healthCheck?.lastCommit) score += 1; + newMedata.healthCheck.score = score / 3; + + return newMedata; +}; + export const createPgSoftwareRepository = (db: Kysely): SoftwareRepository => { const getBySoftwareId = makeGetSoftwareById(db); return { @@ -189,7 +203,7 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi programmingLanguages: softwareExternalData?.programmingLanguages ?? [], referencePublications: softwareExternalData?.referencePublications, identifiers: softwareExternalData?.identifiers, - repoMetadata: softwareExternalData?.repoMetadata, + repoMetadata: computeRepoMetadata(softwareExternalData?.repoMetadata), applicationCategories: software.categories.concat( softwareExternalData?.applicationCategories ?? [] ), @@ -294,7 +308,7 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi programmingLanguages: softwareExternalData?.programmingLanguages ?? [], referencePublications: softwareExternalData?.referencePublications, identifiers: softwareExternalData?.identifiers, - repoMetadata: softwareExternalData?.repoMetadata + repoMetadata: computeRepoMetadata(softwareExternalData?.repoMetadata) }); } ); @@ -564,7 +578,7 @@ const makeGetSoftwareById = programmingLanguages: softwareExternalData?.programmingLanguages ?? [], referencePublications: softwareExternalData?.referencePublications, identifiers: softwareExternalData?.identifiers, - repoMetadata: softwareExternalData?.repoMetadata, + repoMetadata: computeRepoMetadata(softwareExternalData?.repoMetadata), applicationCategories: filterDuplicate( software.categories.concat(softwareExternalData?.applicationCategories ?? []) ), diff --git a/api/src/core/adapters/dbApi/kysely/kysely.database.ts b/api/src/core/adapters/dbApi/kysely/kysely.database.ts index d8a4d0bc..05029257 100644 --- a/api/src/core/adapters/dbApi/kysely/kysely.database.ts +++ b/api/src/core/adapters/dbApi/kysely/kysely.database.ts @@ -97,13 +97,7 @@ type SoftwareExternalDatasTable = { referencePublications: JSONColumnType | null; publicationTime: Date | null; identifiers: JSONColumnType | null; - repoMetadata: JSONColumnType<{ - healthCheck?: { - lastCommit?: number; - lastClosedIssue?: number; - lastClosedIssuePullRequest?: number; - }; - }> | null; + repoMetadata: JSONColumnType | null; }; type SoftwareType = diff --git a/api/src/core/adapters/hal/getHalSoftwareExternalData.ts b/api/src/core/adapters/hal/getHalSoftwareExternalData.ts index e1ead67e..268994f9 100644 --- a/api/src/core/adapters/hal/getHalSoftwareExternalData.ts +++ b/api/src/core/adapters/hal/getHalSoftwareExternalData.ts @@ -228,11 +228,11 @@ export const getHalSoftwareExternalData: GetSoftwareExternalData = memoize( const lastMergeRequest = await apiProject.mergeRequests.getLast(); return { healthCheck: { - lastCommit: lastCommit ? new Date(lastCommit.created_at) : undefined, + lastCommit: lastCommit ? new Date(lastCommit.created_at).valueOf() : undefined, lastClosedIssue: - lastIssue && lastIssue.closed_at ? new Date(lastIssue.closed_at) : undefined, + lastIssue && lastIssue.closed_at ? new Date(lastIssue.closed_at).valueOf() : undefined, lastClosedIssuePullRequest: lastMergeRequest - ? new Date(lastMergeRequest.updated_at) + ? new Date(lastMergeRequest.updated_at).valueOf() : undefined } }; diff --git a/api/src/core/ports/GetSoftwareExternalData.ts b/api/src/core/ports/GetSoftwareExternalData.ts index e3594d49..9742cff7 100644 --- a/api/src/core/ports/GetSoftwareExternalData.ts +++ b/api/src/core/ports/GetSoftwareExternalData.ts @@ -35,9 +35,9 @@ export type SoftwareExternalData = { identifiers: SILL.Identification[]; repoMetadata?: { healthCheck?: { - lastCommit?: Date; - lastClosedIssue?: Date; - lastClosedIssuePullRequest?: Date; + lastCommit?: number; + lastClosedIssue?: number; + lastClosedIssuePullRequest?: number; }; }; }>; diff --git a/api/src/core/usecases/readWriteSillData/types.ts b/api/src/core/usecases/readWriteSillData/types.ts index 652c7eb7..5a29ad09 100644 --- a/api/src/core/usecases/readWriteSillData/types.ts +++ b/api/src/core/usecases/readWriteSillData/types.ts @@ -61,6 +61,7 @@ export type Software = { programmingLanguages: string[]; referencePublications?: SILL.ScholarlyArticle[]; identifiers?: SILL.Identification[]; + repoMetadata?: SILL.RepoMetadata; }; export namespace Software { diff --git a/api/src/types/SILL.ts b/api/src/types/SILL.ts index f50347fc..c9541897 100644 --- a/api/src/types/SILL.ts +++ b/api/src/types/SILL.ts @@ -60,4 +60,14 @@ export namespace SILL { url?: string; affiliations?: Organization[]; }; + + // Created from nowhere + export type RepoMetadata = { + healthCheck?: { + score?: number; + lastCommit?: number; + lastClosedIssue?: number; + lastClosedIssuePullRequest?: number; + }; + }; } From 50068bc9d99389aea65c17ac9bde4ac66dd3a6b3 Mon Sep 17 00:00:00 2001 From: Guillaume Bourdat Date: Thu, 6 Mar 2025 11:32:00 +0100 Subject: [PATCH 3/6] #203 : show partial repo activity --- .../core/usecases/softwareDetails/state.ts | 1 + .../core/usecases/softwareDetails/thunks.ts | 6 +- web/src/ui/i18n/sill_en.json | 6 +- web/src/ui/i18n/sill_fr.json | 6 +- .../ui/pages/softwareDetails/PreviewTab.tsx | 78 +++++++++++++++++-- .../pages/softwareDetails/SoftwareDetails.tsx | 1 + 6 files changed, 89 insertions(+), 9 deletions(-) diff --git a/web/src/core/usecases/softwareDetails/state.ts b/web/src/core/usecases/softwareDetails/state.ts index 7fc57ec1..f8a03d79 100644 --- a/web/src/core/usecases/softwareDetails/state.ts +++ b/web/src/core/usecases/softwareDetails/state.ts @@ -102,6 +102,7 @@ export namespace State { referencePublications?: ApiTypes.SILL.ScholarlyArticle[]; softwareType: ApiTypes.SoftwareType; identifiers: ApiTypes.SILL.Identification[]; + repoMetadata?: ApiTypes.SILL.RepoMetadata; }; } diff --git a/web/src/core/usecases/softwareDetails/thunks.ts b/web/src/core/usecases/softwareDetails/thunks.ts index 66d023a0..527b0853 100644 --- a/web/src/core/usecases/softwareDetails/thunks.ts +++ b/web/src/core/usecases/softwareDetails/thunks.ts @@ -196,7 +196,8 @@ function apiSoftwareToSoftware(params: { keywords, referencePublications, applicationCategories, - identifiers + identifiers, + repoMetadata } = apiSoftware; const { resolveLocalizedString } = createResolveLocalizedString({ @@ -337,6 +338,7 @@ function apiSoftwareToSoftware(params: { applicationCategories, referencePublications, softwareType, - identifiers: identifiers ?? [] + identifiers: identifiers ?? [], + repoMetadata }; } diff --git a/web/src/ui/i18n/sill_en.json b/web/src/ui/i18n/sill_en.json index 523c9415..a9df84f6 100644 --- a/web/src/ui/i18n/sill_en.json +++ b/web/src/ui/i18n/sill_en.json @@ -267,7 +267,11 @@ "softwareType-desktop/mobile": "Software / mobile app", "softwareType-stack": "Stack", "softwareType-cloud": "Cloud Hosted App", - "supportedOS": "Supported OS" + "supportedOS": "Supported OS", + "repoMetadata": "Medata From Repository", + "repoLastCommit": "Last Commit", + "repoLastClosedIssuePullRequest": "Last Merged Request", + "repoLastClosedIssue": "Last Close Issue" }, "referencedInstancesTab": { "publicInstanceCount": "{{instanceCount}} maintained public $t(referencedInstancesTab.instance, {\"count\": {{instanceCount}} }) by {{organizationCount}} public $t(referencedInstancesTab.organization, {\"count\": {{organizationCount}} })", diff --git a/web/src/ui/i18n/sill_fr.json b/web/src/ui/i18n/sill_fr.json index f0f5abc7..7c4758da 100644 --- a/web/src/ui/i18n/sill_fr.json +++ b/web/src/ui/i18n/sill_fr.json @@ -270,7 +270,11 @@ "softwareType-desktop/mobile": "Logiciel Ordinateur / Application mobile", "softwareType-stack": "Stack", "softwareType-cloud": "Application cloud", - "supportedOS": "Système d'exploitation supporté" + "supportedOS": "Système d'exploitation supporté", + "repoMetadata": "Meta données de la forge logicielle", + "repoLastCommit": "Denier Commit", + "repoLastClosedIssuePullRequest": "Dernier merge", + "repoLastClosedIssue": "Dernière issue fermé" }, "referencedInstancesTab": { "publicInstanceCount": "{{instanceCount}} $t(referencedInstancesTab.instance, {\"count\": {{instanceCount}} }) web $t(referencedInstancesTab.maintain, {\"count\": {{instanceCount}} }) par {{organizationCount}} $t(referencedInstancesTab.organization, {\"count\": {{organizationCount}} }) $t(referencedInstancesTab.public, {\"count\": {{organizationCount}} }", diff --git a/web/src/ui/pages/softwareDetails/PreviewTab.tsx b/web/src/ui/pages/softwareDetails/PreviewTab.tsx index 29a30607..d8801784 100644 --- a/web/src/ui/pages/softwareDetails/PreviewTab.tsx +++ b/web/src/ui/pages/softwareDetails/PreviewTab.tsx @@ -2,15 +2,16 @@ import { useLang } from "ui/i18n"; import { Trans, useTranslation } from "react-i18next"; import { fr } from "@codegouvfr/react-dsfr"; import { tss } from "tss-react"; -import { shortEndMonthDate, monthDate } from "ui/useMoment"; +import { shortEndMonthDate, monthDate, useFormattedDate } from "ui/useMoment"; import Tooltip from "@mui/material/Tooltip"; import { capitalize } from "tsafe/capitalize"; import { CnllServiceProviderModal } from "./CnllServiceProviderModal"; import { assert, type Equals } from "tsafe/assert"; import config from "../../config-ui.json"; -import { SILL, SoftwareType } from "api/dist/src/lib/ApiTypes"; +import type { ApiTypes } from "api"; import { SoftwareTypeTable } from "ui/shared/SoftwareTypeTable"; import { LogoURLButton } from "ui/shared/LogoURLButton"; +import { Chip } from "@mui/material"; //TODO: Do not use optional props (?) use ( | undefined ) instead // so we are sure that we don't forget to provide some props @@ -39,8 +40,9 @@ export type Props = { programmingLanguages: string[]; keywords?: string[]; applicationCategories: string[]; - softwareType: SoftwareType; - identifiers: SILL.Identification[]; + softwareType: ApiTypes.SoftwareType; + identifiers: ApiTypes.SILL.Identification[]; + repoMetadata?: ApiTypes.SILL.RepoMetadata; }; export const PreviewTab = (props: Props) => { const { @@ -64,7 +66,8 @@ export const PreviewTab = (props: Props) => { keywords, applicationCategories, softwareType, - identifiers + identifiers, + repoMetadata } = props; const { classes, cx } = useStyles(); @@ -72,6 +75,13 @@ export const PreviewTab = (props: Props) => { const { t } = useTranslation(); const { lang } = useLang(); + const scoreToLabel = (score: number) => { + if (score < 0.1) return "error"; + if (score < 0.34) return "warning"; + if (score < 0.67) return "info"; + return "success"; + }; + return ( <>
@@ -387,6 +397,64 @@ export const PreviewTab = (props: Props) => { )} )} + {repoMetadata && ( +
+

+ {t("previewTab.repoMetadata")} + {repoMetadata?.healthCheck?.score && ( + + )} +

+ {repoMetadata?.healthCheck?.lastClosedIssue && ( +

+ + {t("previewTab.repoLastClosedIssue")} :{" "} + + + {useFormattedDate({ + time: repoMetadata.healthCheck.lastClosedIssue + })} + +

+ )} + {repoMetadata?.healthCheck?.lastClosedIssuePullRequest && ( +

+ + {t("previewTab.repoLastClosedIssuePullRequest")}{" "} + :{" "} + + + {useFormattedDate({ + time: repoMetadata.healthCheck + .lastClosedIssuePullRequest + })} + +

+ )} + {repoMetadata?.healthCheck?.lastCommit && ( +

+ + {t("previewTab.repoLastCommit")} :{" "} + + + {useFormattedDate({ + time: repoMetadata.healthCheck.lastCommit + })} + +

+ )} +
+ )}
) }, From 273f7b96fabf3cd74efe4ecbd04dd9b6d580e9f5 Mon Sep 17 00:00:00 2001 From: Guillaume Bourdat Date: Thu, 6 Mar 2025 16:35:50 +0100 Subject: [PATCH 4/6] #291 : better name & clean console --- api/src/core/adapters/GitLab/api/project.ts | 2 +- .../hal/getHalSoftwareExternalData.ts | 20 ++++++++++--------- api/src/tools/repoAnalyser.ts | 3 --- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/api/src/core/adapters/GitLab/api/project.ts b/api/src/core/adapters/GitLab/api/project.ts index a1cff1f4..a281253b 100644 --- a/api/src/core/adapters/GitLab/api/project.ts +++ b/api/src/core/adapters/GitLab/api/project.ts @@ -37,7 +37,7 @@ const getLastMergeRequest = async (projectUrl: string) => { return getApiCallTakeFirst(`${projectUrl}/merge_requests?state=closed&sort=desc`); }; -export const projectEndpointMaker = (repoUrl: string | URL) => { +export const projectGitLabApiMaker = (repoUrl: string | URL) => { const apiProjectEndpoint = repoUrlToAPIUrl(repoUrl); return { diff --git a/api/src/core/adapters/hal/getHalSoftwareExternalData.ts b/api/src/core/adapters/hal/getHalSoftwareExternalData.ts index 268994f9..c4fde1d7 100644 --- a/api/src/core/adapters/hal/getHalSoftwareExternalData.ts +++ b/api/src/core/adapters/hal/getHalSoftwareExternalData.ts @@ -6,7 +6,7 @@ import { HalFetchError } from "./HalAPI/type"; import { SILL } from "../../../types/SILL"; import { HAL } from "./types/HAL"; import { repoAnalyser, RepoType } from "../../../tools/repoAnalyser"; -import { projectEndpointMaker } from "../GitLab/api/project"; +import { projectGitLabApiMaker } from "../GitLab/api/project"; const buildParentOrganizationTree = async ( structureIdArray: number[] | string[] | undefined @@ -222,17 +222,19 @@ export const getHalSoftwareExternalData: GetSoftwareExternalData = memoize( const getRepoMetadata = async (repoType: RepoType | undefined) => { switch (repoType) { case "GitLab": - const apiProject = projectEndpointMaker(halRawSoftware?.softCodeRepository_s?.[0]); - const lastCommit = await apiProject.commits.getLastCommit(); - const lastIssue = await apiProject.issues.getLastClosedIssue(); - const lastMergeRequest = await apiProject.mergeRequests.getLast(); + const gitLabProjectapi = projectGitLabApiMaker(halRawSoftware?.softCodeRepository_s?.[0]); + const lastGLCommit = await gitLabProjectapi.commits.getLastCommit(); + const lastFLIssue = await gitLabProjectapi.issues.getLastClosedIssue(); + const lastGLMergeRequest = await gitLabProjectapi.mergeRequests.getLast(); return { healthCheck: { - lastCommit: lastCommit ? new Date(lastCommit.created_at).valueOf() : undefined, + lastCommit: lastGLCommit ? new Date(lastGLCommit.created_at).valueOf() : undefined, lastClosedIssue: - lastIssue && lastIssue.closed_at ? new Date(lastIssue.closed_at).valueOf() : undefined, - lastClosedIssuePullRequest: lastMergeRequest - ? new Date(lastMergeRequest.updated_at).valueOf() + lastFLIssue && lastFLIssue.closed_at + ? new Date(lastFLIssue.closed_at).valueOf() + : undefined, + lastClosedIssuePullRequest: lastGLMergeRequest + ? new Date(lastGLMergeRequest.updated_at).valueOf() : undefined } }; diff --git a/api/src/tools/repoAnalyser.ts b/api/src/tools/repoAnalyser.ts index d451b007..de2d6b6a 100644 --- a/api/src/tools/repoAnalyser.ts +++ b/api/src/tools/repoAnalyser.ts @@ -9,9 +9,6 @@ export const repoAnalyser = async (url: string | URL | undefined): Promise Date: Thu, 6 Mar 2025 16:38:22 +0100 Subject: [PATCH 5/6] #292 : implement GitHub calls for repo metadata --- api/src/core/adapters/GitHub/api/repo.ts | 78 +++++++++++++++++++ .../hal/getHalSoftwareExternalData.ts | 26 ++++++- 2 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 api/src/core/adapters/GitHub/api/repo.ts diff --git a/api/src/core/adapters/GitHub/api/repo.ts b/api/src/core/adapters/GitHub/api/repo.ts new file mode 100644 index 00000000..09fba9ab --- /dev/null +++ b/api/src/core/adapters/GitHub/api/repo.ts @@ -0,0 +1,78 @@ +import { Octokit } from "@octokit/rest"; + +import { env } from "../../../../env"; + +export const repoGitHubEndpointMaker = (repoUrl: string | URL) => { + const octokit = new Octokit({ + auth: env.githubPersonalAccessTokenForApiRateLimit + }); + let repoUrlObj = typeof repoUrl === "string" ? URL.parse(repoUrl) : repoUrl; + if (!repoUrlObj) return undefined; + + // Case .git at the end + if (repoUrlObj.pathname.endsWith("/")) repoUrlObj.pathname = repoUrlObj.pathname.slice(0, -1); + if (repoUrlObj.pathname.endsWith(".git")) repoUrlObj.pathname = repoUrlObj.pathname.slice(0, -4); + + const parsed = repoUrlObj.pathname.split("/").filter(text => text); + + const repo = parsed[1]; + const owner = parsed[0]; + + return { + issues: { + getLastClosedIssue: async () => { + try { + const resIssues = await octokit.request("GET /repos/{owner}/{repo}/issues", { + owner, + repo, + headers: { + "X-GitHub-Api-Version": "2022-11-28" + }, + direction: "desc", + state: "closed" + }); + + return resIssues.data[0]; + } catch (error) { + return undefined; + } + } + }, + commits: { + getLastCommit: async () => { + try { + const resCommit = await octokit.request("GET /repos/{owner}/{repo}/commits", { + owner, + repo, + headers: { + "X-GitHub-Api-Version": "2022-11-28" + }, + direction: "desc" + }); + return resCommit.data[0]; + } catch (error) { + return undefined; + } + } + }, + mergeRequests: { + getLast: async () => { + try { + const resPull = await octokit.request("GET /repos/{owner}/{repo}/pulls", { + owner, + repo, + headers: { + "X-GitHub-Api-Version": "2022-11-28" + }, + direction: "desc", + state: "closed" + }); + + return resPull.data[0]; + } catch (error) { + return undefined; + } + } + } + }; +}; diff --git a/api/src/core/adapters/hal/getHalSoftwareExternalData.ts b/api/src/core/adapters/hal/getHalSoftwareExternalData.ts index c4fde1d7..eaebd34e 100644 --- a/api/src/core/adapters/hal/getHalSoftwareExternalData.ts +++ b/api/src/core/adapters/hal/getHalSoftwareExternalData.ts @@ -7,6 +7,7 @@ import { SILL } from "../../../types/SILL"; import { HAL } from "./types/HAL"; import { repoAnalyser, RepoType } from "../../../tools/repoAnalyser"; import { projectGitLabApiMaker } from "../GitLab/api/project"; +import { repoGitHubEndpointMaker } from "../GitHub/api/repo"; const buildParentOrganizationTree = async ( structureIdArray: number[] | string[] | undefined @@ -239,7 +240,30 @@ export const getHalSoftwareExternalData: GetSoftwareExternalData = memoize( } }; case "GitHub": - return undefined; + const gitHubApi = repoGitHubEndpointMaker(halRawSoftware?.softCodeRepository_s?.[0]); + if (!gitHubApi) { + console.error("Bad URL string"); + return undefined; + } + + const lastGHCommit = await gitHubApi.commits.getLastCommit(); + const lastGHCloseIssue = await gitHubApi.issues.getLastClosedIssue(); + const lastGHClosedPull = await gitHubApi.mergeRequests.getLast(); + + return { + healthCheck: { + lastCommit: lastGHCommit?.commit?.author?.date + ? new Date(lastGHCommit.commit.author.date).valueOf() + : undefined, + lastClosedIssue: lastGHCloseIssue?.closed_at + ? new Date(lastGHCloseIssue.closed_at).valueOf() + : undefined, + lastClosedIssuePullRequest: lastGHClosedPull?.closed_at + ? new Date(lastGHClosedPull.closed_at).valueOf() + : undefined + } + }; + case undefined: return undefined; default: From c49fabff0616ed06ba15ffed47894ecff207aefa Mon Sep 17 00:00:00 2001 From: Guillaume Bourdat Date: Fri, 7 Mar 2025 21:34:51 +0100 Subject: [PATCH 6/6] Fix test --- api/src/core/adapters/fetchExternalData.test.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/api/src/core/adapters/fetchExternalData.test.ts b/api/src/core/adapters/fetchExternalData.test.ts index 4ecc3886..4b879506 100644 --- a/api/src/core/adapters/fetchExternalData.test.ts +++ b/api/src/core/adapters/fetchExternalData.test.ts @@ -307,11 +307,6 @@ describe("fetches software extra data (from different providers)", () => { name: "POLLEN ROBOTICS", siren: "820266211" }, - { - url: "https://annuaire.cnll.fr/societes/437827959", - name: "ézéo", - siren: "437827959" - }, { url: "https://annuaire.cnll.fr/societes/483494589", name: "CENTREON",