diff --git a/lib/datasource/api.ts b/lib/datasource/api.ts index 33631ce7be1537..3e45c789a1cdd3 100644 --- a/lib/datasource/api.ts +++ b/lib/datasource/api.ts @@ -1,4 +1,5 @@ import { AdoptiumJavaDatasource } from './adoptium-java'; +import { ArtifactoryDatasource } from './artifactory'; import { BitBucketTagsDatasource } from './bitbucket-tags'; import { CdnJsDatasource } from './cdnjs'; import { ClojureDatasource } from './clojure'; @@ -40,6 +41,7 @@ const api = new Map(); export default api; api.set(AdoptiumJavaDatasource.id, new AdoptiumJavaDatasource()); +api.set(ArtifactoryDatasource.id, new ArtifactoryDatasource()); api.set('bitbucket-tags', new BitBucketTagsDatasource()); api.set('cdnjs', new CdnJsDatasource()); api.set('clojure', new ClojureDatasource()); diff --git a/lib/datasource/artifactory/__fixtures__/releases-as-files.html b/lib/datasource/artifactory/__fixtures__/releases-as-files.html new file mode 100644 index 00000000000000..2bdde583999c96 --- /dev/null +++ b/lib/datasource/artifactory/__fixtures__/releases-as-files.html @@ -0,0 +1,21 @@ + + + + + Repository Title + + + +

Index

+ 
Name      Last modified      Size
+ 
+         ..
+         1.0.0  21-Jul-2021 20:08    -
+         1.0.1  23-Aug-2021 20:03    -
+         1.0.2  21-Jul-2021 20:09    -
+         1.0.3  06-Feb-2021 09:54    -
+
+
+
Artifactory Port 8080
+ + diff --git a/lib/datasource/artifactory/__fixtures__/releases-as-folders.html b/lib/datasource/artifactory/__fixtures__/releases-as-folders.html new file mode 100644 index 00000000000000..16b86d87d66824 --- /dev/null +++ b/lib/datasource/artifactory/__fixtures__/releases-as-folders.html @@ -0,0 +1,21 @@ + + + + + Repository Title + + + +

Index

+ 
Name      Last modified      Size
+ 
+         ../
+         1.0.0/  21-Jul-2021 20:08    -
+         1.0.1/  23-Aug-2021 20:03    -
+         1.0.2/  21-Jul-2021 20:09    -
+         1.0.3/  06-Feb-2021 09:54    -
+
+
+
Artifactory Port 8080
+ + diff --git a/lib/datasource/artifactory/__snapshots__/index.spec.ts.snap b/lib/datasource/artifactory/__snapshots__/index.spec.ts.snap new file mode 100644 index 00000000000000..9c3eaf66a2c7e2 --- /dev/null +++ b/lib/datasource/artifactory/__snapshots__/index.spec.ts.snap @@ -0,0 +1,80 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`datasource/artifactory/index getReleases parses real data (files): without slash at the end 1`] = ` +Object { + "registryUrl": "https://jfrog.company.com/artifactory", + "releases": Array [ + Object { + "releaseTimestamp": "2021-07-21T20:08:00.000Z", + "version": "1.0.0", + }, + Object { + "releaseTimestamp": "2021-08-23T20:03:00.000Z", + "version": "1.0.1", + }, + Object { + "releaseTimestamp": "2021-07-21T20:09:00.000Z", + "version": "1.0.2", + }, + Object { + "releaseTimestamp": "2021-02-06T09:54:00.000Z", + "version": "1.0.3", + }, + ], +} +`; + +exports[`datasource/artifactory/index getReleases parses real data (folders): with slash at the end 1`] = ` +Object { + "registryUrl": "https://jfrog.company.com/artifactory", + "releases": Array [ + Object { + "releaseTimestamp": "2021-07-21T20:08:00.000Z", + "version": "1.0.0", + }, + Object { + "releaseTimestamp": "2021-08-23T20:03:00.000Z", + "version": "1.0.1", + }, + Object { + "releaseTimestamp": "2021-07-21T20:09:00.000Z", + "version": "1.0.2", + }, + Object { + "releaseTimestamp": "2021-02-06T09:54:00.000Z", + "version": "1.0.3", + }, + ], +} +`; + +exports[`datasource/artifactory/index getReleases parses real data (merge strategy with 2 registries) 1`] = ` +Object { + "releases": Array [ + Object { + "registryUrl": "https://jfrog.company.com/artifactory", + "releaseTimestamp": "2021-07-21T20:08:00.000Z", + "version": "1.0.0", + }, + Object { + "registryUrl": "https://jfrog.company.com/artifactory", + "releaseTimestamp": "2021-08-23T20:03:00.000Z", + "version": "1.0.1", + }, + Object { + "registryUrl": "https://jfrog.company.com/artifactory", + "releaseTimestamp": "2021-07-21T20:09:00.000Z", + "version": "1.0.2", + }, + Object { + "registryUrl": "https://jfrog.company.com/artifactory", + "releaseTimestamp": "2021-02-06T09:54:00.000Z", + "version": "1.0.3", + }, + Object { + "registryUrl": "https://jfrog.company.com/artifactory/production", + "version": "1.3.0", + }, + ], +} +`; diff --git a/lib/datasource/artifactory/common.ts b/lib/datasource/artifactory/common.ts new file mode 100644 index 00000000000000..7dd522f5075038 --- /dev/null +++ b/lib/datasource/artifactory/common.ts @@ -0,0 +1 @@ +export const datasource = 'artifactory'; diff --git a/lib/datasource/artifactory/index.spec.ts b/lib/datasource/artifactory/index.spec.ts new file mode 100644 index 00000000000000..ff1611903062c2 --- /dev/null +++ b/lib/datasource/artifactory/index.spec.ts @@ -0,0 +1,156 @@ +import { getPkgReleases } from '..'; +import * as httpMock from '../../../test/http-mock'; +import { loadFixture } from '../../../test/util'; +import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; +import { logger } from '../../logger'; +import { joinUrlParts } from '../../util/url'; +import { ArtifactoryDatasource } from '.'; + +const datasource = ArtifactoryDatasource.id; + +const testRegistryUrl = 'https://jfrog.company.com/artifactory'; +const testLookupName = 'project'; +const testConfig = { + registryUrls: [testRegistryUrl], + depName: testLookupName, +}; +const fixtureReleasesAsFolders = loadFixture('releases-as-folders.html'); +const fixtureReleasesAsFiles = loadFixture('releases-as-files.html'); + +function getPath(folder: string): string { + return `/${folder}`; +} + +describe('datasource/artifactory/index', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + describe('getReleases', () => { + it('parses real data (folders): with slash at the end', async () => { + httpMock + .scope(testRegistryUrl) + .get(getPath(testLookupName)) + .reply(200, fixtureReleasesAsFolders); + const res = await getPkgReleases({ + ...testConfig, + datasource, + lookupName: testLookupName, + }); + expect(res.releases).toHaveLength(4); + expect(res).toMatchSnapshot({ + registryUrl: 'https://jfrog.company.com/artifactory', + }); + }); + + it('parses real data (files): without slash at the end', async () => { + httpMock + .scope(testRegistryUrl) + .get(getPath(testLookupName)) + .reply(200, fixtureReleasesAsFiles); + const res = await getPkgReleases({ + ...testConfig, + datasource, + lookupName: testLookupName, + }); + expect(res.releases).toHaveLength(4); + expect(res).toMatchSnapshot({ + registryUrl: 'https://jfrog.company.com/artifactory', + }); + }); + + it('parses real data (merge strategy with 2 registries)', async () => { + const secondRegistryUrl: string = joinUrlParts( + testRegistryUrl, + 'production' + ); + httpMock + .scope(testRegistryUrl) + .get(getPath(testLookupName)) + .reply(200, fixtureReleasesAsFiles); + httpMock + .scope(secondRegistryUrl) + .get(getPath(testLookupName)) + .reply(200, '\n

Header

\n1.3.0\n'); + const res = await getPkgReleases({ + registryUrls: [testRegistryUrl, secondRegistryUrl], + depName: testLookupName, + datasource, + lookupName: testLookupName, + }); + expect(res.releases).toHaveLength(5); + expect(res).toMatchSnapshot(); + }); + + it('returns null without registryUrl + warning', async () => { + const res = await getPkgReleases({ + datasource, + depName: testLookupName, + lookupName: testLookupName, + }); + expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledWith( + { lookupName: 'project' }, + 'artifactory datasource requires custom registryUrl. Skipping datasource' + ); + expect(res).toBeNull(); + }); + + it('returns null for empty 200 OK', async () => { + httpMock + .scope(testRegistryUrl) + .get(getPath(testLookupName)) + .reply(200, '\n

Header wo. nodes

\n'); + expect( + await getPkgReleases({ + ...testConfig, + datasource, + lookupName: testLookupName, + }) + ).toBeNull(); + }); + + it('404 returns null', async () => { + httpMock.scope(testRegistryUrl).get(getPath(testLookupName)).reply(404); + expect( + await getPkgReleases({ + ...testConfig, + datasource, + lookupName: testLookupName, + }) + ).toBeNull(); + expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledWith( + { + lookupName: 'project', + registryUrl: 'https://jfrog.company.com/artifactory', + }, + 'artifactory: `Not Found` error' + ); + }); + + it('throws for error diff than 404', async () => { + httpMock.scope(testRegistryUrl).get(getPath(testLookupName)).reply(502); + await expect( + getPkgReleases({ + ...testConfig, + datasource, + lookupName: testLookupName, + }) + ).rejects.toThrow(EXTERNAL_HOST_ERROR); + }); + + it('throws no Http error', async () => { + httpMock + .scope(testRegistryUrl) + .get(getPath(testLookupName)) + .replyWithError('unknown error'); + const res = await getPkgReleases({ + ...testConfig, + datasource, + lookupName: testLookupName, + }); + expect(res).toBeNull(); + }); + }); +}); diff --git a/lib/datasource/artifactory/index.ts b/lib/datasource/artifactory/index.ts new file mode 100644 index 00000000000000..2c462be9c9c3d6 --- /dev/null +++ b/lib/datasource/artifactory/index.ts @@ -0,0 +1,113 @@ +import { logger } from '../../logger'; +import { cache } from '../../util/cache/package/decorator'; +import { parse } from '../../util/html'; +import { HttpError } from '../../util/http/types'; +import { joinUrlParts } from '../../util/url'; +import { Datasource } from '../datasource'; +import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; +import { datasource } from './common'; + +export class ArtifactoryDatasource extends Datasource { + static readonly id = datasource; + + constructor() { + super(datasource); + } + + override readonly customRegistrySupport = true; + + override readonly caching = true; + + override readonly registryStrategy = 'merge'; + + @cache({ + namespace: `datasource-${datasource}`, + key: ({ registryUrl, lookupName }: GetReleasesConfig) => + `${registryUrl}:${lookupName}`, + }) + async getReleases({ + lookupName, + registryUrl, + }: GetReleasesConfig): Promise { + if (!registryUrl) { + logger.warn( + { lookupName }, + 'artifactory datasource requires custom registryUrl. Skipping datasource' + ); + return null; + } + + const url = joinUrlParts(registryUrl, lookupName); + + const result: ReleaseResult = { + releases: [], + }; + try { + const response = await this.http.get(url); + const body = parse(response.body, { + blockTextElements: { + script: true, + noscript: true, + style: true, + }, + }); + const nodes = body.querySelectorAll('a'); + + nodes + .filter( + // filter out hyperlink to navigate to parent folder + (node) => node.innerHTML !== '../' && node.innerHTML !== '..' + ) + .forEach( + // extract version and published time for each node + (node) => { + const version: string = + node.innerHTML.slice(-1) === '/' + ? node.innerHTML.slice(0, -1) + : node.innerHTML; + + const published = ArtifactoryDatasource.parseReleaseTimestamp( + node.nextSibling?.text + ); + + const thisRelease: Release = { + version, + releaseTimestamp: published, + }; + + result.releases.push(thisRelease); + } + ); + + if (result.releases.length) { + logger.trace( + { registryUrl, lookupName, versions: result.releases.length }, + 'artifactory: Found versions' + ); + } else { + logger.trace( + { registryUrl, lookupName }, + 'artifactory: No versions found' + ); + } + } catch (err) { + // istanbul ignore else: not testable with nock + if (err instanceof HttpError) { + if (err.response?.statusCode === 404) { + logger.warn( + { registryUrl, lookupName }, + 'artifactory: `Not Found` error' + ); + return null; + } + } + this.handleGenericErrors(err); + } + + return result.releases.length ? result : null; + } + + private static parseReleaseTimestamp(rawText: string): string { + return rawText.trim().replace(/ ?-$/, ''); + } +} diff --git a/lib/datasource/artifactory/readme.md b/lib/datasource/artifactory/readme.md new file mode 100644 index 00000000000000..e8e40c6fe46d76 --- /dev/null +++ b/lib/datasource/artifactory/readme.md @@ -0,0 +1,5 @@ +Artifactory is the recommended registry for Conan packages. + +This datasource returns releases from given custom `registryUrl`(s). + +The target URL is composed by the `registryUrl` and the `lookupName`, which defaults to `depName` when `lookupName` is not defined. diff --git a/lib/util/html.spec.ts b/lib/util/html.spec.ts index c4727d82341aaf..a78a70ae965554 100644 --- a/lib/util/html.spec.ts +++ b/lib/util/html.spec.ts @@ -13,4 +13,22 @@ describe('util/html', () => { const body = parse(''); expect(body.childNodes).toHaveLength(0); }); + + it('parses HTML: PRE block hides child nodes', () => { + const body = parse('
Hello, world!
\n
node A
'); + const childNodesA = body.querySelectorAll('a'); + expect(childNodesA).toHaveLength(0); + }); + + it('parses HTML: use additional options to discover child nodes on PRE blocks', () => { + const body = parse('
Hello, world!
\n
node A
', { + blockTextElements: {}, + }); + const childNodesA = body.querySelectorAll('a'); + expect(childNodesA).toHaveLength(1); + const div = childNodesA[0]; + expect(div.tagName).toBe('A'); + expect(div.textContent).toBe('node A'); + expect(div instanceof HTMLElement).toBe(true); + }); }); diff --git a/lib/util/html.ts b/lib/util/html.ts index 26ac55583333dd..d0ee29fd8b630e 100644 --- a/lib/util/html.ts +++ b/lib/util/html.ts @@ -1,7 +1,11 @@ -import { HTMLElement, parse as _parse } from 'node-html-parser'; +import { HTMLElement, Options, parse as _parse } from 'node-html-parser'; export { HTMLElement }; -export function parse(html: string): HTMLElement { +export function parse(html: string, config?: Partial): HTMLElement { + if (typeof config !== 'undefined') { + return _parse(html, config); + } + return _parse(html); }