diff --git a/package-lock.json b/package-lock.json index f1fa5ad..5394381 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,8 @@ "execa": "^5.1.1", "https-proxy-agent": "^7.0.4", "node-fetch": "^2.7.0", - "tar-fs": "^3.0.5" + "tar-fs": "^3.0.5", + "yauzl": "^3.1.2" }, "devDependencies": { "@types/debug": "^4.1.12", @@ -23,6 +24,7 @@ "@types/node-fetch": "^2.6.11", "@types/tar-fs": "^2.0.4", "@types/uuid": "^9.0.7", + "@types/yauzl": "^2.10.3", "@typescript-eslint/eslint-plugin": "^6.19.1", "@typescript-eslint/parser": "^6.19.1", "eslint": "^8.56.0", @@ -1507,6 +1509,15 @@ "integrity": "sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==", "dev": true }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "6.19.1", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.19.1.tgz", @@ -2070,6 +2081,14 @@ "node-int64": "^0.4.0" } }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "engines": { + "node": "*" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -4241,6 +4260,11 @@ "node": ">=8" } }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" + }, "node_modules/picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", @@ -5191,6 +5215,18 @@ "node": ">=12" } }, + "node_modules/yauzl": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.1.2.tgz", + "integrity": "sha512-621iCPgEG1wXViDZS/L3h9F8TgrdQV1eayJlJ8j5A2SZg8OdY/1DLf+VxNeD+q5QbMFEAbjjR8nITj7g4nKa0Q==", + "dependencies": { + "buffer-crc32": "~0.2.3", + "pend": "~1.2.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index cf9d47d..1a62bba 100644 --- a/package.json +++ b/package.json @@ -47,9 +47,11 @@ "execa": "^5.1.1", "https-proxy-agent": "^7.0.4", "node-fetch": "^2.7.0", - "tar-fs": "^3.0.5" + "tar-fs": "^3.0.5", + "yauzl": "^3.1.2" }, "devDependencies": { + "@types/yauzl": "^2.10.3", "@types/debug": "^4.1.12", "@types/gunzip-maybe": "^1.4.2", "@types/jest": "^29.5.11", diff --git a/src/constants.ts b/src/constants.ts index aa2aaae..ac2ab33 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,11 +1 @@ -export enum Artifacts { - ES = 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch', - OS = 'https://artifacts.opensearch.org/releases/bundle/opensearch', - ZINC = 'https://github.com/zincsearch/zincsearch/releases/download', -} - -export enum EngineType { - ZINCSEARCH = 'zincsearch', - ELASTICSEARCH = 'elasticsearch', - OPENSEARCH = 'opensearch', -} +export enum Artifacts { ES = 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch', OS = 'https://artifacts.opensearch.org/releases/bundle/opensearch', ZINC = 'https://github.com/zincsearch/zincsearch/releases/download', } export enum EngineType { ZINCSEARCH = 'zincsearch', ELASTICSEARCH = 'elasticsearch', OPENSEARCH = 'opensearch', } \ No newline at end of file diff --git a/src/utils.ts b/src/utils.ts index 630d9ee..d8878d1 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,5 @@ import * as fs from 'fs'; -import { access, constants } from 'fs'; +import { constants } from 'fs'; import { promisify } from 'util'; import { debug } from './debug'; import { Artifacts, EngineType } from './constants'; @@ -10,6 +10,7 @@ import * as os from 'os'; import * as zlib from 'zlib'; import { extract } from 'tar-fs'; import { pipeline } from 'node:stream'; +import * as yauzl from 'yauzl'; export const waitForLocalhost = async (engineClient: EngineClient, retries = 30) => { await new Promise((resolve) => setTimeout(() => resolve(0), 2000)); @@ -24,11 +25,9 @@ export const waitForLocalhost = async (engineClient: EngineClient, retries = 30) } }; -export const isFileExists = async (path: string): Promise => { - const fsAccessPromisified = promisify(access); - +export const isFileExists = (path: string): boolean => { try { - await fsAccessPromisified(path, constants.F_OK); + fs.accessSync(path, constants.F_OK); return true; } catch (e) { return false; @@ -42,18 +41,45 @@ const platform = () => { return { sysName: sysName.toLowerCase(), arch: arch.toLowerCase() }; }; +const tryRecursiveDir = (filepath: string) => { + if (!isFileExists(filepath)) { + fs.mkdirSync(filepath, { recursive: true, mode: 0o775 }); + } +}; + const pipelineAsync = promisify(pipeline); +const isZipFile = (filePath: string): boolean => { + const buffer = Buffer.alloc(2); + try { + fs.openSync(filePath, 'r'); + fs.readSync(fs.openSync(filePath, 'r'), buffer, 0, 2, 0); + return buffer.toString('hex') === '504b'; + } catch (error) { + debug(`Error checking file signature: ${error}`); + return false; + } +}; + +const unGzip = async (readPath: string, writePath: string) => { + // Pipe the response body to the decompression stream and then to the extract function + await pipelineAsync( + fs.createReadStream(readPath), + // decompressStream, + zlib.createGunzip(), + extract(writePath, { dmode: 0o775, fmode: 0o775 }), + ); +}; export const download = async (url: string, dir: string, engine: EngineType, version: string) => { const binaryPath = `${dir}/${engine}-${version}`; const writePath = engine === EngineType.ZINCSEARCH ? `${binaryPath}` : `${dir}`; debug(`checking if binary exists: ${binaryPath}`); - if (await isFileExists(`${binaryPath}`)) { + if (isFileExists(`${binaryPath}`)) { debug(`binary already downloaded`); return binaryPath; } else { - fs.mkdirSync(dir, { recursive: true, mode: 0o775 }); + tryRecursiveDir(dir); } debug(`downloading binary, url: ${url}, path: ${binaryPath}`); @@ -62,27 +88,30 @@ export const download = async (url: string, dir: string, engine: EngineType, ver : undefined; try { const res = await fetch(url, { agent: proxyAgent }); - + if (!res.ok) throw new Error(await res.text()); const contentType = res.headers.get('content-type') || ''; debug(`content-type: ${contentType}`); - let decompressStream; if ( ['application/gzip', 'application/x-gzip', 'application/octet-stream'].includes(contentType) ) { - decompressStream = zlib.createGunzip(); + // Pipe the response body to the decompression stream and then to the extract function + await pipelineAsync( + res.body, + // decompressStream, + zlib.createGunzip(), + extract(writePath, { dmode: 0o775, fmode: 0o775 }), + ); } else if (contentType === 'application/zip') { - decompressStream = zlib.createUnzip(); + await pipelineAsync(res.body, fs.createWriteStream(`${binaryPath}.zip`)); + if (isZipFile(`${binaryPath}.zip`)) { + await downloadZip(writePath); + } else { + await unGzip(`${binaryPath}.zip`, writePath); + } } else { debug(`Unsupported content type: ${contentType}`); process.exit(-1); } - - // Pipe the response body to the decompression stream and then to the extract function - await pipelineAsync( - res.body, - decompressStream, - extract(writePath, { dmode: 0o775, fmode: 0o775 }), - ); } catch (err) { debug(`error when downloading and extracting the binary file: ${err}`); process.exit(-1); @@ -90,8 +119,7 @@ export const download = async (url: string, dir: string, engine: EngineType, ver for (let i = 0; i < 5; i++) { const binaryFile = - (await isFileExists(`${binaryPath}/bin/${engine}`)) || - (await isFileExists(`${binaryPath}/${engine}`)); + isFileExists(`${binaryPath}/bin/${engine}`) || isFileExists(`${binaryPath}/${engine}`); await new Promise((resolve) => setTimeout(() => resolve(0), 2000)); if (binaryFile) { @@ -106,23 +134,69 @@ export const download = async (url: string, dir: string, engine: EngineType, ver export const getEngineBinaryURL = (engine: EngineType, version: string) => { const { sysName, arch } = platform(); + debug(`getEngineBinaryURL,sysName: ${sysName}, arch: ${arch}`); const engines: { [engineType: string]: () => string; } = { [EngineType.ELASTICSEARCH]: () => { const archName = arch === 'arm64' ? 'aarch64' : 'x86_64'; + const systemName = sysName === 'win32' ? 'windows' : sysName; + const zipFormat = systemName === 'windows' ? 'zip' : 'tar.gz'; + // https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-8.13.1-windows-x86_64.zip return parseInt(version.charAt(0)) >= 7 - ? `${Artifacts.ES}-${version}-${sysName}-${archName}.tar.gz` - : `${Artifacts.ES}-${version}.tar.gz`; + ? `${Artifacts.ES}-${version}-${systemName}-${archName}.${zipFormat}` + : `${Artifacts.ES}-${version}.${zipFormat}`; + }, + [EngineType.OPENSEARCH]: () => { + const systemName = sysName === 'win32' ? 'windows' : sysName; + const zipFormat = systemName === 'windows' ? 'zip' : 'tar.gz'; + // https://artifacts.opensearch.org/releases/bundle/opensearch/2.13.0/opensearch-2.13.0-windows-x64.zip + return `${Artifacts.OS}/${version}/opensearch-${version}-${systemName}-${arch}.${zipFormat}`; }, - [EngineType.OPENSEARCH]: () => - `${Artifacts.OS}/${version}/opensearch-${version}-linux-${arch}.tar.gz`, [EngineType.ZINCSEARCH]: () => { const archName = arch === 'x64' ? 'x86_64' : arch; - return `${Artifacts.ZINC}/v${version}/zincsearch_${version}_${sysName}_${archName}.tar.gz`; + const systemName = sysName === 'win32' ? 'windows' : sysName; + // https://github.com/zincsearch/zincsearch/releases/download/v0.4.10/zincsearch_0.4.10_windows_x86_64.tar.gz + return `${Artifacts.ZINC}/v${version}/zincsearch_${version}_${systemName}_${archName}.tar.gz`; }, }; return engines[engine](); }; + +const downloadZip = async (zipFilePath: string) => { + try { + return new Promise((resolve, reject) => { + yauzl.open(zipFilePath, { lazyEntries: true }, (err, zipfile) => { + if (err) { + debug(`error while unzip: ${zipFilePath}`); + return reject(err); + } + zipfile.readEntry(); + + zipfile.on('entry', async (entry) => { + debug(`found entry: filePath: ${entry.filePath} fileName: ${entry.fileName}`); + if (/\/$/.test(entry.fileName)) { + zipfile.readEntry(); + } else { + zipfile.openReadStream(entry, (err, readStream) => { + if (err) { + debug(`error while unzip: ${zipFilePath}`); + return reject(err); + } + readStream.on('end', zipfile.readEntry); + tryRecursiveDir(zipFilePath); + readStream.pipe(fs.createWriteStream(`${zipFilePath}/${entry.fileName}`)); + }); + } + }); + zipfile.on('close', resolve); + zipfile.on('error', reject); + }); + }); + } catch (err) { + debug(`error encountered while downloaidng & extract zip file: ${zipFilePath}, err: ${err}`); + throw err; + } +}; diff --git a/tests/engine.test.ts b/tests/engine.test.ts index d468d4b..09d6983 100644 --- a/tests/engine.test.ts +++ b/tests/engine.test.ts @@ -3,7 +3,7 @@ import { engineMartix, indexes } from './utils/fixtures'; import { diagnose, fetchMapping } from './utils/common'; describe('integration test for elasticsearch and opensearch', () => { - it(`should start engine with default config`, async () => { + it.only(`should start engine with default config`, async () => { await startEngine(); const inspect = await diagnose(EngineType.ELASTICSEARCH, 9200);