From edbf844909d2d4922bf68cba7d011b451f0f94c0 Mon Sep 17 00:00:00 2001 From: Eugene Klimov Date: Fri, 29 Nov 2024 10:20:39 +0400 Subject: [PATCH] add blobKeepAliveTimeout, queueKeepAliveTimeout, tableKeepAliveTimeout (#2454) * add blobKeepAliveTimeout, queueKeepAliveTimeout, tableKeepAliveTimeout for fix https://github.com/Azure/Azurite/issues/2053 and workaround https://github.com/ClickHouse/ClickHouse/issues/60447 * ChangeLog.md added blobKeepAliveTimeout, queueKeepAliveTimeout, tableKeepAliveTimeout * format constant imports * actualize *keepAliveTimeout description and default values after review https://github.com/Azure/Azurite/pull/2454#discussion_r1740560554 and https://github.com/Azure/Azurite/pull/2454#discussion_r1740563598 * add blobKeepAliveTimeout, queueKeepAliveTimeout, tableKeepAliveTimeout for fix https://github.com/Azure/Azurite/issues/2053 and workaround https://github.com/ClickHouse/ClickHouse/issues/60447 Signed-off-by: Slach * increase DEFAULT_*_KEEP_ALIVE_TIMEOUT=5000, it was HttpServer default value * actualize *keepAliveTimeout description and switch default values from ms to seconds after review https://github.com/Azure/Azurite/pull/2454/files#r1751590590 * actualize README for https://github.com/Azure/Azurite/pull/2454/files#r1751590590 * fix review comment https://github.com/Azure/Azurite/pull/2454#discussion_r1810136090 * add blobKeepAliveTimeout and queueKeepAliveTimeout, table test need suggestion from https://github.com/Azure/Azurite/pull/2454#discussion_r1849612823 * add tableKeepAliveTimeout * Node 16.x doesn't have keep-alive: timeout=5 header * fix review comment https://github.com/Azure/Azurite/pull/2454#discussion_r1851401426 --------- Signed-off-by: Slach --- ChangeLog.md | 9 +++ README.mcr.md | 7 ++ README.md | 11 ++- package.json | 15 ++++ src/azurite.ts | 2 + src/blob/BlobConfiguration.ts | 5 +- src/blob/BlobEnvironment.ts | 12 ++- src/blob/BlobServerFactory.ts | 2 + src/blob/IBlobEnvironment.ts | 1 + src/blob/SqlBlobConfiguration.ts | 5 +- src/blob/utils/constants.ts | 2 + src/common/ConfigurationBase.ts | 1 + src/common/Environment.ts | 30 +++++++ src/common/ServerBase.ts | 4 + src/common/VSCEnvironment.ts | 12 +++ src/common/VSCServerManagerBlob.ts | 1 + src/common/VSCServerManagerQueue.ts | 1 + src/common/VSCServerManagerTable.ts | 1 + src/queue/IQueueEnvironment.ts | 1 + src/queue/QueueConfiguration.ts | 5 +- src/queue/QueueEnvironment.ts | 4 + src/queue/QueueServer.ts | 4 + src/queue/main.ts | 1 + src/queue/utils/constants.ts | 1 + src/table/ITableEnvironment.ts | 2 + src/table/TableConfiguration.ts | 5 +- src/table/TableEnvironment.ts | 12 ++- src/table/main.ts | 1 + src/table/utils/constants.ts | 1 + tests/BlobTestServerFactory.ts | 3 + tests/blob/blobKeepAliveTimeout.test.ts | 56 +++++++++++++ tests/queue/queueKeepAliveTimeout.test.ts | 78 +++++++++++++++++++ tests/queue/utils/QueueTestServerFactory.ts | 2 + .../KeepAlive/tableKeepAliveTimeout.test.ts | 62 +++++++++++++++ tests/table/utils/TableTestServerFactory.ts | 2 + 35 files changed, 354 insertions(+), 7 deletions(-) create mode 100644 tests/blob/blobKeepAliveTimeout.test.ts create mode 100644 tests/queue/queueKeepAliveTimeout.test.ts create mode 100644 tests/table/KeepAlive/tableKeepAliveTimeout.test.ts diff --git a/ChangeLog.md b/ChangeLog.md index 7ab36027d..2d45ef3d9 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -30,6 +30,15 @@ Blob: - Fixed issue of download a blob range without header x-ms-range-get-content-md5, should not return content-md5. (issue #2409) - Fixed issue of list container without include=metadata should not clear container metadata on server. (issue #2416) - Supported x-ms-copy-source-tag-option in copy blob from Uri. (issue #2398) +- Added blobKeepAliveTimeout option (issue #2053) + +Table: + +- Added tableKeepAliveTimeout option (issue #2053) + +Queue: + +- Added queueKeepAliveTimeout option (issue #2053) ## 2024.06 Version 3.31.0 diff --git a/README.mcr.md b/README.mcr.md index f9275a60e..7dea96e80 100644 --- a/README.mcr.md +++ b/README.mcr.md @@ -80,6 +80,13 @@ Above command will try to start Azurite image with configurations: Please refer to this [document](https://github.com/Azure/Azurite/blob/master/README.md) for **More supported parameters** like HTTPS or OAuth. +**Customize HTTP Keep-Alive behavior** +To avoid TCP RST related errors in tests with SDK which not properly handle KeepAlive-Timeout response header, you could increase keep alive timeout, increase values in seconds. + +```bash +docker run azurite --blobHost 0.0.0.0 --blobKeepAliveTimeout 300 --queueHost 0.0.0.0 --queueKeepAliveTimeout 300 --tableHost 0.0.0.0 --tableKeepAliveTimeout 300 +``` + ## Documentation Please refer to this [document](https://github.com/Azure/Azurite/blob/master/README.md). diff --git a/README.md b/README.md index 48e15ccf9..cc26b4280 100644 --- a/README.md +++ b/README.md @@ -185,10 +185,13 @@ Following extension configurations are supported: - `azurite.blobHost` Blob service listening endpoint, by default 127.0.0.1 - `azurite.blobPort` Blob service listening port, by default 10000 +- `azurite.blobKeepAliveTimeout` Blob service keep alive timeout in seconds, by default 5 - `azurite.queueHost` Queue service listening endpoint, by default 127.0.0.1 - `azurite.queuePort` Queue service listening port, by default 10001 +- `azurite.queueKeepAliveTimeout` Queue service keep alive timeout in seconds, by default 5 - `azurite.tableHost` Table service listening endpoint, by default 127.0.0.1 - `azurite.tablePort` Table service listening port, by default 10002 +- `azurite.tableKeepAliveTimeout` Queue service keep alive timeout in seconds, by default 5 - `azurite.location` Workspace location folder path (can be relative or absolute). By default, in the VS Code extension, the currently opened folder is used. If launched from the command line, the current process working directory is the default. Relative paths are resolved relative to the default folder. - `azurite.silent` Silent mode to disable access log in Visual Studio channel, by default false - `azurite.debug` Output debug log into Azurite channel, by default false @@ -233,7 +236,7 @@ docker run -p 10000:10000 -p 10001:10001 -v c:/azurite:/data mcr.microsoft.com/a #### Customize all Azurite V3 supported parameters for docker image ```bash -docker run -p 7777:7777 -p 8888:8888 -p 9999:9999 -v c:/azurite:/workspace mcr.microsoft.com/azure-storage/azurite azurite -l /workspace -d /workspace/debug.log --blobPort 7777 --blobHost 0.0.0.0 --queuePort 8888 --queueHost 0.0.0.0 --tablePort 9999 --tableHost 0.0.0.0 --loose --skipApiVersionCheck --disableProductStyleUrl +docker run -p 7777:7777 -p 8888:8888 -p 9999:9999 -v c:/azurite:/workspace mcr.microsoft.com/azure-storage/azurite azurite -l /workspace -d /workspace/debug.log --blobPort 7777 --blobHost 0.0.0.0 --blobKeepAliveTimeout 5 --queuePort 8888 --queueHost 0.0.0.0 --queueKeepAliveTimeout 5 --tablePort 9999 --tableHost 0.0.0.0 --tableKeepAliveTimeout 5 --loose --skipApiVersionCheck --disableProductStyleUrl ``` Above command will try to start Azurite image with configurations: @@ -246,14 +249,20 @@ Above command will try to start Azurite image with configurations: `--blobHost 0.0.0.0` defines blob service listening endpoint to accept requests from host machine. +`--blobKeepAliveTimeout 5` blob service keep alive timeout in seconds + `--queuePort 8888` makes Azurite queue service listen to port 8888, while `-p 8888:8888` redirects requests from host machine's port 8888 to docker instance. `--queueHost 0.0.0.0` defines queue service listening endpoint to accept requests from host machine. +`--queueKeepAliveTimeout 5` queue service keep alive timeout in seconds + `--tablePort 9999` makes Azurite table service listen to port 9999, while `-p 9999:9999` redirects requests from host machine's port 9999 to docker instance. `--tableHost 0.0.0.0` defines table service listening endpoint to accept requests from host machine. +`--tableKeepAliveTimeout 5` table service keep alive timeout in seconds + `--loose` enables loose mode which ignore unsupported headers and parameters. `--skipApiVersionCheck` skip the request API version check. diff --git a/package.json b/package.json index eff12fedc..828b12ec4 100644 --- a/package.json +++ b/package.json @@ -207,6 +207,11 @@ "default": 10000, "description": "Blob service listening port, by default 10000" }, + "azurite.blobKeepAliveTimeout": { + "type": "number", + "default": 5, + "description": "Blob service keep alive timeout in seconds, by default 5" + }, "azurite.queueHost": { "type": "string", "default": "127.0.0.1", @@ -217,6 +222,11 @@ "default": 10001, "description": "Queue service listening port, by default 10001" }, + "azurite.queueKeepAliveTimeout": { + "type": "number", + "default": 5, + "description": "Queue service keep alive timeout in seconds, by default 5" + }, "azurite.tableHost": { "type": "string", "default": "127.0.0.1", @@ -227,6 +237,11 @@ "default": 10002, "description": "Table service listening port, by default 10002" }, + "azurite.tableKeepAliveTimeout": { + "type": "number", + "default": 5, + "description": "Table service keep alive timeout in seconds, by default 5" + }, "azurite.skipApiVersionCheck": { "type": "boolean", "default": false, diff --git a/src/azurite.ts b/src/azurite.ts index 7861c37c3..74250b445 100644 --- a/src/azurite.ts +++ b/src/azurite.ts @@ -86,6 +86,7 @@ async function main() { const queueConfig = new QueueConfiguration( env.queueHost(), env.queuePort(), + env.queueKeepAliveTimeout(), join(location, DEFAULT_QUEUE_LOKI_DB_PATH), join(location, DEFAULT_QUEUE_EXTENT_LOKI_DB_PATH), DEFAULT_QUEUE_PERSISTENCE_ARRAY, @@ -106,6 +107,7 @@ async function main() { const tableConfig = new TableConfiguration( env.tableHost(), env.tablePort(), + env.tableKeepAliveTimeout(), join(location, DEFAULT_TABLE_LOKI_DB_PATH), env.debug() !== undefined, !env.silent(), diff --git a/src/blob/BlobConfiguration.ts b/src/blob/BlobConfiguration.ts index 45733c0f9..b77f94a4d 100644 --- a/src/blob/BlobConfiguration.ts +++ b/src/blob/BlobConfiguration.ts @@ -8,7 +8,8 @@ import { DEFAULT_BLOB_PERSISTENCE_ARRAY, DEFAULT_BLOB_SERVER_HOST_NAME, DEFAULT_ENABLE_ACCESS_LOG, - DEFAULT_ENABLE_DEBUG_LOG + DEFAULT_ENABLE_DEBUG_LOG, + DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT } from "./utils/constants"; /** @@ -27,6 +28,7 @@ export default class BlobConfiguration extends ConfigurationBase { public constructor( host: string = DEFAULT_BLOB_SERVER_HOST_NAME, port: number = DEFAULT_BLOB_LISTENING_PORT, + keepAliveTimeout: number = DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, public readonly metadataDBPath: string = DEFAULT_BLOB_LOKI_DB_PATH, public readonly extentDBPath: string = DEFAULT_BLOB_EXTENT_LOKI_DB_PATH, public readonly persistencePathArray: StoreDestinationArray = DEFAULT_BLOB_PERSISTENCE_ARRAY, @@ -47,6 +49,7 @@ export default class BlobConfiguration extends ConfigurationBase { super( host, port, + keepAliveTimeout, enableAccessLog, accessLogWriteStream, enableDebugLog, diff --git a/src/blob/BlobEnvironment.ts b/src/blob/BlobEnvironment.ts index 65e203828..481293a8b 100644 --- a/src/blob/BlobEnvironment.ts +++ b/src/blob/BlobEnvironment.ts @@ -5,7 +5,8 @@ import { dirname } from "path"; import IBlobEnvironment from "./IBlobEnvironment"; import { DEFAULT_BLOB_LISTENING_PORT, - DEFAULT_BLOB_SERVER_HOST_NAME + DEFAULT_BLOB_SERVER_HOST_NAME, + DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT } from "./utils/constants"; if (!(args as any).config.name) { @@ -20,6 +21,11 @@ if (!(args as any).config.name) { "Optional. Customize listening port for blob", DEFAULT_BLOB_LISTENING_PORT ) + .option( + ["", "blobKeepAliveTimeout"], + "Optional. Customize http keep alive timeout for blob", + DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, + ) .option( ["l", "location"], "Optional. Use an existing folder as workspace path, default is current working directory", @@ -75,6 +81,10 @@ export default class BlobEnvironment implements IBlobEnvironment { return this.flags.blobPort; } + public blobKeepAliveTimeout(): number | undefined { + return this.flags.keepAliveTimeout; + } + public async location(): Promise { const location = this.flags.location || process.cwd(); await ensureDir(location); diff --git a/src/blob/BlobServerFactory.ts b/src/blob/BlobServerFactory.ts index a0145418d..158456476 100644 --- a/src/blob/BlobServerFactory.ts +++ b/src/blob/BlobServerFactory.ts @@ -52,6 +52,7 @@ export class BlobServerFactory { const config = new SqlBlobConfiguration( env.blobHost(), env.blobPort(), + env.blobKeepAliveTimeout(), databaseConnectionString!, DEFAULT_SQL_OPTIONS, DEFAULT_BLOB_PERSISTENCE_ARRAY, @@ -73,6 +74,7 @@ export class BlobServerFactory { const config = new BlobConfiguration( env.blobHost(), env.blobPort(), + env.blobKeepAliveTimeout(), join(location, DEFAULT_BLOB_LOKI_DB_PATH), join(location, DEFAULT_BLOB_EXTENT_LOKI_DB_PATH), DEFAULT_BLOB_PERSISTENCE_ARRAY, diff --git a/src/blob/IBlobEnvironment.ts b/src/blob/IBlobEnvironment.ts index b83cddbd3..6974c560d 100644 --- a/src/blob/IBlobEnvironment.ts +++ b/src/blob/IBlobEnvironment.ts @@ -1,6 +1,7 @@ export default interface IBlobEnvironment { blobHost(): string | undefined; blobPort(): number | undefined; + blobKeepAliveTimeout(): number | undefined; location(): Promise; silent(): boolean; loose(): boolean; diff --git a/src/blob/SqlBlobConfiguration.ts b/src/blob/SqlBlobConfiguration.ts index a6e8b0586..a4e785812 100644 --- a/src/blob/SqlBlobConfiguration.ts +++ b/src/blob/SqlBlobConfiguration.ts @@ -8,7 +8,8 @@ import { DEFAULT_BLOB_PERSISTENCE_ARRAY, DEFAULT_BLOB_SERVER_HOST_NAME, DEFAULT_ENABLE_ACCESS_LOG, - DEFAULT_ENABLE_DEBUG_LOG + DEFAULT_ENABLE_DEBUG_LOG, + DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT } from "./utils/constants"; /** @@ -22,6 +23,7 @@ export default class SqlBlobConfiguration extends ConfigurationBase { public constructor( host: string = DEFAULT_BLOB_SERVER_HOST_NAME, port: number = DEFAULT_BLOB_LISTENING_PORT, + keepAliveTimeout: number = DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, public readonly sqlURL: string, public readonly sequelizeOptions: SequelizeOptions = DEFAULT_SQL_OPTIONS, public readonly persistenceArray: StoreDestinationArray = DEFAULT_BLOB_PERSISTENCE_ARRAY, @@ -40,6 +42,7 @@ export default class SqlBlobConfiguration extends ConfigurationBase { super( host, port, + keepAliveTimeout, enableAccessLog, accessLogWriteStream, enableDebugLog, diff --git a/src/blob/utils/constants.ts b/src/blob/utils/constants.ts index b3008e472..e6ef5e4b2 100644 --- a/src/blob/utils/constants.ts +++ b/src/blob/utils/constants.ts @@ -31,6 +31,8 @@ export const EMULATOR_ACCOUNT_KEY = Buffer.from( export const EMULATOR_ACCOUNT_SKUNAME = Models.SkuName.StandardRAGRS; export const EMULATOR_ACCOUNT_KIND = Models.AccountKind.StorageV2; export const EMULATOR_ACCOUNT_ISHIERARCHICALNAMESPACEENABLED = false; +export const DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT = 5; + export const HeaderConstants = { AUTHORIZATION: "authorization", diff --git a/src/common/ConfigurationBase.ts b/src/common/ConfigurationBase.ts index 4ea705d23..be3f0928b 100644 --- a/src/common/ConfigurationBase.ts +++ b/src/common/ConfigurationBase.ts @@ -48,6 +48,7 @@ export default abstract class ConfigurationBase { public constructor( public readonly host: string, public readonly port: number, + public readonly keepAliveTimeout: number, public readonly enableAccessLog: boolean = false, public readonly accessLogWriteStream?: NodeJS.WritableStream, public readonly enableDebugLog: boolean = false, diff --git a/src/common/Environment.ts b/src/common/Environment.ts index 768c53402..80ac881d1 100644 --- a/src/common/Environment.ts +++ b/src/common/Environment.ts @@ -1,16 +1,19 @@ import args from "args"; import { + DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, DEFAULT_BLOB_LISTENING_PORT, DEFAULT_BLOB_SERVER_HOST_NAME } from "../blob/utils/constants"; import { + DEFAULT_QUEUE_KEEP_ALIVE_TIMEOUT, DEFAULT_QUEUE_LISTENING_PORT, DEFAULT_QUEUE_SERVER_HOST_NAME } from "../queue/utils/constants"; import { + DEFAULT_TABLE_KEEP_ALIVE_TIMEOUT, DEFAULT_TABLE_LISTENING_PORT, DEFAULT_TABLE_SERVER_HOST_NAME } from "../table/utils/constants"; @@ -28,6 +31,11 @@ args "Optional. Customize listening port for blob", DEFAULT_BLOB_LISTENING_PORT ) + .option( + ["", "blobKeepAliveTimeout"], + "Optional. Customize http keep alive timeout for blob", + DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, + ) .option( ["", "queueHost"], "Optional. Customize listening address for queue", @@ -38,6 +46,11 @@ args "Optional. Customize listening port for queue", DEFAULT_QUEUE_LISTENING_PORT ) + .option( + ["", "queueKeepAliveTimeout"], + "Optional. Customize http keep alive timeout for queue", + DEFAULT_QUEUE_KEEP_ALIVE_TIMEOUT, + ) .option( ["", "tableHost"], "Optional. Customize listening address for table", @@ -48,6 +61,11 @@ args "Optional. Customize listening port for table", DEFAULT_TABLE_LISTENING_PORT ) + .option( + ["", "tableKeepAliveTimeout"], + "Optional. Customize http keep alive timeout for table", + DEFAULT_TABLE_KEEP_ALIVE_TIMEOUT, + ) .option( ["l", "location"], "Optional. Use an existing folder as workspace path, default is current working directory", @@ -99,6 +117,10 @@ export default class Environment implements IEnvironment { return this.flags.blobPort; } + public blobKeepAliveTimeout(): number | undefined { + return this.flags.blobKeepAliveTimeout; + } + public queueHost(): string | undefined { return this.flags.queueHost; } @@ -107,6 +129,10 @@ export default class Environment implements IEnvironment { return this.flags.queuePort; } + public queueKeepAliveTimeout(): number | undefined { + return this.flags.queueKeepAliveTimeout; + } + public tableHost(): string | undefined { return this.flags.tableHost; } @@ -115,6 +141,10 @@ export default class Environment implements IEnvironment { return this.flags.tablePort; } + public tableKeepAliveTimeout(): number | undefined { + return this.flags.tableKeepAliveTimeout; + } + public async location(): Promise { return this.flags.location || process.cwd(); } diff --git a/src/common/ServerBase.ts b/src/common/ServerBase.ts index e16f4a5f3..4c829d852 100644 --- a/src/common/ServerBase.ts +++ b/src/common/ServerBase.ts @@ -36,6 +36,7 @@ export default abstract class ServerBase implements ICleaner { * @param {number} port Server port, for example, 10000 * @param {http.Server | https.Server} httpServer A HTTP or HTTPS server instance without request listener bound * @param {IRequestListenerFactory} requestListenerFactory A request listener factory + * @param config ConfigurationBase configuration * @memberof ServerBase */ public constructor( @@ -45,6 +46,9 @@ export default abstract class ServerBase implements ICleaner { requestListenerFactory: IRequestListenerFactory, public readonly config: ConfigurationBase ) { + if (this.config.keepAliveTimeout > 0) { + httpServer.keepAliveTimeout = this.config.keepAliveTimeout * 1000; + } // Remove predefined request listeners to avoid double request handling this.httpServer = stoppable(httpServer); this.httpServer.removeAllListeners("request"); diff --git a/src/common/VSCEnvironment.ts b/src/common/VSCEnvironment.ts index f5ea1fa4d..8b6c5a41a 100644 --- a/src/common/VSCEnvironment.ts +++ b/src/common/VSCEnvironment.ts @@ -15,6 +15,10 @@ export default class VSCEnvironment implements IEnvironment { return this.workspaceConfiguration.get("blobPort"); } + public blobKeepAliveTimeout(): number | undefined { + return this.workspaceConfiguration.get("blobKeepAliveTimeout"); + } + public queueHost(): string | undefined { return this.workspaceConfiguration.get("queueHost"); } @@ -23,6 +27,10 @@ export default class VSCEnvironment implements IEnvironment { return this.workspaceConfiguration.get("queuePort"); } + public queueKeepAliveTimeout(): number | undefined { + return this.workspaceConfiguration.get("queueKeepAliveTimeout"); + } + public tableHost(): string | undefined { return this.workspaceConfiguration.get("tableHost"); } @@ -31,6 +39,10 @@ export default class VSCEnvironment implements IEnvironment { return this.workspaceConfiguration.get("tablePort"); } + public tableKeepAliveTimeout(): number | undefined { + return this.workspaceConfiguration.get("tableKeepAliveTimeout"); + } + public async location(): Promise { let location = this.workspaceConfiguration.get("location"); diff --git a/src/common/VSCServerManagerBlob.ts b/src/common/VSCServerManagerBlob.ts index e4c497457..3ad33a061 100644 --- a/src/common/VSCServerManagerBlob.ts +++ b/src/common/VSCServerManagerBlob.ts @@ -74,6 +74,7 @@ export default class VSCServerManagerBlob extends VSCServerManagerBase { const config = new BlobConfiguration( env.blobHost(), env.blobPort(), + env.blobKeepAliveTimeout(), join(location, DEFAULT_BLOB_LOKI_DB_PATH), join(location, DEFAULT_BLOB_EXTENT_LOKI_DB_PATH), DEFAULT_BLOB_PERSISTENCE_ARRAY, diff --git a/src/common/VSCServerManagerQueue.ts b/src/common/VSCServerManagerQueue.ts index 791759cf3..30f055025 100644 --- a/src/common/VSCServerManagerQueue.ts +++ b/src/common/VSCServerManagerQueue.ts @@ -77,6 +77,7 @@ export default class VSCServerManagerBlob extends VSCServerManagerBase { const config = new QueueConfiguration( env.queueHost(), env.queuePort(), + env.queueKeepAliveTimeout(), join(location, DEFAULT_QUEUE_LOKI_DB_PATH), join(location, DEFAULT_QUEUE_EXTENT_LOKI_DB_PATH), DEFAULT_QUEUE_PERSISTENCE_ARRAY, diff --git a/src/common/VSCServerManagerTable.ts b/src/common/VSCServerManagerTable.ts index b830ab5c9..df0403a04 100644 --- a/src/common/VSCServerManagerTable.ts +++ b/src/common/VSCServerManagerTable.ts @@ -66,6 +66,7 @@ export default class VSCServerManagerTable extends VSCServerManagerBase { const config = new TableConfiguration( env.tableHost(), env.tablePort(), + env.tableKeepAliveTimeout(), join(location, DEFAULT_TABLE_LOKI_DB_PATH), (await env.debug()) === true, !env.silent(), diff --git a/src/queue/IQueueEnvironment.ts b/src/queue/IQueueEnvironment.ts index b9ed88b43..6aaa01880 100644 --- a/src/queue/IQueueEnvironment.ts +++ b/src/queue/IQueueEnvironment.ts @@ -1,6 +1,7 @@ export default interface IQueueEnvironment { queueHost(): string | undefined; queuePort(): number | undefined; + queueKeepAliveTimeout(): number | undefined; location(): Promise; silent(): boolean; loose(): boolean; diff --git a/src/queue/QueueConfiguration.ts b/src/queue/QueueConfiguration.ts index 31f52257c..804ce1367 100644 --- a/src/queue/QueueConfiguration.ts +++ b/src/queue/QueueConfiguration.ts @@ -8,7 +8,8 @@ import { DEFAULT_QUEUE_LISTENING_PORT, DEFAULT_QUEUE_LOKI_DB_PATH, DEFAULT_QUEUE_PERSISTENCE_ARRAY, - DEFAULT_QUEUE_SERVER_HOST_NAME + DEFAULT_QUEUE_SERVER_HOST_NAME, + DEFAULT_QUEUE_KEEP_ALIVE_TIMEOUT, } from "./utils/constants"; /** @@ -27,6 +28,7 @@ export default class QueueConfiguration extends ConfigurationBase { public constructor( host: string = DEFAULT_QUEUE_SERVER_HOST_NAME, port: number = DEFAULT_QUEUE_LISTENING_PORT, + keepAliveTimeout: number = DEFAULT_QUEUE_KEEP_ALIVE_TIMEOUT, public readonly metadataDBPath: string = DEFAULT_QUEUE_LOKI_DB_PATH, public readonly extentDBPath: string = DEFAULT_QUEUE_EXTENT_LOKI_DB_PATH, public readonly persistencePathArray: StoreDestinationArray = DEFAULT_QUEUE_PERSISTENCE_ARRAY, @@ -47,6 +49,7 @@ export default class QueueConfiguration extends ConfigurationBase { super( host, port, + keepAliveTimeout, enableAccessLog, accessLogWriteStream, enableDebugLog, diff --git a/src/queue/QueueEnvironment.ts b/src/queue/QueueEnvironment.ts index e169e6788..162db8a45 100644 --- a/src/queue/QueueEnvironment.ts +++ b/src/queue/QueueEnvironment.ts @@ -68,6 +68,10 @@ export default class QueueEnvironment implements IQueueEnvironment { return this.flags.queuePort; } + public queueKeepAliveTimeout(): number | undefined { + return this.flags.keepAliveTimeout; + } + public async location(): Promise { return this.flags.location || process.cwd(); } diff --git a/src/queue/QueueServer.ts b/src/queue/QueueServer.ts index 3910cfa7e..cc61bd200 100644 --- a/src/queue/QueueServer.ts +++ b/src/queue/QueueServer.ts @@ -70,6 +70,10 @@ export default class QueueServer extends ServerBase { httpServer = http.createServer(); } + if (configuration.keepAliveTimeout > 0) { + httpServer.keepAliveTimeout = configuration.keepAliveTimeout * 1000 + } + // We can change the persistency layer implementation by // creating a new XXXDataStore class implementing IBlobDataStore interface // and replace the default LokiBlobDataStore diff --git a/src/queue/main.ts b/src/queue/main.ts index 1e96b4b9d..745a0bff9 100644 --- a/src/queue/main.ts +++ b/src/queue/main.ts @@ -52,6 +52,7 @@ async function main() { const config = new QueueConfiguration( env.queueHost(), env.queuePort(), + env.queueKeepAliveTimeout(), join(location, DEFAULT_QUEUE_LOKI_DB_PATH), join(location, DEFAULT_QUEUE_EXTENT_LOKI_DB_PATH), DEFAULT_QUEUE_PERSISTENCE_ARRAY, diff --git a/src/queue/utils/constants.ts b/src/queue/utils/constants.ts index f9b6ed6b9..e8e6bbad3 100644 --- a/src/queue/utils/constants.ts +++ b/src/queue/utils/constants.ts @@ -33,6 +33,7 @@ export const MESSAGETTL_MIN = 1; export const DEFAULT_UPDATE_VISIBILITYTIMEOUT = 30; // 30s as default. export const UPDATE_VISIBILITYTIMEOUT_MIN = 0; export const UPDATE_VISIBILITYTIMEOUT_MAX = 604800; +export const DEFAULT_QUEUE_KEEP_ALIVE_TIMEOUT = 5; export const EMPTY_EXTENT_CHUNK = { id: "", offset: 0, count: 0 }; diff --git a/src/table/ITableEnvironment.ts b/src/table/ITableEnvironment.ts index d1932d4bb..6b3f5e3d4 100644 --- a/src/table/ITableEnvironment.ts +++ b/src/table/ITableEnvironment.ts @@ -8,6 +8,8 @@ export default interface ITableEnvironment { tableHost(): string | undefined; /** Optional. Customize listening port for table */ tablePort(): number | undefined; + /** Optional. Customize keep alive timeout for table */ + tableKeepAliveTimeout(): number | undefined; /** Optional. Use an existing folder as workspace path, default is current working directory */ location(): Promise; /** Optional. Disable access log displayed in console */ diff --git a/src/table/TableConfiguration.ts b/src/table/TableConfiguration.ts index 7b7e89717..5cdea71b6 100644 --- a/src/table/TableConfiguration.ts +++ b/src/table/TableConfiguration.ts @@ -4,7 +4,8 @@ import { DEFAULT_ENABLE_DEBUG_LOG, DEFAULT_TABLE_LISTENING_PORT, DEFAULT_TABLE_LOKI_DB_PATH, - DEFAULT_TABLE_SERVER_HOST_NAME + DEFAULT_TABLE_SERVER_HOST_NAME, + DEFAULT_TABLE_KEEP_ALIVE_TIMEOUT } from "./utils/constants"; /** @@ -24,6 +25,7 @@ export default class TableConfiguration extends ConfigurationBase { public constructor( host: string = DEFAULT_TABLE_SERVER_HOST_NAME, port: number = DEFAULT_TABLE_LISTENING_PORT, + keepAliveTimeout: number = DEFAULT_TABLE_KEEP_ALIVE_TIMEOUT, public readonly /* Store metadata */ metadataDBPath: string = DEFAULT_TABLE_LOKI_DB_PATH, enableDebugLog: boolean = DEFAULT_ENABLE_DEBUG_LOG, enableAccessLog: boolean = DEFAULT_ENABLE_ACCESS_LOG, @@ -41,6 +43,7 @@ export default class TableConfiguration extends ConfigurationBase { super( host, port, + keepAliveTimeout, enableAccessLog, accessLogWriteStream, enableDebugLog, diff --git a/src/table/TableEnvironment.ts b/src/table/TableEnvironment.ts index 73a7f735f..7d61f766e 100644 --- a/src/table/TableEnvironment.ts +++ b/src/table/TableEnvironment.ts @@ -6,7 +6,8 @@ import args from "args"; import ITableEnvironment from "./ITableEnvironment"; import { DEFAULT_TABLE_LISTENING_PORT, - DEFAULT_TABLE_SERVER_HOST_NAME + DEFAULT_TABLE_SERVER_HOST_NAME, + DEFAULT_TABLE_KEEP_ALIVE_TIMEOUT } from "./utils/constants"; args @@ -20,6 +21,11 @@ args "Optional. Customize listening port for table", DEFAULT_TABLE_LISTENING_PORT ) + .option( + ["", "tableKeepAliveTimeout"], + "Optional. Customize http keep alive timeout for table", + DEFAULT_TABLE_KEEP_ALIVE_TIMEOUT, + ) .option( ["l", "location"], "Optional. Use an existing folder as workspace path, default is current working directory", @@ -70,6 +76,10 @@ export default class TableEnvironment implements ITableEnvironment { return this.flags.tablePort; } + public tableKeepAliveTimeout(): number | undefined { + return this.flags.tableKeepAvlieTimeout; + } + public async location(): Promise { return this.flags.location || process.cwd(); } diff --git a/src/table/main.ts b/src/table/main.ts index 7f497cd14..e86b952f9 100644 --- a/src/table/main.ts +++ b/src/table/main.ts @@ -33,6 +33,7 @@ async function main() { const config = new TableConfiguration( env.tableHost(), env.tablePort(), + env.tableKeepAliveTimeout(), join(location, DEFAULT_TABLE_LOKI_DB_PATH), (await env.debug()) !== undefined, !env.silent(), diff --git a/src/table/utils/constants.ts b/src/table/utils/constants.ts index 6fc26f90f..d2251cbdf 100644 --- a/src/table/utils/constants.ts +++ b/src/table/utils/constants.ts @@ -6,6 +6,7 @@ export const DEFAULT_TABLE_LOKI_DB_PATH = "__azurite_db_table__.json"; export const DEFAULT_TABLE_SERVER_HOST_NAME = "127.0.0.1"; // Change to 0.0.0.0 when needs external access export const DEFAULT_TABLE_LISTENING_PORT = 10002; +export const DEFAULT_TABLE_KEEP_ALIVE_TIMEOUT = 5; export const DEFAULT_ENABLE_ACCESS_LOG = true; export const DEFAULT_ENABLE_DEBUG_LOG = true; export const DEFAULT_TABLE_PERSISTENCE_PATH = "__tablestorage__"; diff --git a/tests/BlobTestServerFactory.ts b/tests/BlobTestServerFactory.ts index d83b5a4c0..d8c4311cf 100644 --- a/tests/BlobTestServerFactory.ts +++ b/tests/BlobTestServerFactory.ts @@ -4,6 +4,7 @@ import SqlBlobConfiguration from "../src/blob/SqlBlobConfiguration"; import SqlBlobServer from "../src/blob/SqlBlobServer"; import { StoreDestinationArray } from "../src/common/persistence/IExtentStore"; import { DEFAULT_SQL_OPTIONS } from "../src/common/utils/constants"; +import { DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT } from "../src/blob/utils/constants"; export default class BlobTestServerFactory { public createServer( @@ -36,6 +37,7 @@ export default class BlobTestServerFactory { const config = new SqlBlobConfiguration( host, port, + DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, databaseConnectionString!, DEFAULT_SQL_OPTIONS, persistenceArray, @@ -59,6 +61,7 @@ export default class BlobTestServerFactory { const config = new BlobConfiguration( host, port, + DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, lokiMetadataDBPath, lokiExtentDBPath, persistenceArray, diff --git a/tests/blob/blobKeepAliveTimeout.test.ts b/tests/blob/blobKeepAliveTimeout.test.ts new file mode 100644 index 000000000..17e3c3dd9 --- /dev/null +++ b/tests/blob/blobKeepAliveTimeout.test.ts @@ -0,0 +1,56 @@ +import { + newPipeline, + BlobServiceClient, + StorageSharedKeyCredential, +} from "@azure/storage-blob"; +import * as assert from "assert"; + +import { configLogger } from "../../src/common/Logger"; +import { DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT } from "../../src/blob/utils/constants"; + +import BlobTestServerFactory from "../BlobTestServerFactory"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, +} from "../testutils"; + +// Set true to enable debug log +configLogger(false); + +describe("Blob Keep-Alive header response test", () => { + const factory = new BlobTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const blobServiceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + keepAliveOptions: { enable: true } + } + ) + ); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + it("request with enabled keep-alive shall return DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT", async () => { + const properties = await blobServiceClient.getProperties(); + const keepAliveHeader = properties._response.headers.get("keep-alive"); + if (keepAliveHeader !== undefined) { + assert.strictEqual(keepAliveHeader, "timeout="+DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT); + } + }); + +}); diff --git a/tests/queue/queueKeepAliveTimeout.test.ts b/tests/queue/queueKeepAliveTimeout.test.ts new file mode 100644 index 000000000..0a33c2b44 --- /dev/null +++ b/tests/queue/queueKeepAliveTimeout.test.ts @@ -0,0 +1,78 @@ +import { + StorageSharedKeyCredential, + newPipeline, + QueueServiceClient +} from "@azure/storage-queue"; +import * as assert from "assert"; + +import { configLogger } from "../../src/common/Logger"; +import { StoreDestinationArray } from "../../src/common/persistence/IExtentStore"; +import Server from "../../src/queue/QueueServer"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + rmRecursive, +} from "../testutils"; +import QueueTestServerFactory from "./utils/QueueTestServerFactory"; +import { DEFAULT_QUEUE_KEEP_ALIVE_TIMEOUT } from "../../src/queue/utils/constants"; + +// Set true to enable debug log +configLogger(false); + +describe("Queue Keep-Alive header response test", () => { + // TODO: Create a server factory as tests utils + const host = "127.0.0.1"; + const port = 11001; + const metadataDbPath = "__queueTestsStorage__"; + const extentDbPath = "__extentTestsStorage__"; + const persistencePath = "__queueTestsPersistence__"; + + const DEFAULT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ + { + locationId: "queueTest", + locationPath: persistencePath, + maxConcurrency: 10 + } + ]; + + const baseURL = `http://${host}:${port}/devstoreaccount1`; + const queueServiceClient = new QueueServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + keepAliveOptions: { enable: true } + } + ) + ); + + let server: Server; + before(async () => { + server = new QueueTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + extentDBPath: metadataDbPath, + persistencePathArray: DEFAULT_QUEUE_PERSISTENCE_ARRAY + }); + await server.start(); + }); + + after(async () => { + await server.close(); + await rmRecursive(metadataDbPath); + await rmRecursive(extentDbPath); + await rmRecursive(persistencePath); + }); + + it("request with enabled keep-alive shall return DEFAULT_QUEUE_KEEP_ALIVE_TIMEOUT", async () => { + const properties = await queueServiceClient.getProperties(); + const keepAliveHeader = properties._response.headers.get("keep-alive"); + if (keepAliveHeader !== undefined) { + assert.strictEqual(keepAliveHeader, "timeout=" + DEFAULT_QUEUE_KEEP_ALIVE_TIMEOUT); + } + }); + +}); diff --git a/tests/queue/utils/QueueTestServerFactory.ts b/tests/queue/utils/QueueTestServerFactory.ts index de2c0303a..19f7078ba 100644 --- a/tests/queue/utils/QueueTestServerFactory.ts +++ b/tests/queue/utils/QueueTestServerFactory.ts @@ -1,6 +1,7 @@ import { StoreDestinationArray } from "../../../src/common/persistence/IExtentStore" import QueueConfiguration from "../../../src/queue/QueueConfiguration" import QueueServer from "../../../src/queue/QueueServer" +import { DEFAULT_QUEUE_KEEP_ALIVE_TIMEOUT } from "../../../src/queue/utils/constants"; export interface IQueueTestServerFactoryParams { metadataDBPath: string @@ -27,6 +28,7 @@ export default class QueueTestServerFactory { const config = new QueueConfiguration( host, port, + DEFAULT_QUEUE_KEEP_ALIVE_TIMEOUT, params.metadataDBPath, params.extentDBPath, params.persistencePathArray, diff --git a/tests/table/KeepAlive/tableKeepAliveTimeout.test.ts b/tests/table/KeepAlive/tableKeepAliveTimeout.test.ts new file mode 100644 index 000000000..5981ad371 --- /dev/null +++ b/tests/table/KeepAlive/tableKeepAliveTimeout.test.ts @@ -0,0 +1,62 @@ +import * as assert from "assert"; +import * as Azure from "azure-storage"; + +import { configLogger } from "../../../src/common/Logger"; +import TableServer from "../../../src/table/TableServer"; +import { + overrideRequest, restoreBuildRequestOptions +} from "../../testutils"; +import { + createConnectionStringForTest, + createTableServerForTest +} from "../utils/table.entity.test.utils"; +import { DEFAULT_TABLE_KEEP_ALIVE_TIMEOUT } from "../../../src/table/utils/constants"; + +// Set true to enable debug log +configLogger(false); + +// For convenience, we have a switch to control the use +// of a local Azurite instance, otherwise we need an +// ENV VAR called AZURE_TABLE_STORAGE added to mocha +// script or launch.json containing +// Azure Storage Connection String (using SAS or Key). +const testLocalAzuriteInstance = true; + +describe("Table Keep-Alive header response test", () => { + let server: TableServer; + const tableService = Azure.createTableService( + createConnectionStringForTest(testLocalAzuriteInstance) + ); + tableService.enableGlobalHttpAgent = true; + + const requestOverride = { headers: {} }; + + before(async () => { + overrideRequest(requestOverride, tableService); + server = createTableServerForTest(); + await server.start(); + }); + + after(async () => { + restoreBuildRequestOptions(tableService); + tableService.removeAllListeners(); + await server.close(); + }); + + it("request with enabled keep-alive shall return DEFAULT_TABLE_KEEP_ALIVE_TIMEOUT", (done) => { + tableService.getServiceProperties( + (error, _, response) => { + if (!error) { + if (response.headers !== undefined) { + const keepAliveHeader = response.headers["keep-alive"]; + if (keepAliveHeader !== undefined) { + assert.strictEqual(keepAliveHeader, "timeout="+DEFAULT_TABLE_KEEP_ALIVE_TIMEOUT); + } + } + } else { + assert.fail(error); + } + done(); + }); + }); +}); \ No newline at end of file diff --git a/tests/table/utils/TableTestServerFactory.ts b/tests/table/utils/TableTestServerFactory.ts index 8f7ace176..026ebddd6 100644 --- a/tests/table/utils/TableTestServerFactory.ts +++ b/tests/table/utils/TableTestServerFactory.ts @@ -1,5 +1,6 @@ import TableConfiguration from "../../../src/table/TableConfiguration"; import TableServer from "../../../src/table/TableServer"; +import { DEFAULT_TABLE_KEEP_ALIVE_TIMEOUT } from "../../../src/table/utils/constants"; export interface ITableTestServerFactoryParams { metadataDBPath: string @@ -28,6 +29,7 @@ export default class TableTestServerFactory { const config = new TableConfiguration( host, port, + DEFAULT_TABLE_KEEP_ALIVE_TIMEOUT, params.metadataDBPath, params.enableDebugLog, false,