diff --git a/packages/middleware-flexible-checksums/src/configuration.ts b/packages/middleware-flexible-checksums/src/configuration.ts index a6a066c74924..e9deff02e06f 100644 --- a/packages/middleware-flexible-checksums/src/configuration.ts +++ b/packages/middleware-flexible-checksums/src/configuration.ts @@ -66,4 +66,9 @@ export interface PreviouslyResolved { * Collects streams into buffers. */ streamCollector: StreamCollector; + + /** + * Minimum bytes from a stream to buffer into a chunk before passing to chunked encoding. + */ + requestStreamBufferSize: number; } diff --git a/packages/middleware-flexible-checksums/src/flexibleChecksumsMiddleware.ts b/packages/middleware-flexible-checksums/src/flexibleChecksumsMiddleware.ts index 5759f42ec60b..34e371797312 100644 --- a/packages/middleware-flexible-checksums/src/flexibleChecksumsMiddleware.ts +++ b/packages/middleware-flexible-checksums/src/flexibleChecksumsMiddleware.ts @@ -9,6 +9,7 @@ import { HandlerExecutionContext, MetadataBearer, } from "@smithy/types"; +import { createBufferedReadable } from "@smithy/util-stream"; import { PreviouslyResolved } from "./configuration"; import { ChecksumAlgorithm, DEFAULT_CHECKSUM_ALGORITHM, RequestChecksumCalculation } from "./constants"; @@ -119,13 +120,18 @@ export const flexibleChecksumsMiddleware = const checksumAlgorithmFn = selectChecksumAlgorithmFunction(checksumAlgorithm, config); if (isStreaming(requestBody)) { const { getAwsChunkedEncodingStream, bodyLengthChecker } = config; - updatedBody = getAwsChunkedEncodingStream(requestBody, { - base64Encoder, - bodyLengthChecker, - checksumLocationName, - checksumAlgorithmFn, - streamHasher, - }); + updatedBody = getAwsChunkedEncodingStream( + config.requestStreamBufferSize + ? createBufferedReadable(requestBody, config.requestStreamBufferSize, context.logger) + : requestBody, + { + base64Encoder, + bodyLengthChecker, + checksumLocationName, + checksumAlgorithmFn, + streamHasher, + } + ); updatedHeaders = { ...headers, "content-encoding": headers["content-encoding"] diff --git a/packages/middleware-flexible-checksums/src/resolveFlexibleChecksumsConfig.ts b/packages/middleware-flexible-checksums/src/resolveFlexibleChecksumsConfig.ts index 57cfd22e7ed9..a6ffb6a18cfd 100644 --- a/packages/middleware-flexible-checksums/src/resolveFlexibleChecksumsConfig.ts +++ b/packages/middleware-flexible-checksums/src/resolveFlexibleChecksumsConfig.ts @@ -18,11 +18,25 @@ export interface FlexibleChecksumsInputConfig { * Determines when checksum validation will be performed on response payloads. */ responseChecksumValidation?: ResponseChecksumValidation | Provider; + + /** + * Default 65536. + * + * Minimum number of bytes to buffer into a chunk when processing input streams + * with chunked encoding (that is, when request checksums are enabled). + * A minimum of 8kb = 8 * 1024 is required, and 64kb or higher is recommended. + * + * See https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html. + * + * To turn off this feature, configure the value to zero or false. + */ + requestStreamBufferSize?: number | false; } export interface FlexibleChecksumsResolvedConfig { requestChecksumCalculation: Provider; responseChecksumValidation: Provider; + requestStreamBufferSize: number; } export const resolveFlexibleChecksumsConfig = ( @@ -35,4 +49,5 @@ export const resolveFlexibleChecksumsConfig = ( responseChecksumValidation: normalizeProvider( input.responseChecksumValidation ?? DEFAULT_RESPONSE_CHECKSUM_VALIDATION ), + requestStreamBufferSize: Number(input.requestStreamBufferSize ?? 64 * 1024), }); diff --git a/packages/middleware-sdk-s3/src/check-content-length-header.spec.ts b/packages/middleware-sdk-s3/src/check-content-length-header.spec.ts index 6efc3e96dc9a..17945fa08582 100644 --- a/packages/middleware-sdk-s3/src/check-content-length-header.spec.ts +++ b/packages/middleware-sdk-s3/src/check-content-length-header.spec.ts @@ -130,4 +130,24 @@ describe("checkContentLengthHeaderMiddleware", () => { expect(spy).not.toHaveBeenCalled(); }); + + it("does not warn if uploading a payload of known length via alternate header x-amz-decoded-content-length", async () => { + const handler = checkContentLengthHeader()(mockNextHandler, {}); + + await handler({ + request: { + method: null, + protocol: null, + hostname: null, + path: null, + query: {}, + headers: { + "x-amz-decoded-content-length": "5", + }, + }, + input: {}, + }); + + expect(spy).not.toHaveBeenCalled(); + }); }); diff --git a/packages/middleware-sdk-s3/src/check-content-length-header.ts b/packages/middleware-sdk-s3/src/check-content-length-header.ts index ca45e78484bd..4425b4e479c7 100644 --- a/packages/middleware-sdk-s3/src/check-content-length-header.ts +++ b/packages/middleware-sdk-s3/src/check-content-length-header.ts @@ -12,6 +12,7 @@ import { } from "@smithy/types"; const CONTENT_LENGTH_HEADER = "content-length"; +const DECODED_CONTENT_LENGTH_HEADER = "x-amz-decoded-content-length"; /** * @internal @@ -28,7 +29,7 @@ export function checkContentLengthHeader(): FinalizeRequestMiddleware const { request } = args; if (HttpRequest.isInstance(request)) { - if (!(CONTENT_LENGTH_HEADER in request.headers)) { + if (!(CONTENT_LENGTH_HEADER in request.headers) && !(DECODED_CONTENT_LENGTH_HEADER in request.headers)) { const message = `Are you using a Stream of unknown length as the Body of a PutObject request? Consider using Upload instead from @aws-sdk/lib-storage.`; if (typeof context?.logger?.warn === "function" && !(context.logger instanceof NoOpLogger)) { context.logger.warn(message); diff --git a/private/aws-client-api-test/src/client-interface-tests/client-s3/impl/initializeWithMaximalConfiguration.ts b/private/aws-client-api-test/src/client-interface-tests/client-s3/impl/initializeWithMaximalConfiguration.ts index adbf6c675449..9989684a276b 100644 --- a/private/aws-client-api-test/src/client-interface-tests/client-s3/impl/initializeWithMaximalConfiguration.ts +++ b/private/aws-client-api-test/src/client-interface-tests/client-s3/impl/initializeWithMaximalConfiguration.ts @@ -130,6 +130,7 @@ export const initializeWithMaximalConfiguration = () => { requestChecksumCalculation: DEFAULT_REQUEST_CHECKSUM_CALCULATION, responseChecksumValidation: DEFAULT_RESPONSE_CHECKSUM_VALIDATION, userAgentAppId: "testApp", + requestStreamBufferSize: 8 * 1024, }; const s3 = new S3Client(config); diff --git a/supplemental-docs/CLIENTS.md b/supplemental-docs/CLIENTS.md index 66ab91f51a1b..e05f01fcee82 100644 --- a/supplemental-docs/CLIENTS.md +++ b/supplemental-docs/CLIENTS.md @@ -753,7 +753,8 @@ See also https://aws.amazon.com/blogs/developer/middleware-stack-modular-aws-sdk ### S3 -`followRegionRedirects`: +#### `followRegionRedirects`: + This feature was previously called the S3 Global Client. Setting this option to true enables failed requests to be retried with a corrected region when receiving a permanent redirect error with status 301. Note that this can result in additional latency owing to the retried request. This feature should only be used as a last resort if you do not know the region of your bucket(s) ahead of time. ```ts @@ -763,6 +764,20 @@ new S3Client({ }); ``` +#### `requestChecksumCalculation` and `responseChecksumValidation`: + +These may be set to `WHEN_REQUIRED` or `WHEN_SUPPORTED`. See https://github.com/aws/aws-sdk-js-v3/issues/6810. + +#### `requestStreamBufferSize`: + +This has a default value of `64 * 1024` bytes, or 64kb. This only comes into play when request checksums are enabled. +When this is enabled, user input streams that emit chunks having size less than the value configured will be buffered +until the buffer exceeds the desired size before continuing to flow. + +This may be set to `false` to disable, or a numeric byte size. + +See https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html. + ### SQS #### Using Queue Names with SQS Client