fix(middleware-flexible-checksums): buffer stream chunks to minimum r…

…equired size

packages/middleware-flexible-checksums/src/configuration.ts

@@ -66,4 +66,9 @@ export interface PreviouslyResolved {
    * Collects streams into buffers.
    */
   streamCollector: StreamCollector;
+
+  /**
+   * Minimum bytes from a stream to buffer into a chunk before passing to chunked encoding.
+   */
+  requestStreamBufferSize: number;
 }

packages/middleware-flexible-checksums/src/flexibleChecksumsMiddleware.ts

@@ -9,6 +9,7 @@ import {
   HandlerExecutionContext,
   MetadataBearer,
 } from "@smithy/types";
+import { createBufferedReadable } from "@smithy/util-stream";
 
 import { PreviouslyResolved } from "./configuration";
 import { ChecksumAlgorithm, DEFAULT_CHECKSUM_ALGORITHM, RequestChecksumCalculation } from "./constants";
@@ -119,13 +120,18 @@ export const flexibleChecksumsMiddleware =
       const checksumAlgorithmFn = selectChecksumAlgorithmFunction(checksumAlgorithm, config);
       if (isStreaming(requestBody)) {
         const { getAwsChunkedEncodingStream, bodyLengthChecker } = config;
-        updatedBody = getAwsChunkedEncodingStream(requestBody, {
-          base64Encoder,
-          bodyLengthChecker,
-          checksumLocationName,
-          checksumAlgorithmFn,
-          streamHasher,
-        });
+        updatedBody = getAwsChunkedEncodingStream(
+          config.requestStreamBufferSize
+            ? createBufferedReadable(requestBody, config.requestStreamBufferSize, context.logger)
+            : requestBody,
+          {
+            base64Encoder,
+            bodyLengthChecker,
+            checksumLocationName,
+            checksumAlgorithmFn,
+            streamHasher,
+          }
+        );
         updatedHeaders = {
           ...headers,
           "content-encoding": headers["content-encoding"]

packages/middleware-flexible-checksums/src/resolveFlexibleChecksumsConfig.ts

@@ -18,11 +18,25 @@ export interface FlexibleChecksumsInputConfig {
    * Determines when checksum validation will be performed on response payloads.
    */
   responseChecksumValidation?: ResponseChecksumValidation | Provider<ResponseChecksumValidation>;
+
+  /**
+   * Default 65536.
+   *
+   * Minimum number of bytes to buffer into a chunk when processing input streams
+   * with chunked encoding (that is, when request checksums are enabled).
+   * A minimum of 8kb = 8 * 1024 is required, and 64kb or higher is recommended.
+   *
+   * See https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html.
+   *
+   * To turn off this feature, configure the value to zero or false.
+   */
+  requestStreamBufferSize?: number | false;
 }
 
 export interface FlexibleChecksumsResolvedConfig {
   requestChecksumCalculation: Provider<RequestChecksumCalculation>;
   responseChecksumValidation: Provider<ResponseChecksumValidation>;
+  requestStreamBufferSize: number;
 }
 
 export const resolveFlexibleChecksumsConfig = <T>(
@@ -35,4 +49,5 @@ export const resolveFlexibleChecksumsConfig = <T>(
   responseChecksumValidation: normalizeProvider(
     input.responseChecksumValidation ?? DEFAULT_RESPONSE_CHECKSUM_VALIDATION
   ),
+  requestStreamBufferSize: Number(input.requestStreamBufferSize ?? 64 * 1024),
 });

packages/middleware-sdk-s3/src/check-content-length-header.spec.ts

@@ -130,4 +130,24 @@ describe("checkContentLengthHeaderMiddleware", () => {
 
     expect(spy).not.toHaveBeenCalled();
   });
+
+  it("does not warn if uploading a payload of known length via alternate header x-amz-decoded-content-length", async () => {
+    const handler = checkContentLengthHeader()(mockNextHandler, {});
+
+    await handler({
+      request: {
+        method: null,
+        protocol: null,
+        hostname: null,
+        path: null,
+        query: {},
+        headers: {
+          "x-amz-decoded-content-length": "5",
+        },
+      },
+      input: {},
+    });
+
+    expect(spy).not.toHaveBeenCalled();
+  });
 });

packages/middleware-sdk-s3/src/check-content-length-header.ts

@@ -12,6 +12,7 @@ import {
 } from "@smithy/types";
 
 const CONTENT_LENGTH_HEADER = "content-length";
+const DECODED_CONTENT_LENGTH_HEADER = "x-amz-decoded-content-length";
 
 /**
  * @internal
@@ -28,7 +29,7 @@ export function checkContentLengthHeader(): FinalizeRequestMiddleware<any, any>
       const { request } = args;
 
       if (HttpRequest.isInstance(request)) {
-        if (!(CONTENT_LENGTH_HEADER in request.headers)) {
+        if (!(CONTENT_LENGTH_HEADER in request.headers) && !(DECODED_CONTENT_LENGTH_HEADER in request.headers)) {
           const message = `Are you using a Stream of unknown length as the Body of a PutObject request? Consider using Upload instead from @aws-sdk/lib-storage.`;
           if (typeof context?.logger?.warn === "function" && !(context.logger instanceof NoOpLogger)) {
             context.logger.warn(message);

private/aws-client-api-test/src/client-interface-tests/client-s3/impl/initializeWithMaximalConfiguration.ts

@@ -130,6 +130,7 @@ export const initializeWithMaximalConfiguration = () => {
     requestChecksumCalculation: DEFAULT_REQUEST_CHECKSUM_CALCULATION,
     responseChecksumValidation: DEFAULT_RESPONSE_CHECKSUM_VALIDATION,
     userAgentAppId: "testApp",
+    requestStreamBufferSize: 8 * 1024,
   };
 
   const s3 = new S3Client(config);

supplemental-docs/CLIENTS.md

@@ -753,7 +753,8 @@ See also https://aws.amazon.com/blogs/developer/middleware-stack-modular-aws-sdk
 
 ### S3
 
-`followRegionRedirects`:
+#### `followRegionRedirects`:
+
 This feature was previously called the S3 Global Client. Setting this option to true enables failed requests to be retried with a corrected region when receiving a permanent redirect error with status 301. Note that this can result in additional latency owing to the retried request. This feature should only be used as a last resort if you do not know the region of your bucket(s) ahead of time.
 
 ```ts
@@ -763,6 +764,20 @@ new S3Client({
 });
 ```
 
+#### `requestChecksumCalculation` and `responseChecksumValidation`:
+
+These may be set to `WHEN_REQUIRED` or `WHEN_SUPPORTED`. See https://github.com/aws/aws-sdk-js-v3/issues/6810.
+
+#### `requestStreamBufferSize`:
+
+This has a default value of `64 * 1024` bytes, or 64kb. This only comes into play when request checksums are enabled.
+When this is enabled, user input streams that emit chunks having size less than the value configured will be buffered
+until the buffer exceeds the desired size before continuing to flow.
+
+This may be set to `false` to disable, or a numeric byte size.
+
+See https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html.
+
 ### SQS
 
 #### Using Queue Names with SQS Client