chore(middleware-sdk-s3): add string fallback for S3#Expires field (#5715)

Author: kuhe

clients/client-s3/src/commands/GetObjectCommand.ts

@@ -1,5 +1,6 @@
 // smithy-typescript generated code
 import { getFlexibleChecksumsPlugin } from "@aws-sdk/middleware-flexible-checksums";
+import { getS3ExpiresMiddlewarePlugin } from "@aws-sdk/middleware-sdk-s3";
 import { getSsecPlugin } from "@aws-sdk/middleware-ssec";
 import { getEndpointPlugin } from "@smithy/middleware-endpoint";
 import { getSerdePlugin } from "@smithy/middleware-serde";
@@ -240,6 +241,7 @@ export interface GetObjectCommandOutput extends Omit<GetObjectOutput, "Body">, _
  * //   ContentRange: "STRING_VALUE",
  * //   ContentType: "STRING_VALUE",
  * //   Expires: new Date("TIMESTAMP"),
+ * //   ExpiresString: "STRING_VALUE",
  * //   WebsiteRedirectLocation: "STRING_VALUE",
  * //   ServerSideEncryption: "AES256" || "aws:kms" || "aws:kms:dsse",
  * //   Metadata: { // Metadata
@@ -351,6 +353,7 @@ export class GetObjectCommand extends $Command
       getSerdePlugin(config, this.serialize, this.deserialize),
       getEndpointPlugin(config, Command.getEndpointParameterInstructions()),
       getSsecPlugin(config),
+      getS3ExpiresMiddlewarePlugin(config),
       getFlexibleChecksumsPlugin(config, {
         input: this.input,
         requestChecksumRequired: false,

clients/client-s3/src/commands/HeadObjectCommand.ts

@@ -1,4 +1,5 @@
 // smithy-typescript generated code
+import { getS3ExpiresMiddlewarePlugin } from "@aws-sdk/middleware-sdk-s3";
 import { getSsecPlugin } from "@aws-sdk/middleware-ssec";
 import { getEndpointPlugin } from "@smithy/middleware-endpoint";
 import { getSerdePlugin } from "@smithy/middleware-serde";
@@ -215,6 +216,7 @@ export interface HeadObjectCommandOutput extends HeadObjectOutput, __MetadataBea
  * //   ContentLanguage: "STRING_VALUE",
  * //   ContentType: "STRING_VALUE",
  * //   Expires: new Date("TIMESTAMP"),
+ * //   ExpiresString: "STRING_VALUE",
  * //   WebsiteRedirectLocation: "STRING_VALUE",
  * //   ServerSideEncryption: "AES256" || "aws:kms" || "aws:kms:dsse",
  * //   Metadata: { // Metadata
@@ -289,6 +291,7 @@ export class HeadObjectCommand extends $Command
       getSerdePlugin(config, this.serialize, this.deserialize),
       getEndpointPlugin(config, Command.getEndpointParameterInstructions()),
       getSsecPlugin(config),
+      getS3ExpiresMiddlewarePlugin(config),
     ];
   })
   .s("AmazonS3", "HeadObject", {})

clients/client-s3/src/models/models_0.ts

@@ -1,6 +1,5 @@
 // smithy-typescript generated code
 import { ExceptionOptionType as __ExceptionOptionType, SENSITIVE_STRING } from "@smithy/smithy-client";
-
 import { StreamingBlobTypes } from "@smithy/types";
 
 import { S3ServiceException as __BaseException } from "./S3ServiceException";
@@ -9037,10 +9036,18 @@ export interface GetObjectOutput {
 
   /**
    * @public
-   * <p>The date and time at which the object is no longer cacheable.</p>
+   * @deprecated
+   *
+   * Deprecated in favor of ExpiresString.
    */
   Expires?: Date;
 
+  /**
+   * @public
+   * <p>The date and time at which the object is no longer cacheable.</p>
+   */
+  ExpiresString?: string;
+
   /**
    * @public
    * <p>If the bucket is configured as a website, redirects requests for this object to another
@@ -10772,10 +10779,18 @@ export interface HeadObjectOutput {
 
   /**
    * @public
-   * <p>The date and time at which the object is no longer cacheable.</p>
+   * @deprecated
+   *
+   * Deprecated in favor of ExpiresString.
    */
   Expires?: Date;
 
+  /**
+   * @public
+   * <p>The date and time at which the object is no longer cacheable.</p>
+   */
+  ExpiresString?: string;
+
   /**
    * @public
    * <p>If the bucket is configured as a website, redirects requests for this object to another

clients/client-s3/src/models/models_1.ts

@@ -1,6 +1,5 @@
 // smithy-typescript generated code
 import { ExceptionOptionType as __ExceptionOptionType, SENSITIVE_STRING } from "@smithy/smithy-client";
-
 import { StreamingBlobTypes } from "@smithy/types";
 
 import {
@@ -28,7 +27,6 @@ import {
   StorageClass,
   Tag,
 } from "./models_0";
-
 import { S3ServiceException as __BaseException } from "./S3ServiceException";
 
 /**

clients/client-s3/src/protocols/Aws_restXml.ts

@@ -4950,6 +4950,7 @@ export const de_GetObjectCommand = async (
     [_CR]: [, output.headers[_cr]],
     [_CT]: [, output.headers[_ct]],
     [_E]: [() => void 0 !== output.headers[_e], () => __expectNonNull(__parseRfc7231DateTime(output.headers[_e]))],
+    [_ES]: [, output.headers[_ex]],
     [_WRL]: [, output.headers[_xawrl]],
     [_SSE]: [, output.headers[_xasse]],
     [_SSECA]: [, output.headers[_xasseca]],
@@ -5440,6 +5441,7 @@ export const de_HeadObjectCommand = async (
     [_CL]: [, output.headers[_cl]],
     [_CT]: [, output.headers[_ct]],
     [_E]: [() => void 0 !== output.headers[_e], () => __expectNonNull(__parseRfc7231DateTime(output.headers[_e]))],
+    [_ES]: [, output.headers[_ex]],
     [_WRL]: [, output.headers[_xawrl]],
     [_SSE]: [, output.headers[_xasse]],
     [_SSECA]: [, output.headers[_xasseca]],
@@ -8359,7 +8361,7 @@ const se_LifecycleRule = (input: LifecycleRule, context: __SerdeContext): any =>
     bn.c(se_LifecycleRuleFilter(input[_F], context).n(_F));
   }
   if (input[_S] != null) {
-    bn.c(__XmlNode.of(_ES, input[_S]).n(_S));
+    bn.c(__XmlNode.of(_ESx, input[_S]).n(_S));
   }
   bn.l(input, "Transitions", "Transition", () => se_TransitionList(input[_Tr]!, context));
   bn.l(input, "NoncurrentVersionTransitions", "NoncurrentVersionTransition", () =>
@@ -11772,8 +11774,9 @@ const _EODM = "ExpiredObjectDeleteMarker";
 const _EOR = "ExistingObjectReplication";
 const _EORS = "ExistingObjectReplicationStatus";
 const _ERP = "EnableRequestProgress";
-const _ES = "ExpirationStatus";
+const _ES = "ExpiresString";
 const _ESBO = "ExpectedSourceBucketOwner";
+const _ESx = "ExpirationStatus";
 const _ET = "EncodingType";
 const _ETa = "ETag";
 const _ETn = "EncryptionType";
@@ -12116,6 +12119,7 @@ const _e = "expires";
 const _en = "encryption";
 const _et = "encoding-type";
 const _eta = "etag";
+const _ex = "expiresstring";
 const _fo = "fetch-owner";
 const _i = "id";
 const _im = "if-match";

codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java

@@ -269,6 +269,11 @@ && containsInputMembers(m, o, BUCKET_ENDPOINT_INPUT_KEYS))
                     HAS_MIDDLEWARE)
                 .servicePredicate((m, s) -> isS3(s))
                 .build(),
+            RuntimeClientPlugin.builder()
+                .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "S3ExpiresMiddleware",
+                    HAS_MIDDLEWARE)
+                .operationPredicate((m, s, o) -> containsExpiresOutput(m, o))
+                .build(),
             RuntimeClientPlugin.builder()
                 .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "S3Express",
                     HAS_MIDDLEWARE)
@@ -288,6 +293,16 @@ private static boolean containsInputMembers(
             .isPresent();
     }
 
+    private static boolean containsExpiresOutput(
+        Model model,
+        OperationShape operationShape
+    ) {
+        OperationIndex operationIndex = OperationIndex.of(model);
+        return operationIndex.getOutput(operationShape)
+            .filter(input -> input.getMemberNames().stream().anyMatch("Expires"::equals))
+            .isPresent();
+    }
+
     private static boolean isS3(Shape serviceShape) {
         return serviceShape.getTrait(ServiceTrait.class).map(ServiceTrait::getSdkId).orElse("").equals("S3");
     }

packages/middleware-sdk-s3/src/index.ts

@@ -1,6 +1,7 @@
 export * from "./check-content-length-header";
 export * from "./region-redirect-endpoint-middleware";
 export * from "./region-redirect-middleware";
+export * from "./s3-expires-middleware";
 export * from "./s3-express/index";
 export * from "./s3Configuration";
 export * from "./throw-200-exceptions";

packages/middleware-sdk-s3/src/s3-expires-middleware.e2e.spec.ts

@@ -0,0 +1,126 @@
+import { S3 } from "@aws-sdk/client-s3";
+import { GetCallerIdentityCommandOutput, STS } from "@aws-sdk/client-sts";
+
+jest.setTimeout(25000);
+
+describe("S3 Expires e2e test", () => {
+  const s3 = new S3({
+    region: "us-west-2",
+    logger: {
+      trace() {},
+      debug() {},
+      info() {},
+      warn: jest.fn(),
+      error() {},
+    },
+  });
+  const stsClient = new STS({ region: "us-west-2" });
+
+  let callerID = null as unknown as GetCallerIdentityCommandOutput;
+  let Bucket: string;
+
+  // random element limited to 2 letters to avoid concurrent IO, and
+  // to limit bucket count to 676 if there is failure to delete them.
+  const alphabet = "abcdefghijklmnopqrstuvwxyz";
+  const randId = alphabet[(Math.random() * alphabet.length) | 0] + alphabet[(Math.random() * alphabet.length) | 0];
+
+  beforeAll(async () => {
+    callerID = await stsClient.getCallerIdentity({});
+    Bucket = `${callerID.Account}-${randId}-s3-expires`;
+    await s3.createBucket({
+      Bucket,
+    });
+  });
+
+  afterAll(async () => {
+    await deleteBucket(s3, Bucket);
+  });
+
+  const staticDate = new Date(0);
+  const dateString = "Thu, 01 Jan 1970 00:00:00 GMT";
+
+  it("should parse Expires from response if it is valid date-time, and include ExpiresString", async () => {
+    await s3.putObject({
+      Bucket,
+      Key: "good-expires",
+      Expires: staticDate,
+      Body: "good-expires",
+    });
+
+    const get = await s3.getObject({
+      Bucket,
+      Key: "good-expires",
+    });
+    await get.Body?.transformToByteArray(); // drain stream.
+
+    expect(get.Expires?.getTime()).toEqual(staticDate.getTime());
+    expect(get.ExpiresString).toEqual(dateString);
+  });
+
+  it("should fail with a non-blocking warning if Expires is not a valid date-time, and include the raw string in ExpiresString", async () => {
+    await s3.putObject({
+      Bucket,
+      Key: "bad-expires",
+      Expires: new Date("invalid date"),
+      Body: "bad-expires",
+    });
+
+    const get = await s3.getObject({
+      Bucket,
+      Key: "bad-expires",
+    });
+    await get.Body?.transformToByteArray(); // drain stream.
+
+    expect(get.Expires).toBeUndefined();
+    expect(s3.config.logger.warn).toHaveBeenCalledWith(
+      `AWS SDK Warning for S3Client::GetObjectCommand response parsing (undefined, NaN undefined NaN NaN:NaN:NaN GMT): TypeError: Invalid RFC-7231 date-time value`
+    );
+    expect(get.ExpiresString).toEqual("undefined, NaN undefined NaN NaN:NaN:NaN GMT");
+  });
+});
+
+async function deleteBucket(s3: S3, bucketName: string) {
+  const Bucket = bucketName;
+
+  try {
+    await s3.headBucket({
+      Bucket,
+    });
+  } catch (e) {
+    return;
+  }
+
+  const list = await s3
+    .listObjects({
+      Bucket,
+    })
+    .catch((e) => {
+      if (!String(e).includes("NoSuchBucket")) {
+        throw e;
+      }
+      return {
+        Contents: [],
+      };
+    });
+
+  const promises = [] as any[];
+  for (const key of list.Contents ?? []) {
+    promises.push(
+      s3.deleteObject({
+        Bucket,
+        Key: key.Key,
+      })
+    );
+  }
+  await Promise.all(promises);
+
+  try {
+    return await s3.deleteBucket({
+      Bucket,
+    });
+  } catch (e) {
+    if (!String(e).includes("NoSuchBucket")) {
+      throw e;
+    }
+  }
+}

packages/middleware-sdk-s3/src/s3-expires-middleware.ts

@@ -0,0 +1,72 @@
+import { HttpResponse } from "@smithy/protocol-http";
+import { parseRfc7231DateTime } from "@smithy/smithy-client";
+import {
+  DeserializeHandler,
+  DeserializeHandlerArguments,
+  DeserializeHandlerOutput,
+  DeserializeMiddleware,
+  HandlerExecutionContext,
+  MetadataBearer,
+  Pluggable,
+  RelativeMiddlewareOptions,
+} from "@smithy/types";
+
+/**
+ * @internal
+ */
+interface PreviouslyResolved {}
+
+/**
+ * @internal
+ *
+ * From the S3 Expires compatibility spec.
+ * A model transform will ensure S3#Expires remains a timestamp shape, though
+ * it is deprecated.
+ * If a particular object has a non-date string set as the Expires value,
+ * the SDK will have the raw string as "ExpiresString" on the response.
+ *
+ */
+export const s3ExpiresMiddleware = (config: PreviouslyResolved): DeserializeMiddleware<any, any> => {
+  return <Output extends MetadataBearer>(
+      next: DeserializeHandler<any, Output>,
+      context: HandlerExecutionContext
+    ): DeserializeHandler<any, Output> =>
+    async (args: DeserializeHandlerArguments<any>): Promise<DeserializeHandlerOutput<Output>> => {
+      const result = await next(args);
+      const { response } = result;
+      if (HttpResponse.isInstance(response)) {
+        if (response.headers.expires) {
+          response.headers.expiresstring = response.headers.expires;
+          try {
+            parseRfc7231DateTime(response.headers.expires);
+          } catch (e) {
+            context.logger?.warn(
+              `AWS SDK Warning for ${context.clientName}::${context.commandName} response parsing (${response.headers.expires}): ${e}`
+            );
+            delete response.headers.expires;
+          }
+        }
+      }
+      return result;
+    };
+};
+
+/**
+ * @internal
+ */
+export const s3ExpiresMiddlewareOptions: RelativeMiddlewareOptions = {
+  tags: ["S3"],
+  name: "s3ExpiresMiddleware",
+  override: true,
+  relation: "after",
+  toMiddleware: "deserializerMiddleware",
+};
+
+/**
+ * @internal
+ */
+export const getS3ExpiresMiddlewarePlugin = (clientConfig: PreviouslyResolved): Pluggable<any, any> => ({
+  applyToStack: (clientStack) => {
+    clientStack.addRelativeTo(s3ExpiresMiddleware(clientConfig), s3ExpiresMiddlewareOptions);
+  },
+});

scripts/generate-clients/s3-hack.js

@@ -10,6 +10,8 @@ const s3ModelObject = require(s3ModelLocation);
 
 /**
  * Activates a hack for S3-express Smithy suppression.
+ * And another one for S3 Expires.
+ *
  * @returns a function that undoes the hack.
  */
 module.exports = function () {
@@ -21,6 +23,53 @@ module.exports = function () {
     namespace: "com.amazonaws.s3",
   });
 
+  const expiresShape = s3ModelObject.shapes["com.amazonaws.s3#Expires"];
+  if (expiresShape) {
+    // enforce that Expires retains type timestamp.
+    expiresShape.type = "timestamp";
+
+    // add the ExpiresString string shape.
+    const newShapes = {};
+    for (const [shapeId, shape] of Object.entries(s3ModelObject.shapes)) {
+      newShapes[shapeId] = shape;
+      if (shapeId === "com.amazonaws.s3#Expires") {
+        newShapes["com.amazonaws.s3#ExpiresString"] = {
+          type: "string",
+        };
+      }
+    }
+    s3ModelObject.shapes = newShapes;
+
+    // add ExpiresString alongside output shapes containing Expires.
+    for (const [shapeId, shape] of Object.entries(s3ModelObject.shapes)) {
+      if (shape?.traits?.["smithy.api#output"]) {
+        const newMembers = {};
+        for (const [memberName, member] of Object.entries(shape.members)) {
+          newMembers[memberName] = member;
+          if (member.target === "com.amazonaws.s3#Expires") {
+            const existingDoc = member.traits["smithy.api#documentation"];
+            if (!member.traits) {
+              member.traits = {};
+            }
+
+            newMembers.ExpiresString = {
+              target: "com.amazonaws.s3#ExpiresString",
+              traits: {
+                ...member.traits,
+                "smithy.api#httpHeader": "ExpiresString",
+                "smithy.api#documentation": existingDoc,
+              },
+            };
+
+            member.traits["smithy.api#deprecated"] = {};
+            member.traits["smithy.api#documentation"] = "Deprecated in favor of ExpiresString.";
+          }
+        }
+        shape.members = newMembers;
+      }
+    }
+  }
+
   fs.writeFileSync(s3ModelLocation, JSON.stringify(s3ModelObject, null, 2));
 
   return () => {