feat(credential-provider-node): use dynamic import for credential providers (#5698)

Author: kuhe

Makefile

@@ -20,6 +20,9 @@ unlink-smithy:
 	rm ./node_modules/\@smithy
 	yarn --check-files
 
+copy-smithy:
+	node ./scripts/copy-smithy-dist-files
+
 # Runs build for all packages using Turborepo
 turbo-build:
 	(cd scripts/remote-cache && yarn)

package.json

@@ -69,7 +69,7 @@
     "@tsconfig/recommended": "1.0.1",
     "@types/chai-as-promised": "^7.1.2",
     "@types/fs-extra": "^8.0.1",
-    "@types/jest": "29.5.2",
+    "@types/jest": "29.5.11",
     "@typescript-eslint/eslint-plugin": "5.55.0",
     "@typescript-eslint/parser": "5.55.0",
     "async": "3.2.4",
@@ -93,8 +93,8 @@
     "glob": "7.1.6",
     "husky": "^4.2.3",
     "jasmine-core": "^3.5.0",
-    "jest": "29.5.0",
-    "jest-environment-jsdom": "29.5.0",
+    "jest": "29.7.0",
+    "jest-environment-jsdom": "29.7.0",
     "jmespath": "^0.15.0",
     "json5": "^2.2.0",
     "karma": "6.4.0",
@@ -113,7 +113,7 @@
     "mocha": "10.0.0",
     "prettier": "2.8.5",
     "rimraf": "3.0.2",
-    "ts-jest": "29.1.0",
+    "ts-jest": "29.1.1",
     "ts-loader": "9.4.2",
     "ts-mocha": "10.0.0",
     "ts-node": "10.9.1",

packages/credential-provider-node/src/defaultProvider.spec.ts

@@ -3,10 +3,10 @@ import { fromIni } from "@aws-sdk/credential-provider-ini";
 import { fromProcess } from "@aws-sdk/credential-provider-process";
 import { fromSSO } from "@aws-sdk/credential-provider-sso";
 import { fromTokenFile } from "@aws-sdk/credential-provider-web-identity";
-import { chain, CredentialsProviderError, memoize } from "@smithy/property-provider";
+import { CredentialsProviderError } from "@smithy/property-provider";
 import { ENV_PROFILE, loadSharedConfigFiles } from "@smithy/shared-ini-file-loader";
 
-import { defaultProvider } from "./defaultProvider";
+import { credentialsTreatedAsExpired, credentialsWillNeedRefresh, defaultProvider } from "./defaultProvider";
 import { remoteProvider } from "./remoteProvider";
 
 jest.mock("@aws-sdk/credential-provider-env");
@@ -15,7 +15,6 @@ jest.mock("@aws-sdk/credential-provider-ini");
 jest.mock("@aws-sdk/credential-provider-process");
 jest.mock("@aws-sdk/credential-provider-sso");
 jest.mock("@aws-sdk/credential-provider-web-identity");
-jest.mock("@smithy/property-provider");
 jest.mock("@smithy/shared-ini-file-loader");
 jest.mock("./remoteProvider");
 
@@ -25,19 +24,24 @@ describe(defaultProvider.name, () => {
     secretAccessKey: "mockSecretAccessKey",
   };
 
+  const credentials = () => {
+    throw new CredentialsProviderError("test", true);
+  };
+
+  const finalCredentials = () => {
+    return mockCreds;
+  };
+
   const mockInit = {
     profile: "mockProfile",
   };
 
-  const mockEnvFn = jest.fn();
-  const mockSsoFn = jest.fn();
-  const mockIniFn = jest.fn();
-  const mockProcessFn = jest.fn();
-  const mockTokenFileFn = jest.fn();
-  const mockRemoteProviderFn = jest.fn();
-
-  const mockChainFn = jest.fn();
-  const mockMemoizeFn = jest.fn().mockResolvedValue(mockCreds);
+  const mockEnvFn = jest.fn().mockImplementation(() => credentials());
+  const mockSsoFn = jest.fn().mockImplementation(() => credentials());
+  const mockIniFn = jest.fn().mockImplementation(() => credentials());
+  const mockProcessFn = jest.fn().mockImplementation(() => credentials());
+  const mockTokenFileFn = jest.fn().mockImplementation(() => credentials());
+  const mockRemoteProviderFn = jest.fn().mockImplementation(() => finalCredentials());
 
   beforeEach(() => {
     [
@@ -47,51 +51,48 @@ describe(defaultProvider.name, () => {
       [fromProcess, mockProcessFn],
       [fromTokenFile, mockTokenFileFn],
       [remoteProvider, mockRemoteProviderFn],
-      [chain, mockChainFn],
-      [memoize, mockMemoizeFn],
     ].forEach(([fromFn, mockFn]) => {
       (fromFn as jest.Mock).mockReturnValue(mockFn);
     });
   });
 
   afterEach(async () => {
-    const errorFnIndex = (chain as jest.Mock).mock.calls[0].length;
-    const errorFn = (chain as jest.Mock).mock.calls[0][errorFnIndex - 1];
-    const expectedError = new CredentialsProviderError("Could not load credentials from any providers", false);
-    try {
-      await errorFn();
-      fail(`expected ${expectedError}`);
-    } catch (error) {
-      expect(error.toString()).toStrictEqual(expectedError.toString());
-    }
-
-    expect(memoize).toHaveBeenCalledWith(mockChainFn, expect.any(Function), expect.any(Function));
-
     jest.clearAllMocks();
   });
 
   describe("without fromEnv", () => {
-    afterEach(() => {
-      expect(chain).toHaveBeenCalledWith(
-        mockSsoFn,
-        mockIniFn,
-        mockProcessFn,
-        mockTokenFileFn,
-        mockRemoteProviderFn,
-        expect.any(Function)
-      );
-    });
-
     it("creates provider chain and memoizes it", async () => {
-      const receivedCreds = await defaultProvider(mockInit)();
-      expect(receivedCreds).toStrictEqual(mockCreds);
+      const provider = defaultProvider(mockInit);
 
-      expect(fromEnv).not.toHaveBeenCalled();
-      for (const fromFn of [fromSSO, fromIni, fromProcess, fromTokenFile, remoteProvider]) {
-        expect(fromFn).toHaveBeenCalledWith(mockInit);
+      // initial call proceeds through the chain.
+      {
+        const receivedCreds = await provider();
+        expect(receivedCreds).toEqual(mockCreds);
+
+        expect(fromEnv).not.toHaveBeenCalled();
+        expect(fromSSO).toHaveBeenCalledWith(mockInit);
+        expect(fromIni).toHaveBeenCalledWith(mockInit);
+        expect(fromProcess).toHaveBeenCalledWith(mockInit);
+        expect(fromTokenFile).toHaveBeenCalledWith(mockInit);
+        expect(remoteProvider).toHaveBeenCalledWith(mockInit);
+
+        expect(loadSharedConfigFiles).not.toHaveBeenCalled();
       }
 
-      expect(loadSharedConfigFiles).not.toHaveBeenCalled();
+      jest.clearAllMocks();
+
+      // subsequent call does not enter the chain.
+      {
+        const receivedCreds = await provider();
+        expect(receivedCreds).toEqual(mockCreds);
+
+        expect(fromEnv).not.toHaveBeenCalled();
+        expect(fromSSO).not.toHaveBeenCalledWith(mockInit);
+        expect(fromIni).not.toHaveBeenCalledWith(mockInit);
+        expect(fromProcess).not.toHaveBeenCalledWith(mockInit);
+        expect(fromTokenFile).not.toHaveBeenCalledWith(mockInit);
+        expect(remoteProvider).not.toHaveBeenCalledWith(mockInit);
+      }
     });
 
     it(`if env['${ENV_PROFILE}'] is set`, async () => {
@@ -123,63 +124,42 @@ describe(defaultProvider.name, () => {
     for (const fromFn of [fromSSO, fromIni, fromProcess, fromTokenFile, remoteProvider]) {
       expect(fromFn).toHaveBeenCalledWith(mockInitWithoutProfile);
     }
-
-    expect(chain).toHaveBeenCalledWith(
-      mockEnvFn,
-      mockSsoFn,
-      mockIniFn,
-      mockProcessFn,
-      mockTokenFileFn,
-      mockRemoteProviderFn,
-      expect.any(Function)
-    );
   });
 
-  describe("memoize isExpired", () => {
+  describe(credentialsTreatedAsExpired.name, () => {
     const mockDateNow = Date.now();
     beforeEach(async () => {
       jest.spyOn(Date, "now").mockReturnValueOnce(mockDateNow);
-      await defaultProvider(mockInit)();
     });
 
     it("returns true if expiration is defined, and creds have expired", () => {
-      const memoizeExpiredFn = (memoize as jest.Mock).mock.calls[0][1];
       const expiration = new Date(mockDateNow - 24 * 60 * 60 * 1000);
-      expect(memoizeExpiredFn({ expiration })).toEqual(true);
+      expect(credentialsTreatedAsExpired({ ...mockCreds, expiration })).toEqual(true);
     });
 
     it("returns true if expiration is defined, and creds expire in <5 mins", () => {
-      const memoizeExpiredFn = (memoize as jest.Mock).mock.calls[0][1];
       const expiration = new Date(mockDateNow + 299 * 1000);
-      expect(memoizeExpiredFn({ expiration })).toEqual(true);
+      expect(credentialsTreatedAsExpired({ ...mockCreds, expiration })).toEqual(true);
     });
 
     it("returns false if expiration is defined, but creds expire in >5 mins", () => {
-      const memoizeExpiredFn = (memoize as jest.Mock).mock.calls[0][1];
       const expiration = new Date(mockDateNow + 301 * 1000);
-      expect(memoizeExpiredFn({ expiration })).toEqual(false);
+      expect(credentialsTreatedAsExpired({ ...mockCreds, expiration })).toEqual(false);
     });
 
     it("returns false if expiration is not defined", () => {
-      const memoizeExpiredFn = (memoize as jest.Mock).mock.calls[0][1];
-      expect(memoizeExpiredFn({})).toEqual(false);
+      expect(credentialsTreatedAsExpired({ ...mockCreds })).toEqual(false);
     });
   });
 
-  describe("memoize requiresRefresh", () => {
-    beforeEach(async () => {
-      await defaultProvider(mockInit)();
-    });
-
+  describe(credentialsWillNeedRefresh.name, () => {
     it("returns true if expiration is not defined", () => {
-      const memoizeRefreshFn = (memoize as jest.Mock).mock.calls[0][2];
-      const expiration = Date.now();
-      expect(memoizeRefreshFn({ expiration })).toEqual(true);
+      const expiration = new Date();
+      expect(credentialsWillNeedRefresh({ ...mockCreds, expiration })).toEqual(true);
     });
 
     it("returns false if expiration is not defined", () => {
-      const memoizeRefreshFn = (memoize as jest.Mock).mock.calls[0][2];
-      expect(memoizeRefreshFn({})).toEqual(false);
+      expect(credentialsWillNeedRefresh({ ...mockCreds })).toEqual(false);
     });
   });
 });

packages/credential-provider-node/src/defaultProvider.ts

@@ -1,9 +1,8 @@
-import { fromEnv } from "@aws-sdk/credential-provider-env";
-import { fromIni, FromIniInit } from "@aws-sdk/credential-provider-ini";
-import { fromProcess, FromProcessInit } from "@aws-sdk/credential-provider-process";
-import { fromSSO, FromSSOInit } from "@aws-sdk/credential-provider-sso";
-import { fromTokenFile, FromTokenFileInit } from "@aws-sdk/credential-provider-web-identity";
-import { RemoteProviderInit } from "@smithy/credential-provider-imds";
+import type { FromIniInit } from "@aws-sdk/credential-provider-ini";
+import type { FromProcessInit } from "@aws-sdk/credential-provider-process";
+import type { FromSSOInit } from "@aws-sdk/credential-provider-sso";
+import type { FromTokenFileInit } from "@aws-sdk/credential-provider-web-identity";
+import type { RemoteProviderInit } from "@smithy/credential-provider-imds";
 import { chain, CredentialsProviderError, memoize } from "@smithy/property-provider";
 import { ENV_PROFILE } from "@smithy/shared-ini-file-loader";
 import { AwsCredentialIdentity, MemoizedProvider } from "@smithy/types";
@@ -49,16 +48,52 @@ export type DefaultProviderInit = FromIniInit & RemoteProviderInit & FromProcess
 export const defaultProvider = (init: DefaultProviderInit = {}): MemoizedProvider<AwsCredentialIdentity> =>
   memoize(
     chain(
-      ...(init.profile || process.env[ENV_PROFILE] ? [] : [fromEnv()]),
-      fromSSO(init),
-      fromIni(init),
-      fromProcess(init),
-      fromTokenFile(init),
-      remoteProvider(init),
+      ...(init.profile || process.env[ENV_PROFILE]
+        ? []
+        : [
+            async () => {
+              const { fromEnv } = await import("@aws-sdk/credential-provider-env");
+              return fromEnv()();
+            },
+          ]),
+      async () => {
+        const { fromSSO } = await import("@aws-sdk/credential-provider-sso");
+        return fromSSO(init)();
+      },
+      async () => {
+        const { fromIni } = await import("@aws-sdk/credential-provider-ini");
+        return fromIni(init)();
+      },
+      async () => {
+        const { fromProcess } = await import("@aws-sdk/credential-provider-process");
+        return fromProcess(init)();
+      },
+      async () => {
+        const { fromTokenFile } = await import("@aws-sdk/credential-provider-web-identity");
+        return fromTokenFile(init)();
+      },
+      async () => {
+        return (await remoteProvider(init))();
+      },
       async () => {
         throw new CredentialsProviderError("Could not load credentials from any providers", false);
       }
     ),
-    (credentials) => credentials.expiration !== undefined && credentials.expiration.getTime() - Date.now() < 300000,
-    (credentials) => credentials.expiration !== undefined
+    credentialsTreatedAsExpired,
+    credentialsWillNeedRefresh
   );
+
+/**
+ * @internal
+ *
+ * @returns credentials have expiration.
+ */
+export const credentialsWillNeedRefresh = (credentials: AwsCredentialIdentity) => credentials?.expiration !== undefined;
+
+/**
+ * @internal
+ *
+ * @returns credentials with less than 5 minutes left.
+ */
+export const credentialsTreatedAsExpired = (credentials: AwsCredentialIdentity) =>
+  credentials?.expiration !== undefined && credentials.expiration.getTime() - Date.now() < 300000;

packages/credential-provider-node/src/remoteProvider.spec.ts

@@ -42,7 +42,7 @@ describe(remoteProvider.name, () => {
     "returns fromContainerMetadata if env['%s'] is set",
     async (key) => {
       process.env[key] = "defined";
-      const receivedCreds = await remoteProvider(mockInit)();
+      const receivedCreds = await (await remoteProvider(mockInit))();
       expect(receivedCreds).toStrictEqual(mockCredsFromContainer);
       expect(fromContainerMetadata).toHaveBeenCalledWith(mockInit);
       expect(fromInstanceMetadata).not.toHaveBeenCalled();
@@ -53,7 +53,9 @@ describe(remoteProvider.name, () => {
     process.env[ENV_IMDS_DISABLED] = "1";
     const expectedError = new CredentialsProviderError("EC2 Instance Metadata Service access disabled");
     try {
-      await remoteProvider(mockInit)();
+      await (
+        await remoteProvider(mockInit)
+      )();
       fail(`expectedError ${expectedError}`);
     } catch (error) {
       expect(error).toStrictEqual(expectedError);
@@ -63,7 +65,7 @@ describe(remoteProvider.name, () => {
   });
 
   it("returns fromInstanceMetadata if environment variables are not set", async () => {
-    const receivedCreds = await remoteProvider(mockInit)();
+    const receivedCreds = await (await remoteProvider(mockInit))();
     expect(receivedCreds).toStrictEqual(mockSourceCredsFromInstanceMetadata);
     expect(fromInstanceMetadata).toHaveBeenCalledWith(mockInit);
     expect(fromContainerMetadata).not.toHaveBeenCalled();

packages/credential-provider-node/src/remoteProvider.ts

@@ -1,16 +1,17 @@
-import {
-  ENV_CMDS_FULL_URI,
-  ENV_CMDS_RELATIVE_URI,
-  fromContainerMetadata,
-  fromInstanceMetadata,
-  RemoteProviderInit,
-} from "@smithy/credential-provider-imds";
+import type { RemoteProviderInit } from "@smithy/credential-provider-imds";
 import { CredentialsProviderError } from "@smithy/property-provider";
-import { AwsCredentialIdentityProvider } from "@smithy/types";
+import type { AwsCredentialIdentityProvider } from "@smithy/types";
 
 export const ENV_IMDS_DISABLED = "AWS_EC2_METADATA_DISABLED";
 
-export const remoteProvider = (init: RemoteProviderInit): AwsCredentialIdentityProvider => {
+/**
+ * @internal
+ */
+export const remoteProvider = async (init: RemoteProviderInit): Promise<AwsCredentialIdentityProvider> => {
+  const { ENV_CMDS_FULL_URI, ENV_CMDS_RELATIVE_URI, fromContainerMetadata, fromInstanceMetadata } = await import(
+    "@smithy/credential-provider-imds"
+  );
+
   if (process.env[ENV_CMDS_RELATIVE_URI] || process.env[ENV_CMDS_FULL_URI]) {
     return fromContainerMetadata(init);
   }

packages/middleware-sdk-s3/src/middleware-sdk-s3.integ.spec.ts

@@ -1,7 +1,4 @@
-import { GetObjectCommand, PutObjectCommand, S3 } from "@aws-sdk/client-s3";
-import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
-import { NoOpLogger } from "@smithy/smithy-client";
-import { parseUrl } from "@smithy/url-parser";
+import { S3 } from "@aws-sdk/client-s3";
 
 import { requireRequestsFrom } from "../../../private/aws-util-test/src";
 

packages/polly-request-presigner/src/getSignedUrls.spec.ts

@@ -14,12 +14,17 @@ jest.mock("@aws-sdk/util-format-url", () => ({
   formatUrl: (url: any) => url,
 }));
 
-import { RequestPresigningArguments } from "@smithy/types";
+import { AwsCredentialIdentity, RequestPresigningArguments } from "@smithy/types";
 
 import { getSignedUrl } from "./getSignedUrls";
 
 describe("getSignedUrl", () => {
-  const clientParams = { region: "us-foo-1" };
+  const credentials: AwsCredentialIdentity = {
+    secretAccessKey: "unit-test",
+    accessKeyId: "unit-test",
+    sessionToken: "unit-test",
+  };
+  const clientParams = { region: "us-foo-1", credentials };
 
   beforeEach(() => {
     mockPresign.mockReset();

private/aws-client-retry-test/src/ClientRetryTest.spec.ts

@@ -1,6 +1,6 @@
 import { HeadObjectCommand, S3, S3Client, S3ServiceException } from "@aws-sdk/client-s3";
 import { HttpHandler, HttpResponse } from "@smithy/protocol-http";
-import { RequestHandlerOutput } from "@smithy/types";
+import { AwsCredentialIdentity, RequestHandlerOutput } from "@smithy/types";
 import { ConfiguredRetryStrategy, StandardRetryStrategy } from "@smithy/util-retry";
 import { Readable } from "stream";
 
@@ -22,6 +22,12 @@ class MockRequestHandler implements HttpHandler {
 }
 
 describe("util-retry integration tests", () => {
+  const credentials: AwsCredentialIdentity = {
+    accessKeyId: "test",
+    secretAccessKey: "test",
+    sessionToken: "test",
+  };
+
   const mockThrottled: RequestHandlerOutput<HttpResponse> = {
     response: new HttpResponse({
       statusCode: 429,
@@ -48,6 +54,7 @@ describe("util-retry integration tests", () => {
         httpHandlerConfigs: () => ({}),
       },
       region: MOCK_REGION,
+      credentials,
     });
     expect(await client.config.retryStrategy()).toBeInstanceOf(StandardRetryStrategy);
     const response = await client.send(headObjectCommand);
@@ -69,6 +76,7 @@ describe("util-retry integration tests", () => {
         updateHttpClientConfig: () => {},
       },
       region: MOCK_REGION,
+      credentials,
     });
     expect(await client.config.retryStrategy()).toBeInstanceOf(StandardRetryStrategy);
     const response = await client.send(headObjectCommand);
@@ -97,6 +105,7 @@ describe("util-retry integration tests", () => {
         updateHttpClientConfig: () => {},
       },
       region: MOCK_REGION,
+      credentials,
     });
     expect(await client.config.retryStrategy()).toBeInstanceOf(StandardRetryStrategy);
     try {
@@ -125,6 +134,7 @@ describe("util-retry integration tests", () => {
       requestHandler: new MockRequestHandler(),
       retryStrategy,
       region: MOCK_REGION,
+      credentials,
     });
 
     expect(retryStrategy.getCapacity()).toEqual(expectedInitialCapacity);

private/aws-middleware-test/src/middleware-content-length.spec.ts

@@ -2,7 +2,7 @@ import { AccessAnalyzer } from "@aws-sdk/client-accessanalyzer";
 import { S3 } from "@aws-sdk/client-s3";
 import { XRay } from "@aws-sdk/client-xray";
 
-import { requireRequestsFrom } from "../../aws-util-test/src";
+import { requireRequestsFrom } from "../../../private/aws-util-test/src";
 
 describe("middleware-content-length", () => {
   describe(AccessAnalyzer.name, () => {

scripts/compilation/Inliner.js

@@ -157,6 +157,9 @@ module.exports = class Inliner {
       mainFields: ["main"],
       allowOverwrite: true,
       entryPoints: [path.join(root, this.subfolder, this.package, "src", "index.ts")],
+      supported: {
+        "dynamic-import": false,
+      },
       outfile: this.outfile,
       keepNames: true,
       packages: "external",