Add a persisting cache layer for HTTP credentials (async-aws#600)

jderusse · Nyholm · web-flow · commit 5294f172c0f6 · 2020-05-13T14:23:01.000+02:00
* Add a persisting cache layer for HTTP credentials

* Add Psr Provider and remove default injection

* Add example in doc

* Apply suggestions

* Integrate with Symfony

* Fix CS

* adjustExpireDate not nullable

* Apply suggestions

* Cache only expirable credentials

Co-authored-by: Tobias Nyholm &lt;tobias.nyholm@gmail.com&gt;
diff --git a/composer.json b/composer.json
@@ -31,6 +31,7 @@
         "nette/php-generator": "^3.3",
         "nette/utils": "^3.0",
         "nyholm/symfony-bundle-test": "^1.6.1",
+        "psr/cache": "^1.0",
         "symfony/cache": "^4.4 || ^5.0",
         "symfony/config": "^4.4 || ^5.0",
         "symfony/console": "^4.4 || ^5.0",
diff --git a/docs/authentication/index.md b/docs/authentication/index.md
@@ -30,3 +30,25 @@ The providers are currently chained in the following order:
 
 The default provider chain could be too slow or too complex for testing. It is recommended
 to use the `NullProvider` in tests where you don't provide valid configuration values.
+
+## Caching credential
+
+Fetching credential from AWS metadata endpoint (EC2 Instance, ECS Container,
+WebIdentity...) for each request can introduce latency. AsyncAws provides a
+`PsrCacheProvider` and a `SymfonyCacheProvider` decorator to persist and reuse
+credentials between each request.
+
+```php
+use AsyncAws\Core\Credentials\CacheProvider;
+use AsyncAws\Core\Credentials\ChainProvider;
+use AsyncAws\Core\Credentials\SymfonyCacheProvider;
+use Symfony\Component\Cache\Adapter\FilesystemAdapter;
+
+$provider = ChainProvider::createDefaultChain();
+$provider = new SymfonyCacheProvider($provider, new FilesystemAdapter('test'));
+
+// optional, decorate the provider with the in-memory Cache provider
+$provider = new CacheProvider($provider);
+
+$s3 = new S3Client([], $provider);
+```
diff --git a/src/Core/CHANGELOG.md b/src/Core/CHANGELOG.md
@@ -6,6 +6,8 @@
 
 - Support for EventBridge in `AwsClientFactory`
 - Support for IAM in `AwsClientFactory`
+- Add a `PsrCacheProvider` and `SymfonyCacheProvider` to persists crendentials in a cache pool
+- Add a `Credential::adjustExpireDate` method for adjusting the time according to the time difference with AWS clock
 - Support for global and regional endpoints
 
 ## 1.1.0
diff --git a/src/Core/composer.json b/src/Core/composer.json
@@ -15,7 +15,8 @@
         "ext-SimpleXML": "*",
         "ext-hash": "*",
         "ext-json": "*",
-        "psr/log": "~1.0",
+        "psr/cache": "^1.0",
+        "psr/log": "^1.0",
         "symfony/http-client": "^4.4 || ^5.0",
         "symfony/http-client-contracts": "^1.1.8 || ^2.0",
         "symfony/service-contracts": "^1.0 || ^2.0"
diff --git a/src/Core/src/AbstractApi.php b/src/Core/src/AbstractApi.php
@@ -6,12 +6,7 @@
 
 use AsyncAws\Core\Credentials\CacheProvider;
 use AsyncAws\Core\Credentials\ChainProvider;
-use AsyncAws\Core\Credentials\ConfigurationProvider;
-use AsyncAws\Core\Credentials\ContainerProvider;
 use AsyncAws\Core\Credentials\CredentialProvider;
-use AsyncAws\Core\Credentials\IniFileProvider;
-use AsyncAws\Core\Credentials\InstanceProvider;
-use AsyncAws\Core\Credentials\WebIdentityProvider;
 use AsyncAws\Core\Exception\InvalidArgument;
 use AsyncAws\Core\Signer\Signer;
 use AsyncAws\Core\Signer\SignerV4;
@@ -68,13 +63,7 @@ public function __construct($configuration = [], ?CredentialProvider $credential
         $this->httpClient = $httpClient ?? HttpClient::create();
         $this->logger = $logger ?? new NullLogger();
         $this->configuration = $configuration;
-        $this->credentialProvider = $credentialProvider ?? new CacheProvider(new ChainProvider([
-            new ConfigurationProvider(),
-            new WebIdentityProvider($this->logger),
-            new IniFileProvider($this->logger),
-            new ContainerProvider($this->httpClient, $this->logger),
-            new InstanceProvider($this->httpClient, $this->logger),
-        ]));
+        $this->credentialProvider = $credentialProvider ?? new CacheProvider(ChainProvider::createDefaultChain($this->httpClient, $this->logger));
     }
 
     final public function getConfiguration(): Configuration
diff --git a/src/Core/src/AwsClientFactory.php b/src/Core/src/AwsClientFactory.php
@@ -10,12 +10,7 @@
 use AsyncAws\CognitoIdentityProvider\CognitoIdentityProviderClient;
 use AsyncAws\Core\Credentials\CacheProvider;
 use AsyncAws\Core\Credentials\ChainProvider;
-use AsyncAws\Core\Credentials\ConfigurationProvider;
-use AsyncAws\Core\Credentials\ContainerProvider;
 use AsyncAws\Core\Credentials\CredentialProvider;
-use AsyncAws\Core\Credentials\IniFileProvider;
-use AsyncAws\Core\Credentials\InstanceProvider;
-use AsyncAws\Core\Credentials\WebIdentityProvider;
 use AsyncAws\Core\Exception\InvalidArgument;
 use AsyncAws\Core\Exception\MissingDependency;
 use AsyncAws\Core\Sts\StsClient;
@@ -79,13 +74,7 @@ public function __construct($configuration = [], ?CredentialProvider $credential
         $this->httpClient = $httpClient ?? HttpClient::create();
         $this->logger = $logger ?? new NullLogger();
         $this->configuration = $configuration;
-        $this->credentialProvider = $credentialProvider ?? new CacheProvider(new ChainProvider([
-            new ConfigurationProvider(),
-            new WebIdentityProvider($this->logger),
-            new IniFileProvider($this->logger),
-            new ContainerProvider($this->httpClient, $this->logger),
-            new InstanceProvider($this->httpClient, $this->logger),
-        ]));
+        $this->credentialProvider = $credentialProvider ?? new CacheProvider(ChainProvider::createDefaultChain($this->httpClient, $this->logger));
     }
 
     public function cloudFormation(): CloudFormationClient
diff --git a/src/Core/src/Credentials/CacheProvider.php b/src/Core/src/Credentials/CacheProvider.php
@@ -8,7 +8,7 @@
 use Symfony\Contracts\Service\ResetInterface;
 
 /**
- * Cache the Credential generated by the decorated CredentialProvider.
+ * Cache the Credential generated by the decorated CredentialProvider in memory.
  *
  * The Credential will be reused until it expires.
  *
diff --git a/src/Core/src/Credentials/ChainProvider.php b/src/Core/src/Credentials/ChainProvider.php
@@ -5,6 +5,10 @@
 namespace AsyncAws\Core\Credentials;
 
 use AsyncAws\Core\Configuration;
+use Psr\Log\LoggerInterface;
+use Psr\Log\NullLogger;
+use Symfony\Component\HttpClient\HttpClient;
+use Symfony\Contracts\HttpClient\HttpClientInterface;
 use Symfony\Contracts\Service\ResetInterface;
 
 /**
@@ -60,4 +64,18 @@ public function reset()
     {
         $this->lastSuccessfulProvider = [];
     }
+
+    public static function createDefaultChain(?HttpClientInterface $httpClient = null, ?LoggerInterface $logger = null): CredentialProvider
+    {
+        $httpClient = $httpClient ?? HttpClient::create();
+        $logger = $logger ?? new NullLogger();
+
+        return new ChainProvider([
+            new ConfigurationProvider(),
+            new WebIdentityProvider($logger, null, $httpClient),
+            new IniFileProvider($logger, null, $httpClient),
+            new ContainerProvider($httpClient, $logger),
+            new InstanceProvider($httpClient, $logger),
+        ]);
+    }
 }
diff --git a/src/Core/src/Credentials/ContainerProvider.php b/src/Core/src/Credentials/ContainerProvider.php
@@ -61,11 +61,15 @@ public function getCredentials(Configuration $configuration): ?Credentials
             return null;
         }
 
+        if (null !== $date = $response->getHeaders(false)['date'][0] ?? null) {
+            $date = new \DateTimeImmutable($date);
+        }
+
         return new Credentials(
             $result['AccessKeyId'],
             $result['SecretAccessKey'],
             $result['Token'],
-            new \DateTimeImmutable($result['Expiration'])
+            Credentials::adjustExpireDate(new \DateTimeImmutable($result['Expiration']), $date)
         );
     }
 }
diff --git a/src/Core/src/Credentials/Credentials.php b/src/Core/src/Credentials/Credentials.php
@@ -13,6 +13,8 @@
  */
 final class Credentials implements CredentialProvider
 {
+    private const EXPIRATION_DRIFT = 30;
+
     private $accessKeyId;
 
     private $secretKey;
@@ -62,4 +64,13 @@ public function getCredentials(Configuration $configuration): ?Credentials
     {
         return $this->isExpired() ? null : $this;
     }
+
+    public static function adjustExpireDate(\DateTimeImmutable $expireDate, ?\DateTimeImmutable $reference = null): \DateTimeImmutable
+    {
+        if (null !== $reference) {
+            $expireDate = (new \DateTimeImmutable())->add($reference->diff($expireDate));
+        }
+
+        return $expireDate->sub(new \DateInterval(sprintf('PT%dS', self::EXPIRATION_DRIFT)));
+    }
 }
diff --git a/src/Core/src/Credentials/IniFileProvider.php b/src/Core/src/Credentials/IniFileProvider.php
@@ -8,6 +8,7 @@
 use AsyncAws\Core\Sts\StsClient;
 use Psr\Log\LoggerInterface;
 use Psr\Log\NullLogger;
+use Symfony\Contracts\HttpClient\HttpClientInterface;
 
 /**
  * Provides Credentials from standard AWS ini file.
@@ -22,10 +23,13 @@ final class IniFileProvider implements CredentialProvider
 
     private $logger;
 
-    public function __construct(?LoggerInterface $logger = null, ?IniFileLoader $iniFileLoader = null)
+    private $httpClient;
+
+    public function __construct(?LoggerInterface $logger = null, ?IniFileLoader $iniFileLoader = null, ?HttpClientInterface $httpClient = null)
     {
         $this->logger = $logger ?? new NullLogger();
         $this->iniFileLoader = $iniFileLoader ?? new IniFileLoader($this->logger);
+        $this->httpClient = $httpClient;
     }
 
     public function getCredentials(Configuration $configuration): ?Credentials
@@ -95,7 +99,11 @@ private function getCredentialsFromRole(array $profilesData, array $profileData,
             return null;
         }
 
-        $stsClient = new StsClient(isset($profilesData[$sourceProfileName][IniFileLoader::KEY_REGION]) ? ['region' => $profilesData[$sourceProfileName][IniFileLoader::KEY_REGION]] : [], $sourceCredentials);
+        $stsClient = new StsClient(
+            isset($profilesData[$sourceProfileName][IniFileLoader::KEY_REGION]) ? ['region' => $profilesData[$sourceProfileName][IniFileLoader::KEY_REGION]] : [],
+            $sourceCredentials,
+            $this->httpClient
+        );
         $result = $stsClient->assumeRole([
             'RoleArn' => $roleArn,
             'RoleSessionName' => $roleSessionName,
@@ -111,11 +119,16 @@ private function getCredentialsFromRole(array $profilesData, array $profileData,
             return null;
         }
 
+        $date = null;
+        if ((null !== $response = $result->info()['response'] ?? null) && null !== $date = $response->getHeaders(false)['date'][0] ?? null) {
+            $date = new \DateTimeImmutable($date);
+        }
+
         return new Credentials(
             $credentials->getAccessKeyId(),
             $credentials->getSecretAccessKey(),
             $credentials->getSessionToken(),
-            $credentials->getExpiration()
+            Credentials::adjustExpireDate($credentials->getExpiration(), $date)
         );
     }
 }
diff --git a/src/Core/src/Credentials/InstanceProvider.php b/src/Core/src/Credentials/InstanceProvider.php
@@ -80,11 +80,15 @@ public function getCredentials(Configuration $configuration): ?Credentials
             return null;
         }
 
+        if (null !== $date = $response->getHeaders(false)['date'][0] ?? null) {
+            $date = new \DateTimeImmutable($date);
+        }
+
         return new Credentials(
             $result['AccessKeyId'],
             $result['SecretAccessKey'],
             $result['Token'],
-            new \DateTimeImmutable($result['Expiration'])
+            Credentials::adjustExpireDate(new \DateTimeImmutable($result['Expiration']), $date)
         );
     }
 
diff --git a/src/Core/src/Credentials/PsrCacheProvider.php b/src/Core/src/Credentials/PsrCacheProvider.php
@@ -0,0 +1,43 @@
+<?php
+
+declare(strict_types=1);
+
+namespace AsyncAws\Core\Credentials;
+
+use AsyncAws\Core\Configuration;
+use Psr\Cache\CacheItemPoolInterface;
+
+/**
+ * Cache the Credential generated by the decorated CredentialProvider with PSR-6.
+ *
+ * The Credential will be reused until it expires.
+ *
+ * @author Jérémy Derussé <jeremy@derusse.com>
+ */
+final class PsrCacheProvider implements CredentialProvider
+{
+    private $cache;
+
+    private $decorated;
+
+    public function __construct(CredentialProvider $decorated, CacheItemPoolInterface $cache)
+    {
+        $this->decorated = $decorated;
+        $this->cache = $cache;
+    }
+
+    public function getCredentials(Configuration $configuration): ?Credentials
+    {
+        $item = $this->cache->getItem('AsyncAws.Credentials.' . sha1(\serialize([$configuration, \get_class($this->decorated)])));
+        if (!$item->isHit()) {
+            $item->set($credential = $this->decorated->getCredentials($configuration));
+
+            if (null !== $credential && null !== $exp = $credential->getExpireDate()) {
+                $item->expiresAt($exp);
+                $this->cache->save($item);
+            }
+        }
+
+        return $item->get();
+    }
+}
diff --git a/src/Core/src/Credentials/SymfonyCacheProvider.php b/src/Core/src/Credentials/SymfonyCacheProvider.php
@@ -0,0 +1,46 @@
+<?php
+
+declare(strict_types=1);
+
+namespace AsyncAws\Core\Credentials;
+
+use AsyncAws\Core\Configuration;
+use Symfony\Contracts\Cache\CacheInterface;
+use Symfony\Contracts\Cache\ItemInterface;
+
+/**
+ * Cache the Credential generated by the decorated CredentialProvider with Symfony Cache.
+ * Symfony Cache provides stampede protection which is preferred on applications with more than
+ * 1 or 2 requests per second.
+ *
+ * The Credential will be reused until it expires.
+ *
+ * @author Jérémy Derussé <jeremy@derusse.com>
+ */
+final class SymfonyCacheProvider implements CredentialProvider
+{
+    private $cache;
+
+    private $decorated;
+
+    public function __construct(CredentialProvider $decorated, CacheInterface $cache)
+    {
+        $this->decorated = $decorated;
+        $this->cache = $cache;
+    }
+
+    public function getCredentials(Configuration $configuration): ?Credentials
+    {
+        return $this->cache->get('AsyncAws.Credentials.' . sha1(\serialize([$configuration, \get_class($this->decorated)])), function (ItemInterface $item) use ($configuration) {
+            $credential = $this->decorated->getCredentials($configuration);
+
+            if (null !== $credential && null !== $exp = $credential->getExpireDate()) {
+                $item->expiresAt($exp);
+            } else {
+                $item->expiresAfter(0);
+            }
+
+            return $credential;
+        });
+    }
+}
diff --git a/src/Core/src/Credentials/WebIdentityProvider.php b/src/Core/src/Credentials/WebIdentityProvider.php
@@ -8,6 +8,7 @@
 use AsyncAws\Core\Sts\StsClient;
 use Psr\Log\LoggerInterface;
 use Psr\Log\NullLogger;
+use Symfony\Contracts\HttpClient\HttpClientInterface;
 
 /**
  * Provides Credentials from Web Identity or OpenID Connect Federation.
@@ -22,10 +23,13 @@ final class WebIdentityProvider implements CredentialProvider
 
     private $logger;
 
-    public function __construct(?LoggerInterface $logger = null, ?IniFileLoader $iniFileLoader = null)
+    private $httpClient;
+
+    public function __construct(?LoggerInterface $logger = null, ?IniFileLoader $iniFileLoader = null, ?HttpClientInterface $httpClient = null)
     {
         $this->logger = $logger ?? new NullLogger();
         $this->iniFileLoader = $iniFileLoader ?? new IniFileLoader($this->logger);
+        $this->httpClient = $httpClient;
     }
 
     public function getCredentials(Configuration $configuration): ?Credentials
@@ -91,7 +95,7 @@ private function getCredentialsFromRole(string $roleArn, string $tokenFile, ?str
             return null;
         }
 
-        $stsClient = new StsClient(['region' => $region], new NullProvider());
+        $stsClient = new StsClient(['region' => $region], new NullProvider(), $this->httpClient);
         $result = $stsClient->assumeRoleWithWebIdentity([
             'RoleArn' => $roleArn,
             'RoleSessionName' => $sessionName,
@@ -108,11 +112,16 @@ private function getCredentialsFromRole(string $roleArn, string $tokenFile, ?str
             return null;
         }
 
+        $date = null;
+        if ((null !== $response = $result->info()['response'] ?? null) && null !== $date = $response->getHeaders(false)['date'][0] ?? null) {
+            $date = new \DateTimeImmutable($date);
+        }
+
         return new Credentials(
             $credentials->getAccessKeyId(),
             $credentials->getSecretAccessKey(),
             $credentials->getSessionToken(),
-            $credentials->getExpiration()
+            Credentials::adjustExpireDate($credentials->getExpiration(), $date)
         );
     }
 }
diff --git a/src/Integration/Symfony/Bundle/src/DependencyInjection/AsyncAwsExtension.php b/src/Integration/Symfony/Bundle/src/DependencyInjection/AsyncAwsExtension.php

Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,7 @@`
`8`	`8`	`use Symfony\Contracts\Service\ResetInterface;`
`9`	`9`
`10`	`10`	`/**`
`11`		`- * Cache the Credential generated by the decorated CredentialProvider.`
	`11`	`+ * Cache the Credential generated by the decorated CredentialProvider in memory.`
`12`	`12`	`*`
`13`	`13`	`* The Credential will be reused until it expires.`
`14`	`14`	`*`
Original file line number	Diff line number	Diff line change
`@@ -61,11 +61,15 @@ public function getCredentials(Configuration $configuration): ?Credentials`
`61`	`61`	`return null;`
`62`	`62`	`}`
`63`	`63`
	`64`	`+ if (null !== $date = $response->getHeaders(false)['date'][0] ?? null) {`
	`65`	`+ $date = new \DateTimeImmutable($date);`
	`66`	`+ }`
	`67`	`+`
`64`	`68`	`return new Credentials(`
`65`	`69`	`$result['AccessKeyId'],`
`66`	`70`	`$result['SecretAccessKey'],`
`67`	`71`	`$result['Token'],`
`68`		`- new \DateTimeImmutable($result['Expiration'])`
	`72`	`+ Credentials::adjustExpireDate(new \DateTimeImmutable($result['Expiration']), $date)`
`69`	`73`	`);`
`70`	`74`	`}`
`71`	`75`	`}`
Original file line number	Diff line number	Diff line change
`@@ -13,6 +13,8 @@`
`13`	`13`	`*/`
`14`	`14`	`final class Credentials implements CredentialProvider`
`15`	`15`	`{`
	`16`	`+ private const EXPIRATION_DRIFT = 30;`
	`17`	`+`
`16`	`18`	`private $accessKeyId;`
`17`	`19`
`18`	`20`	`private $secretKey;`
`@@ -62,4 +64,13 @@ public function getCredentials(Configuration $configuration): ?Credentials`
`62`	`64`	`{`
`63`	`65`	`return $this->isExpired() ? null : $this;`
`64`	`66`	`}`
	`67`	`+`
	`68`	`+ public static function adjustExpireDate(\DateTimeImmutable $expireDate, ?\DateTimeImmutable $reference = null): \DateTimeImmutable`
	`69`	`+ {`
	`70`	`+ if (null !== $reference) {`
	`71`	`+ $expireDate = (new \DateTimeImmutable())->add($reference->diff($expireDate));`
	`72`	`+ }`
	`73`	`+`
	`74`	`+ return $expireDate->sub(new \DateInterval(sprintf('PT%dS', self::EXPIRATION_DRIFT)));`
	`75`	`+ }`
`65`	`76`	`}`