diff --git a/README.md b/README.md index fa2ec7a..41ba207 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,15 @@ Amazon Web Services integration for ASP.NET Core data protection. Server keys can be stored in S3 and encrypted using KMS. -[![Build status](https://ci.appveyor.com/api/projects/status/5k00d5fdfspjv20e?svg=true)](https://ci.appveyor.com/project/hotchkj/aspnetcore-dataprotection-aws) +[![Build status](https://ci.appveyor.com/api/projects/status/5k00d5fdfspjv20e/branch/master?svg=true)](https://ci.appveyor.com/project/hotchkj/aspnetcore-dataprotection-aws/branch/master) ## S3 Persistence By default, ASP.NET Core stores encryption keys locally which causes issues with key mismatches across server farms. S3 can be used instead of a shared filesystem to provide key storage. Server-side S3 encryption of AES256 is enabled by default for all keys written to S3. It remains the client's responsibility to ensure access control to the S3 bucket is appropriately configured. +[Guidance](https://github.com/aspnet/DataProtection/issues/158) from Microsoft indicates that the repository itself cannot clean up key data as the usage lifetime is not known to the key management layer. If S3 usage over time is a concern, clients need to trade off key lifetime (and corresponding revocation lifetime) vs S3 storage costs. A suitable approach might be S3 lifecycle policies to remove ancient key files that could not possibly be in use in the client's deployed scenario. Key files generated by typical XmlKeyManager runs are less than 1kB each. + ### Configuration In Startup.cs, specified as part of DataProtection configuration: ```csharp @@ -17,7 +19,7 @@ public void ConfigureServices(IServiceCollection services) services.AddDataProtection(); services.ConfigureDataProtection(configure => { - configure.PersistKeysToS3(new AmazonS3Client(), new S3XmlRepositoryConfig("my-bucket-name") + configure.PersistKeysToAwsS3(new AmazonS3Client(), new S3XmlRepositoryConfig("my-bucket-name") // Configuration has defaults; all below are optional { // How many concurrent connections will be made to S3 to retrieve key data diff --git a/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/EphemeralXmlRepository.cs b/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/EphemeralXmlRepository.cs new file mode 100644 index 0000000..c2185e0 --- /dev/null +++ b/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/EphemeralXmlRepository.cs @@ -0,0 +1,53 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using Microsoft.AspNet.DataProtection.Repositories; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Xml.Linq; + +namespace AspNetCore.DataProtection.Aws.IntegrationTests +{ + /// + /// Borrowed straight from https://github.com/aspnet/DataProtection/blob/master/src/Microsoft.AspNetCore.DataProtection/Repositories/EphemeralXmlRepository.cs + /// since Microsoft made this internal, which makes external testing that much harder + /// + internal class EphemeralXmlRepository : IXmlRepository + { + private readonly List _storedElements = new List(); + + public virtual IReadOnlyCollection GetAllElements() + { + // force complete enumeration under lock for thread safety + lock (_storedElements) + { + return GetAllElementsCore().ToList().AsReadOnly(); + } + } + + private IEnumerable GetAllElementsCore() + { + // this method must be called under lock + foreach (XElement element in _storedElements) + { + yield return new XElement(element); // makes a deep copy so caller doesn't inadvertently modify it + } + } + + public virtual void StoreElement(XElement element, string friendlyName) + { + if (element == null) + { + throw new ArgumentNullException(nameof(element)); + } + + XElement cloned = new XElement(element); // makes a deep copy so caller doesn't inadvertently modify it + + // under lock for thread safety + lock (_storedElements) + { + _storedElements.Add(cloned); + } + } + } +} diff --git a/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/KmsIntegrationTests.cs b/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/KmsIntegrationTests.cs index 982fb3b..05685c4 100644 --- a/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/KmsIntegrationTests.cs +++ b/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/KmsIntegrationTests.cs @@ -5,6 +5,7 @@ using Amazon.KeyManagementService.Model; using AspNetCore.DataProtection.Aws.Kms; using Microsoft.Extensions.DependencyInjection; +using System; using System.Threading; using System.Threading.Tasks; using System.Xml.Linq; @@ -12,15 +13,17 @@ namespace AspNetCore.DataProtection.Aws.IntegrationTests { - public class KmsIntegrationTests + public class KmsIntegrationTests : IDisposable { private readonly KmsXmlEncryptor encryptor; private readonly KmsXmlDecryptor decryptor; private readonly IAmazonKeyManagementService kmsClient; private readonly KmsXmlEncryptorConfig encryptConfig; - private const string ApplicationName = "hotchkj-test-app"; + internal const string ApplicationName = "hotchkj-test-app"; private const string ElementName = "name"; private const string ElementContent = "test"; + // Expectation that whatever key is in use has this alias + internal const string KmsTestingKey = "alias/KmsIntegrationTesting"; public KmsIntegrationTests() { @@ -31,8 +34,7 @@ public KmsIntegrationTests() // Expectation that local SDK has been configured correctly, whether via VS Tools or user config files kmsClient = new AmazonKeyManagementServiceClient(RegionEndpoint.EUWest1); - // Expectation that whatever key is in use has this alias - encryptConfig = new KmsXmlEncryptorConfig(ApplicationName, "alias/KmsIntegrationTesting"); + encryptConfig = new KmsXmlEncryptorConfig(ApplicationName, KmsTestingKey); encryptor = new KmsXmlEncryptor(kmsClient, encryptConfig, svcProvider); diff --git a/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/KmsManagerIntegrationTests.cs b/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/KmsManagerIntegrationTests.cs new file mode 100644 index 0000000..7600a8d --- /dev/null +++ b/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/KmsManagerIntegrationTests.cs @@ -0,0 +1,63 @@ +// Copyright(c) 2016 Jeff Hotchkiss +// Licensed under the MIT License. See License.md in the project root for license information. +using Amazon; +using Amazon.KeyManagementService; +using AspNetCore.DataProtection.Aws.Kms; +using Microsoft.AspNet.DataProtection.AuthenticatedEncryption; +using Microsoft.AspNet.DataProtection.AuthenticatedEncryption.ConfigurationModel; +using Microsoft.AspNet.DataProtection.KeyManagement; +using Microsoft.AspNet.DataProtection.Repositories; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.Linq; +using Xunit; + +namespace AspNetCore.DataProtection.Aws.IntegrationTests +{ + public class KmsManagerIntegrationTests : IDisposable + { + private readonly IAmazonKeyManagementService kmsClient; + + public KmsManagerIntegrationTests() + { + // Expectation that local SDK has been configured correctly, whether via VS Tools or user config files + kmsClient = new AmazonKeyManagementServiceClient(RegionEndpoint.EUWest1); + } + + public void Dispose() + { + kmsClient.Dispose(); + } + + [Fact] + public void ExpectFullKeyManagerStoreRetrieveToSucceed() + { + var config = new KmsXmlEncryptorConfig(KmsIntegrationTests.ApplicationName, KmsIntegrationTests.KmsTestingKey); + + var serviceCollection = new ServiceCollection(); + serviceCollection.AddInstance(kmsClient); + serviceCollection.AddInstance(new AuthenticatedEncryptorConfiguration(new AuthenticatedEncryptionOptions())); + serviceCollection.AddDataProtection(); + serviceCollection.ConfigureDataProtection(configure => + { + configure.ProtectKeysWithAwsKms(config); + }); + serviceCollection.AddInstance(new EphemeralXmlRepository()); + var serviceProvider = serviceCollection.BuildServiceProvider(); + + var keyManager = new XmlKeyManager(serviceProvider.GetRequiredService(), + serviceProvider.GetRequiredService(), + serviceProvider); + + var activationDate = new DateTimeOffset(new DateTime(1980, 1, 1)); + var expirationDate = new DateTimeOffset(new DateTime(1980, 6, 1)); + keyManager.CreateNewKey(activationDate, expirationDate); + + var keys = keyManager.GetAllKeys(); + + Assert.Equal(1, keys.Count); + Assert.Equal(activationDate, keys.Single().ActivationDate); + Assert.Equal(expirationDate, keys.Single().ExpirationDate); + } + } +} diff --git a/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/S3IntegrationTests.cs b/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/S3IntegrationTests.cs index 7d6f264..187fa54 100644 --- a/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/S3IntegrationTests.cs +++ b/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/S3IntegrationTests.cs @@ -5,7 +5,6 @@ using AspNetCore.DataProtection.Aws.S3; using System; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using System.Xml.Linq; @@ -22,13 +21,14 @@ public sealed class S3IntegrationTests : IDisposable private const int LargeTestNumber = 2100; private const string ElementName = "name"; private const string ElementContent = "test"; + // Sadly S3 bucket names are globally unique, so other testers without write access need to change this name + internal const string BucketName = "hotchkj-dataprotection-s3-integration-tests-eu-west-1"; public S3IntegrationTests() { // Expectation that local SDK has been configured correctly, whether via VS Tools or user config files s3client = new AmazonS3Client(RegionEndpoint.EUWest1); - // Sadly S3 bucket names are globally unique, so other testers without write access need to change this name - config = new S3XmlRepositoryConfig("hotchkj-dataprotection-s3-integration-tests-eu-west-1"); + config = new S3XmlRepositoryConfig(BucketName); xmlRepo = new S3XmlRepository(s3client, config); } @@ -47,7 +47,7 @@ public async Task PrepareLargeQueryTest() for (int i = 0; i < LargeTestNumber; ++i) { - await xmlRepo.StoreElementAsync(myXml, "LargeQueryTest" + i.ToString(), CancellationToken.None); + await xmlRepo.StoreElementAsync(myXml, "LargeQueryTest" + i, CancellationToken.None); } } finally diff --git a/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/S3ManagerIntegrationTests.cs b/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/S3ManagerIntegrationTests.cs new file mode 100644 index 0000000..8969ec1 --- /dev/null +++ b/integrate/AspNetCore.DataProtection.Aws.IntegrationTests/S3ManagerIntegrationTests.cs @@ -0,0 +1,88 @@ +// Copyright(c) 2016 Jeff Hotchkiss +// Licensed under the MIT License. See License.md in the project root for license information. +using Amazon; +using Amazon.S3; +using Amazon.S3.Model; +using AspNetCore.DataProtection.Aws.S3; +using Microsoft.AspNet.DataProtection.AuthenticatedEncryption; +using Microsoft.AspNet.DataProtection.AuthenticatedEncryption.ConfigurationModel; +using Microsoft.AspNet.DataProtection.KeyManagement; +using Microsoft.AspNet.DataProtection.Repositories; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.Linq; +using System.Threading.Tasks; +using Xunit; + +namespace AspNetCore.DataProtection.Aws.IntegrationTests +{ + public sealed class S3ManagerIntegrationTests : IDisposable + { + private readonly IAmazonS3 s3client; + + public S3ManagerIntegrationTests() + { + // Expectation that local SDK has been configured correctly, whether via VS Tools or user config files + s3client = new AmazonS3Client(RegionEndpoint.EUWest1); + } + + public void Dispose() + { + s3client.Dispose(); + } + + [Fact] + public async Task ExpectFullKeyManagerStoreRetrieveToSucceed() + { + var config = new S3XmlRepositoryConfig(S3IntegrationTests.BucketName); + config.KeyPrefix = "RealXmlKeyManager/"; + await ClearKeys(config.KeyPrefix); + + var serviceCollection = new ServiceCollection(); + serviceCollection.AddInstance(s3client); + serviceCollection.AddInstance(new AuthenticatedEncryptorConfiguration(new AuthenticatedEncryptionOptions())); + serviceCollection.AddDataProtection(); + serviceCollection.ConfigureDataProtection(configure => + { + configure.PersistKeysToAwsS3(config); + }); + var serviceProvider = serviceCollection.BuildServiceProvider(); + + var keyManager = new XmlKeyManager(serviceProvider.GetRequiredService(), + serviceProvider.GetRequiredService(), + serviceProvider); + + var activationDate = new DateTimeOffset(new DateTime(1980, 1, 1)); + var expirationDate = new DateTimeOffset(new DateTime(1980, 6, 1)); + keyManager.CreateNewKey(activationDate, expirationDate); + + var keys = keyManager.GetAllKeys(); + + Assert.Equal(1, keys.Count); + Assert.Equal(activationDate, keys.Single().ActivationDate); + Assert.Equal(expirationDate, keys.Single().ExpirationDate); + } + + private async Task ClearKeys(string prefix) + { + // XmlKeyManager uses a GUID for the naming so we cannot overwrite the same entry in the test + // Thus we must first clear out any keys that old tests put in + + var listed = await s3client.ListObjectsV2Async(new ListObjectsV2Request + { + BucketName = S3IntegrationTests.BucketName, + Prefix = prefix + }); + + // In sequence as we do not expect more than one or two of these assuming the tests work properly + foreach (var s3Obj in listed.S3Objects) + { + await s3client.DeleteObjectAsync(new DeleteObjectRequest + { + BucketName = S3IntegrationTests.BucketName, + Key = s3Obj.Key + }); + } + } + } +} diff --git a/src/AspNetCore.DataProtection.Aws.Kms/KmsConstants.cs b/src/AspNetCore.DataProtection.Aws.Kms/KmsConstants.cs index a2e8fa5..22302fe 100644 --- a/src/AspNetCore.DataProtection.Aws.Kms/KmsConstants.cs +++ b/src/AspNetCore.DataProtection.Aws.Kms/KmsConstants.cs @@ -2,7 +2,7 @@ // Licensed under the MIT License. See License.md in the project root for license information. namespace AspNetCore.DataProtection.Aws.Kms { - public class KmsConstants + public static class KmsConstants { /// /// Additional context supplied to all KMS operations used by AspNetCore.DataProtection.Aws.Kms diff --git a/src/AspNetCore.DataProtection.Aws.Kms/KmsXmlDecryptor.cs b/src/AspNetCore.DataProtection.Aws.Kms/KmsXmlDecryptor.cs index e90ff22..a23e04f 100644 --- a/src/AspNetCore.DataProtection.Aws.Kms/KmsXmlDecryptor.cs +++ b/src/AspNetCore.DataProtection.Aws.Kms/KmsXmlDecryptor.cs @@ -65,6 +65,8 @@ public XElement Decrypt(XElement encryptedElement) public async Task DecryptAsync(XElement encryptedElement, CancellationToken ct) { + _logger?.LogDebug("Decrypting ciphertext DataProtection key using AWS key {0}", Config.KeyId); + using (var memoryStream = new MemoryStream()) { var protectedKey = Convert.FromBase64String((string)encryptedElement.Element("value")); diff --git a/src/AspNetCore.DataProtection.Aws.Kms/KmsXmlEncryptor.cs b/src/AspNetCore.DataProtection.Aws.Kms/KmsXmlEncryptor.cs index dd56eb0..3ce2345 100644 --- a/src/AspNetCore.DataProtection.Aws.Kms/KmsXmlEncryptor.cs +++ b/src/AspNetCore.DataProtection.Aws.Kms/KmsXmlEncryptor.cs @@ -77,6 +77,8 @@ public EncryptedXmlInfo Encrypt(XElement plaintextElement) public async Task EncryptAsync(XElement plaintextElement, CancellationToken ct) { + _logger?.LogDebug("Encrypting plaintext DataProtection key using AWS key {0}", Config.KeyId); + // Some implementations of this e.g. DpapiXmlEncryptor go to enormous lengths to // create a memory stream, use unsafe code to zero it, and so on. // diff --git a/src/AspNetCore.DataProtection.Aws.S3/DataProtectionBuilderExtensions.cs b/src/AspNetCore.DataProtection.Aws.S3/DataProtectionBuilderExtensions.cs index 50bd378..90019d0 100644 --- a/src/AspNetCore.DataProtection.Aws.S3/DataProtectionBuilderExtensions.cs +++ b/src/AspNetCore.DataProtection.Aws.S3/DataProtectionBuilderExtensions.cs @@ -1,10 +1,10 @@ -using System; +// Copyright(c) 2016 Jeff Hotchkiss +// Licensed under the MIT License. See License.md in the project root for license information. +using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.AspNet.DataProtection; using Microsoft.AspNet.DataProtection.Repositories; using Amazon.S3; -// Copyright(c) 2016 Jeff Hotchkiss -// Licensed under the MIT License. See License.md in the project root for license information. namespace AspNetCore.DataProtection.Aws.S3 { diff --git a/src/AspNetCore.DataProtection.Aws.S3/S3XmlRepository.cs b/src/AspNetCore.DataProtection.Aws.S3/S3XmlRepository.cs index 990f699..b8bd509 100644 --- a/src/AspNetCore.DataProtection.Aws.S3/S3XmlRepository.cs +++ b/src/AspNetCore.DataProtection.Aws.S3/S3XmlRepository.cs @@ -122,6 +122,7 @@ private async Task GetElementFromKey(S3Object item, SemaphoreSlim thro try { + _logger?.LogDebug("Retrieving DataProtection key at S3 location {0} in bucket {1}", item.Key, Config.Bucket); var gr = new GetObjectRequest { BucketName = Config.Bucket, @@ -162,11 +163,12 @@ public async Task StoreElementAsync(XElement element, string friendlyName, Cance if (!IsSafeS3Key(friendlyName)) { key = Config.KeyPrefix + Guid.NewGuid() + ".xml"; - _logger?.LogWarning("DataProtection key friendly name {0} is not safe for S3, ignoring and using key {1}", friendlyName, key); + _logger?.LogWarning("Storing DataProtection key with friendly name {0} is not safe for S3. Ignoring and storing at S3 location {1} in bucket {2}", friendlyName, key, Config.Bucket); } else { key = Config.KeyPrefix + friendlyName + ".xml"; + _logger?.LogDebug("Storing DataProtection key at S3 location {0} in bucket {1}", key, Config.Bucket); } using (var stream = new MemoryStream()) diff --git a/test/AspNetCore.DataProtection.Aws.Tests/KmsDecryptorTests.cs b/test/AspNetCore.DataProtection.Aws.Tests/KmsDecryptorTests.cs index 4feac4d..737dda2 100644 --- a/test/AspNetCore.DataProtection.Aws.Tests/KmsDecryptorTests.cs +++ b/test/AspNetCore.DataProtection.Aws.Tests/KmsDecryptorTests.cs @@ -19,7 +19,6 @@ public class KmsDecryptorTests : IDisposable private readonly KmsXmlDecryptor decryptor; private readonly MockRepository repository; private readonly Mock kmsClient; - private readonly Mock serviceProvider; private readonly KmsXmlEncryptorConfig encryptConfig; private const string AppName = "appName"; private const string KeyId = "keyId"; @@ -31,7 +30,7 @@ public KmsDecryptorTests() repository = new MockRepository(MockBehavior.Strict); kmsClient = repository.Create(); - serviceProvider = repository.Create(); + var serviceProvider = repository.Create(); serviceProvider.Setup(x => x.GetService(typeof(KmsXmlEncryptorConfig))) .Returns(encryptConfig);