Merge pull request #4 from hotchkj/add-xmlkeymanager-tests

Fixes #3. Adds XmlKeyManager tests for S3 & KMS integrations so that …

README.md

@@ -2,13 +2,15 @@
 Amazon Web Services integration for ASP.NET Core data protection.
 Server keys can be stored in S3 and encrypted using KMS.
 
-[![Build status](https://ci.appveyor.com/api/projects/status/5k00d5fdfspjv20e?svg=true)](https://ci.appveyor.com/project/hotchkj/aspnetcore-dataprotection-aws)
+[![Build status](https://ci.appveyor.com/api/projects/status/5k00d5fdfspjv20e/branch/master?svg=true)](https://ci.appveyor.com/project/hotchkj/aspnetcore-dataprotection-aws/branch/master)
 
 ## S3 Persistence
 By default, ASP.NET Core stores encryption keys locally which causes issues with key mismatches across server farms. S3 can be used instead of a shared filesystem to provide key storage.
 
 Server-side S3 encryption of AES256 is enabled by default for all keys written to S3. It remains the client's responsibility to ensure access control to the S3 bucket is appropriately configured.
 
+[Guidance](https://github.com/aspnet/DataProtection/issues/158) from Microsoft indicates that the repository itself cannot clean up key data as the usage lifetime is not known to the key management layer. If S3 usage over time is a concern, clients need to trade off key lifetime (and corresponding revocation lifetime) vs S3 storage costs. A suitable approach might be S3 lifecycle policies to remove ancient key files that could not possibly be in use in the client's deployed scenario. Key files generated by typical XmlKeyManager runs are less than 1kB each.
+
 ### Configuration
 In Startup.cs, specified as part of DataProtection configuration:
 ```csharp
@@ -17,7 +19,7 @@ public void ConfigureServices(IServiceCollection services)
     services.AddDataProtection();
     services.ConfigureDataProtection(configure =>
     {
-        configure.PersistKeysToS3(new AmazonS3Client(), new S3XmlRepositoryConfig("my-bucket-name")
+        configure.PersistKeysToAwsS3(new AmazonS3Client(), new S3XmlRepositoryConfig("my-bucket-name")
         // Configuration has defaults; all below are optional
         {
             // How many concurrent connections will be made to S3 to retrieve key data

integrate/AspNetCore.DataProtection.Aws.IntegrationTests/EphemeralXmlRepository.cs

@@ -0,0 +1,53 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using Microsoft.AspNet.DataProtection.Repositories;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Xml.Linq;
+
+namespace AspNetCore.DataProtection.Aws.IntegrationTests
+{
+    /// <summary>
+    /// Borrowed straight from https://github.com/aspnet/DataProtection/blob/master/src/Microsoft.AspNetCore.DataProtection/Repositories/EphemeralXmlRepository.cs
+    /// since Microsoft made this internal, which makes external testing that much harder
+    /// </summary>
+    internal class EphemeralXmlRepository : IXmlRepository
+    {
+        private readonly List<XElement> _storedElements = new List<XElement>();
+
+        public virtual IReadOnlyCollection<XElement> GetAllElements()
+        {
+            // force complete enumeration under lock for thread safety
+            lock (_storedElements)
+            {
+                return GetAllElementsCore().ToList().AsReadOnly();
+            }
+        }
+
+        private IEnumerable<XElement> GetAllElementsCore()
+        {
+            // this method must be called under lock
+            foreach (XElement element in _storedElements)
+            {
+                yield return new XElement(element); // makes a deep copy so caller doesn't inadvertently modify it
+            }
+        }
+
+        public virtual void StoreElement(XElement element, string friendlyName)
+        {
+            if (element == null)
+            {
+                throw new ArgumentNullException(nameof(element));
+            }
+
+            XElement cloned = new XElement(element); // makes a deep copy so caller doesn't inadvertently modify it
+
+            // under lock for thread safety
+            lock (_storedElements)
+            {
+                _storedElements.Add(cloned);
+            }
+        }
+    }
+}

integrate/AspNetCore.DataProtection.Aws.IntegrationTests/KmsIntegrationTests.cs

@@ -5,22 +5,25 @@
 using Amazon.KeyManagementService.Model;
 using AspNetCore.DataProtection.Aws.Kms;
 using Microsoft.Extensions.DependencyInjection;
+using System;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Xml.Linq;
 using Xunit;
 
 namespace AspNetCore.DataProtection.Aws.IntegrationTests
 {
-    public class KmsIntegrationTests
+    public class KmsIntegrationTests : IDisposable
     {
         private readonly KmsXmlEncryptor encryptor;
         private readonly KmsXmlDecryptor decryptor;
         private readonly IAmazonKeyManagementService kmsClient;
         private readonly KmsXmlEncryptorConfig encryptConfig;
-        private const string ApplicationName = "hotchkj-test-app";
+        internal const string ApplicationName = "hotchkj-test-app";
         private const string ElementName = "name";
         private const string ElementContent = "test";
+        // Expectation that whatever key is in use has this alias
+        internal const string KmsTestingKey = "alias/KmsIntegrationTesting";
 
         public KmsIntegrationTests()
         {
@@ -31,8 +34,7 @@ public KmsIntegrationTests()
 
             // Expectation that local SDK has been configured correctly, whether via VS Tools or user config files
             kmsClient = new AmazonKeyManagementServiceClient(RegionEndpoint.EUWest1);
-            // Expectation that whatever key is in use has this alias
-            encryptConfig = new KmsXmlEncryptorConfig(ApplicationName, "alias/KmsIntegrationTesting");
+            encryptConfig = new KmsXmlEncryptorConfig(ApplicationName, KmsTestingKey);
 
             encryptor = new KmsXmlEncryptor(kmsClient, encryptConfig, svcProvider);
 

integrate/AspNetCore.DataProtection.Aws.IntegrationTests/KmsManagerIntegrationTests.cs

@@ -0,0 +1,63 @@
+// Copyright(c) 2016 Jeff Hotchkiss
+// Licensed under the MIT License. See License.md in the project root for license information.
+using Amazon;
+using Amazon.KeyManagementService;
+using AspNetCore.DataProtection.Aws.Kms;
+using Microsoft.AspNet.DataProtection.AuthenticatedEncryption;
+using Microsoft.AspNet.DataProtection.AuthenticatedEncryption.ConfigurationModel;
+using Microsoft.AspNet.DataProtection.KeyManagement;
+using Microsoft.AspNet.DataProtection.Repositories;
+using Microsoft.Extensions.DependencyInjection;
+using System;
+using System.Linq;
+using Xunit;
+
+namespace AspNetCore.DataProtection.Aws.IntegrationTests
+{
+    public class KmsManagerIntegrationTests : IDisposable
+    {
+        private readonly IAmazonKeyManagementService kmsClient;
+
+        public KmsManagerIntegrationTests()
+        {
+            // Expectation that local SDK has been configured correctly, whether via VS Tools or user config files
+            kmsClient = new AmazonKeyManagementServiceClient(RegionEndpoint.EUWest1);
+        }
+
+        public void Dispose()
+        {
+            kmsClient.Dispose();
+        }
+
+        [Fact]
+        public void ExpectFullKeyManagerStoreRetrieveToSucceed()
+        {
+            var config = new KmsXmlEncryptorConfig(KmsIntegrationTests.ApplicationName, KmsIntegrationTests.KmsTestingKey);
+
+            var serviceCollection = new ServiceCollection();
+            serviceCollection.AddInstance(kmsClient);
+            serviceCollection.AddInstance<IAuthenticatedEncryptorConfiguration>(new AuthenticatedEncryptorConfiguration(new AuthenticatedEncryptionOptions()));
+            serviceCollection.AddDataProtection();
+            serviceCollection.ConfigureDataProtection(configure =>
+            {
+                configure.ProtectKeysWithAwsKms(config);
+            });
+            serviceCollection.AddInstance<IXmlRepository>(new EphemeralXmlRepository());
+            var serviceProvider = serviceCollection.BuildServiceProvider();
+
+            var keyManager = new XmlKeyManager(serviceProvider.GetRequiredService<IXmlRepository>(),
+                                               serviceProvider.GetRequiredService<IAuthenticatedEncryptorConfiguration>(),
+                                               serviceProvider);
+
+            var activationDate = new DateTimeOffset(new DateTime(1980, 1, 1));
+            var expirationDate = new DateTimeOffset(new DateTime(1980, 6, 1));
+            keyManager.CreateNewKey(activationDate, expirationDate);
+
+            var keys = keyManager.GetAllKeys();
+
+            Assert.Equal(1, keys.Count);
+            Assert.Equal(activationDate, keys.Single().ActivationDate);
+            Assert.Equal(expirationDate, keys.Single().ExpirationDate);
+        }
+    }
+}

integrate/AspNetCore.DataProtection.Aws.IntegrationTests/S3IntegrationTests.cs

@@ -5,7 +5,6 @@
 using AspNetCore.DataProtection.Aws.S3;
 using System;
 using System.Linq;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Xml.Linq;
@@ -22,13 +21,14 @@ public sealed class S3IntegrationTests : IDisposable
         private const int LargeTestNumber = 2100;
         private const string ElementName = "name";
         private const string ElementContent = "test";
+        // Sadly S3 bucket names are globally unique, so other testers without write access need to change this name
+        internal const string BucketName = "hotchkj-dataprotection-s3-integration-tests-eu-west-1";
 
         public S3IntegrationTests()
         {
             // Expectation that local SDK has been configured correctly, whether via VS Tools or user config files
             s3client = new AmazonS3Client(RegionEndpoint.EUWest1);
-            // Sadly S3 bucket names are globally unique, so other testers without write access need to change this name
-            config = new S3XmlRepositoryConfig("hotchkj-dataprotection-s3-integration-tests-eu-west-1");
+            config = new S3XmlRepositoryConfig(BucketName);
             xmlRepo = new S3XmlRepository(s3client, config);
         }
 
@@ -47,7 +47,7 @@ public async Task PrepareLargeQueryTest()
 
                 for (int i = 0; i < LargeTestNumber; ++i)
                 {
-                    await xmlRepo.StoreElementAsync(myXml, "LargeQueryTest" + i.ToString(), CancellationToken.None);
+                    await xmlRepo.StoreElementAsync(myXml, "LargeQueryTest" + i, CancellationToken.None);
                 }
             }
             finally

integrate/AspNetCore.DataProtection.Aws.IntegrationTests/S3ManagerIntegrationTests.cs

@@ -0,0 +1,88 @@
+// Copyright(c) 2016 Jeff Hotchkiss
+// Licensed under the MIT License. See License.md in the project root for license information.
+using Amazon;
+using Amazon.S3;
+using Amazon.S3.Model;
+using AspNetCore.DataProtection.Aws.S3;
+using Microsoft.AspNet.DataProtection.AuthenticatedEncryption;
+using Microsoft.AspNet.DataProtection.AuthenticatedEncryption.ConfigurationModel;
+using Microsoft.AspNet.DataProtection.KeyManagement;
+using Microsoft.AspNet.DataProtection.Repositories;
+using Microsoft.Extensions.DependencyInjection;
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Xunit;
+
+namespace AspNetCore.DataProtection.Aws.IntegrationTests
+{
+    public sealed class S3ManagerIntegrationTests : IDisposable
+    {
+        private readonly IAmazonS3 s3client;
+
+        public S3ManagerIntegrationTests()
+        {
+            // Expectation that local SDK has been configured correctly, whether via VS Tools or user config files
+            s3client = new AmazonS3Client(RegionEndpoint.EUWest1);
+        }
+
+        public void Dispose()
+        {
+            s3client.Dispose();
+        }
+
+        [Fact]
+        public async Task ExpectFullKeyManagerStoreRetrieveToSucceed()
+        {
+            var config = new S3XmlRepositoryConfig(S3IntegrationTests.BucketName);
+            config.KeyPrefix = "RealXmlKeyManager/";
+            await ClearKeys(config.KeyPrefix);
+
+            var serviceCollection = new ServiceCollection();
+            serviceCollection.AddInstance(s3client);
+            serviceCollection.AddInstance<IAuthenticatedEncryptorConfiguration>(new AuthenticatedEncryptorConfiguration(new AuthenticatedEncryptionOptions()));
+            serviceCollection.AddDataProtection();
+            serviceCollection.ConfigureDataProtection(configure =>
+            {
+                configure.PersistKeysToAwsS3(config);
+            });
+            var serviceProvider = serviceCollection.BuildServiceProvider();
+
+            var keyManager = new XmlKeyManager(serviceProvider.GetRequiredService<IXmlRepository>(),
+                                               serviceProvider.GetRequiredService<IAuthenticatedEncryptorConfiguration>(),
+                                               serviceProvider);
+
+            var activationDate = new DateTimeOffset(new DateTime(1980, 1, 1));
+            var expirationDate = new DateTimeOffset(new DateTime(1980, 6, 1));
+            keyManager.CreateNewKey(activationDate, expirationDate);
+
+            var keys = keyManager.GetAllKeys();
+
+            Assert.Equal(1, keys.Count);
+            Assert.Equal(activationDate, keys.Single().ActivationDate);
+            Assert.Equal(expirationDate, keys.Single().ExpirationDate);
+        }
+
+        private async Task ClearKeys(string prefix)
+        {
+            // XmlKeyManager uses a GUID for the naming so we cannot overwrite the same entry in the test
+            // Thus we must first clear out any keys that old tests put in
+
+            var listed = await s3client.ListObjectsV2Async(new ListObjectsV2Request
+            {
+                BucketName = S3IntegrationTests.BucketName,
+                Prefix = prefix
+            });
+
+            // In sequence as we do not expect more than one or two of these assuming the tests work properly
+            foreach (var s3Obj in listed.S3Objects)
+            {
+                await s3client.DeleteObjectAsync(new DeleteObjectRequest
+                {
+                    BucketName = S3IntegrationTests.BucketName,
+                    Key = s3Obj.Key
+                });
+            }
+        }
+    }
+}

src/AspNetCore.DataProtection.Aws.Kms/KmsConstants.cs

@@ -2,7 +2,7 @@
 // Licensed under the MIT License. See License.md in the project root for license information.
 namespace AspNetCore.DataProtection.Aws.Kms
 {
-    public class KmsConstants
+    public static class KmsConstants
     {
         /// <summary>
         /// Additional context supplied to all KMS operations used by AspNetCore.DataProtection.Aws.Kms

src/AspNetCore.DataProtection.Aws.Kms/KmsXmlDecryptor.cs

@@ -65,6 +65,8 @@ public XElement Decrypt(XElement encryptedElement)
 
         public async Task<XElement> DecryptAsync(XElement encryptedElement, CancellationToken ct)
         {
+            _logger?.LogDebug("Decrypting ciphertext DataProtection key using AWS key {0}", Config.KeyId);
+
             using (var memoryStream = new MemoryStream())
             {
                 var protectedKey = Convert.FromBase64String((string)encryptedElement.Element("value"));

src/AspNetCore.DataProtection.Aws.Kms/KmsXmlEncryptor.cs

@@ -77,6 +77,8 @@ public EncryptedXmlInfo Encrypt(XElement plaintextElement)
 
         public async Task<EncryptedXmlInfo> EncryptAsync(XElement plaintextElement, CancellationToken ct)
         {
+            _logger?.LogDebug("Encrypting plaintext DataProtection key using AWS key {0}", Config.KeyId);
+
             // Some implementations of this e.g. DpapiXmlEncryptor go to enormous lengths to
             // create a memory stream, use unsafe code to zero it, and so on.
             //

src/AspNetCore.DataProtection.Aws.S3/DataProtectionBuilderExtensions.cs

@@ -1,10 +1,10 @@
-using System;
+// Copyright(c) 2016 Jeff Hotchkiss
+// Licensed under the MIT License. See License.md in the project root for license information.
+using System;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.AspNet.DataProtection;
 using Microsoft.AspNet.DataProtection.Repositories;
 using Amazon.S3;
-// Copyright(c) 2016 Jeff Hotchkiss
-// Licensed under the MIT License. See License.md in the project root for license information.
 
 namespace AspNetCore.DataProtection.Aws.S3
 {

src/AspNetCore.DataProtection.Aws.S3/S3XmlRepository.cs

@@ -122,6 +122,7 @@ private async Task<XElement> GetElementFromKey(S3Object item, SemaphoreSlim thro
 
             try
             {
+                _logger?.LogDebug("Retrieving DataProtection key at S3 location {0} in bucket {1}", item.Key, Config.Bucket);
                 var gr = new GetObjectRequest
                 {
                     BucketName = Config.Bucket,
@@ -162,11 +163,12 @@ public async Task StoreElementAsync(XElement element, string friendlyName, Cance
             if (!IsSafeS3Key(friendlyName))
             {
                 key = Config.KeyPrefix + Guid.NewGuid() + ".xml";
-                _logger?.LogWarning("DataProtection key friendly name {0} is not safe for S3, ignoring and using key {1}", friendlyName, key);
+                _logger?.LogWarning("Storing DataProtection key with friendly name {0} is not safe for S3. Ignoring and storing at S3 location {1} in bucket {2}", friendlyName, key, Config.Bucket);
             }
             else
             {
                 key = Config.KeyPrefix + friendlyName + ".xml";
+                _logger?.LogDebug("Storing DataProtection key at S3 location {0} in bucket {1}", key, Config.Bucket);
             }
 
             using (var stream = new MemoryStream())

test/AspNetCore.DataProtection.Aws.Tests/KmsDecryptorTests.cs

@@ -19,7 +19,6 @@ public class KmsDecryptorTests : IDisposable
         private readonly KmsXmlDecryptor decryptor;
         private readonly MockRepository repository;
         private readonly Mock<IAmazonKeyManagementService> kmsClient;
-        private readonly Mock<IServiceProvider> serviceProvider;
         private readonly KmsXmlEncryptorConfig encryptConfig;
         private const string AppName = "appName";
         private const string KeyId = "keyId";
@@ -31,7 +30,7 @@ public KmsDecryptorTests()
 
             repository = new MockRepository(MockBehavior.Strict);
             kmsClient = repository.Create<IAmazonKeyManagementService>();
-            serviceProvider = repository.Create<IServiceProvider>();
+            var serviceProvider = repository.Create<IServiceProvider>();
 
             serviceProvider.Setup(x => x.GetService(typeof(KmsXmlEncryptorConfig)))
                            .Returns(encryptConfig);