resilien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add support for SSE-C to S3 backend 

These changes add support for encrypting terraform remote-state in S3 using customer-supplied encryption keys (SSE-C).
This commit is contained in:
Brian Williams 2019-07-05 10:54:07 -05:00
parent 805ae28876
commit 5e3c3bafb8
5 changed files with 158 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
backend/remote-state/s3/backend.go
View File

@ -2,7 +2,9 @@ package s3
import (
"context"
"encoding/base64"
"errors"
"fmt"
"strings"
"github.com/aws/aws-sdk-go/aws"
@ -185,6 +187,21 @@ func New() backend.Backend {
Default: false,
},
"sse_customer_key": {
Type: schema.TypeString,
Optional: true,
Description: "The base64-encoded encryption key to use for server-side encryption with customer-provided keys (SSE-C).",
DefaultFunc: schema.EnvDefaultFunc("AWS_SSE_CUSTOMER_KEY", ""),
Sensitive: true,
ValidateFunc: func(v interface{}, s string) ([]string, []error) {
key := v.(string)
if key != "" && len(key) != 44 {
return nil, []error{errors.New("sse_customer_key must be 44 characters in length (256 bits, base64 encoded)")}
}
return nil, nil
},
},
"role_arn": {
Type: schema.TypeString,
Optional: true,
@ -255,13 +272,14 @@ type Backend struct {
s3Client *s3.S3
dynClient *dynamodb.DynamoDB
bucketName string
keyName string
serverSideEncryption bool
acl string
kmsKeyID string
ddbTable string
workspaceKeyPrefix string
bucketName string
keyName string
serverSideEncryption bool
customerEncryptionKey []byte
acl string
kmsKeyID string
ddbTable string
workspaceKeyPrefix string
}
func (b *Backend) configure(ctx context.Context) error {
@ -280,10 +298,23 @@ func (b *Backend) configure(ctx context.Context) error {
b.bucketName = data.Get("bucket").(string)
b.keyName = data.Get("key").(string)
b.serverSideEncryption = data.Get("encrypt").(bool)
b.acl = data.Get("acl").(string)
b.kmsKeyID = data.Get("kms_key_id").(string)
b.workspaceKeyPrefix = data.Get("workspace_key_prefix").(string)
b.serverSideEncryption = data.Get("encrypt").(bool)
b.kmsKeyID = data.Get("kms_key_id").(string)
customerKeyString := data.Get("sse_customer_key").(string)
if customerKeyString != "" {
if b.kmsKeyID != "" {
return errors.New(encryptionKeyConflictError)
}
var err error
b.customerEncryptionKey, err = base64.StdEncoding.DecodeString(customerKeyString)
if err != nil {
return fmt.Errorf("Failed to decode sse_customer_key: %s", err.Error())
}
}
b.ddbTable = data.Get("dynamodb_table").(string)
if b.ddbTable == "" {
@ -330,3 +361,9 @@ func (b *Backend) configure(ctx context.Context) error {
return nil
}
const encryptionKeyConflictError = `Cannot have both kms_key_id and sse_customer_key set.
The kms_key_id is used for encryption with KMS-Managed Keys (SSE-KMS)
while sse_customer_key is used for encryption with customer-managed keys (SSE-C).
Please choose one or the other.`

17
backend/remote-state/s3/backend_state.go
View File

@ -108,14 +108,15 @@ func (b *Backend) remoteClient(name string) (*RemoteClient, error) {
}
client := &RemoteClient{
s3Client: b.s3Client,
dynClient: b.dynClient,
bucketName: b.bucketName,
path: b.path(name),
serverSideEncryption: b.serverSideEncryption,
acl: b.acl,
kmsKeyID: b.kmsKeyID,
ddbTable: b.ddbTable,
s3Client: b.s3Client,
dynClient: b.dynClient,
bucketName: b.bucketName,
path: b.path(name),
serverSideEncryption: b.serverSideEncryption,
customerEncryptionKey: b.customerEncryptionKey,
acl: b.acl,
kmsKeyID: b.kmsKeyID,
ddbTable: b.ddbTable,
}
return client, nil

69
backend/remote-state/s3/backend_test.go
View File

@ -82,6 +82,58 @@ func TestBackendConfig_invalidKey(t *testing.T) {
}
}
func TestBackendConfig_invalidSSECustomerKeyLength(t *testing.T) {
testACC(t)
cfg := hcl2shim.HCL2ValueFromConfigValue(map[string]interface{}{
"region": "us-west-1",
"bucket": "tf-test",
"encrypt": true,
"key": "state",
"dynamodb_table": "dynamoTable",
"sse_customer_key": "key",
})
_, diags := New().PrepareConfig(cfg)
if !diags.HasErrors() {
t.Fatal("expected error for invalid sse_customer_key length")
}
}
func TestBackendConfig_invalidSSECustomerKeyEncoding(t *testing.T) {
testACC(t)
cfg := hcl2shim.HCL2ValueFromConfigValue(map[string]interface{}{
"region": "us-west-1",
"bucket": "tf-test",
"encrypt": true,
"key": "state",
"dynamodb_table": "dynamoTable",
"sse_customer_key": "====CT70aTYB2JGff7AjQtwbiLkwH4npICay1PWtmdka",
})
diags := New().Configure(cfg)
if !diags.HasErrors() {
t.Fatal("expected error for failing to decode sse_customer_key")
}
}
func TestBackendConfig_conflictingEncryptionSchema(t *testing.T) {
testACC(t)
cfg := hcl2shim.HCL2ValueFromConfigValue(map[string]interface{}{
"region": "us-west-1",
"bucket": "tf-test",
"key": "state",
"encrypt": true,
"dynamodb_table": "dynamoTable",
"sse_customer_key": "1hwbcNPGWL+AwDiyGmRidTWAEVmCWMKbEHA+Es8w75o=",
"kms_key_id": "arn:aws:kms:us-west-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab",
})
diags := New().Configure(cfg)
if !diags.HasErrors() {
t.Fatal("expected error for simultaneous usage of kms_key_id and sse_customer_key")
}
}
func TestBackend(t *testing.T) {
testACC(t)
@ -129,6 +181,23 @@ func TestBackendLocked(t *testing.T) {
backend.TestBackendStateForceUnlock(t, b1, b2)
}
func TestBackendSSECustomerKey(t *testing.T) {
testACC(t)
bucketName := fmt.Sprintf("terraform-remote-s3-test-%x", time.Now().Unix())
b := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
"bucket": bucketName,
"encrypt": true,
"key": "test-SSE-C",
"sse_customer_key": "4Dm1n4rphuFgawxuzY/bEfvLf6rYK0gIjfaDSLlfXNk=",
})).(*Backend)
createS3Bucket(t, b.s3Client, bucketName)
defer deleteS3Bucket(t, b.s3Client, bucketName)
backend.TestBackendStates(t, b)
}
// add some extra junk in S3 to try and confuse the env listing.
func TestBackendExtraPaths(t *testing.T) {
testACC(t)

42
backend/remote-state/s3/client.go
View File

@ -3,6 +3,7 @@ package s3
import (
"bytes"
"crypto/md5"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
@ -23,19 +24,21 @@ import (
// Store the last saved serial in dynamo with this suffix for consistency checks.
const (
s3EncryptionAlgorithm = "AES256"
stateIDSuffix = "-md5"
s3ErrCodeInternalError = "InternalError"
)
type RemoteClient struct {
s3Client *s3.S3
dynClient *dynamodb.DynamoDB
bucketName string
path string
serverSideEncryption bool
acl string
kmsKeyID string
ddbTable string
s3Client *s3.S3
dynClient *dynamodb.DynamoDB
bucketName string
path string
serverSideEncryption bool
customerEncryptionKey []byte
acl string
kmsKeyID string
ddbTable string
}
var (
@ -98,10 +101,18 @@ func (c *RemoteClient) get() (*remote.Payload, error) {
var output *s3.GetObjectOutput
var err error
output, err = c.s3Client.GetObject(&s3.GetObjectInput{
input := &s3.GetObjectInput{
Bucket: &c.bucketName,
Key: &c.path,
})
}
if c.serverSideEncryption && c.customerEncryptionKey != nil {
input.SetSSECustomerKey(string(c.customerEncryptionKey))
input.SetSSECustomerAlgorithm(s3EncryptionAlgorithm)
input.SetSSECustomerKeyMD5(c.getSSECustomerKeyMD5())
}
output, err = c.s3Client.GetObject(input)
if err != nil {
if awserr, ok := err.(awserr.Error); ok {
@ -152,8 +163,12 @@ func (c *RemoteClient) Put(data []byte) error {
if c.kmsKeyID != "" {
i.SSEKMSKeyId = &c.kmsKeyID
i.ServerSideEncryption = aws.String("aws:kms")
} else if c.customerEncryptionKey != nil {
i.SetSSECustomerKey(string(c.customerEncryptionKey))
i.SetSSECustomerAlgorithm(s3EncryptionAlgorithm)
i.SetSSECustomerKeyMD5(c.getSSECustomerKeyMD5())
} else {
i.ServerSideEncryption = aws.String("AES256")
i.ServerSideEncryption = aws.String(s3EncryptionAlgorithm)
}
}
@ -383,6 +398,11 @@ func (c *RemoteClient) lockPath() string {
return fmt.Sprintf("%s/%s", c.bucketName, c.path)
}
func (c *RemoteClient) getSSECustomerKeyMD5() string {
b := md5.Sum(c.customerEncryptionKey)
return base64.StdEncoding.EncodeToString(b[:])
}
const errBadChecksumFmt = `state data in S3 does not have the expected content.
This may be caused by unusually long delays in S3 processing a previous state

3
website/docs/backends/types/s3.html.md
View File

@ -190,6 +190,9 @@ The following configuration options or environment variables are supported:
* `skip_region_validation` - (**DEPRECATED**, Optional) Skip validation of provided region name.
* `skip_requesting_account_id` - (**DEPRECATED**, Optional) Skip requesting the account ID.
* `skip_metadata_api_check` - (Optional) Skip the AWS Metadata API check.
* `sse_customer_key` / `AWS_SSE_CUSTOMER_KEY` - (Optional) The key to use for encrypting state with [Server-Side Encryption with Customer-Provided Keys (SSE-C)](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html).
This is the base64-encoded value of the key, which must decode to 256 bits. Due to the sensitivity of the value, it is recommended to set it using the `AWS_SSE_CUSTOMER_KEY` environment variable.
Setting it inside a terraform file will cause it to be persisted to disk in `terraform.tfstate`.
* `max_retries` - (Optional) The maximum number of times an AWS API request is retried on retryable failure. Defaults to 5.
## Multi-account AWS Architecture