Merge pull request #16936 from negz/gcskeys

Support 'customer supplied encryption keys' in the GCS backend

backend/remote-state/gcs/backend.go

@@ -3,6 +3,7 @@ package gcs
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -30,6 +31,8 @@ type gcsBackend struct {
 	prefix           string
 	defaultStateFile string
 
+	encryptionKey []byte
+
 	projectID string
 	region    string
 }
@@ -65,6 +68,13 @@ func New() backend.Backend {
 				Default:     "",
 			},
 
+			"encryption_key": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "A 32 byte base64 encoded 'customer supplied encryption key' used to encrypt all state.",
+				Default:     "",
+			},
+
 			"project": {
 				Type:        schema.TypeString,
 				Optional:    true,
@@ -154,6 +164,30 @@ func (b *gcsBackend) configure(ctx context.Context) error {
 
 	b.storageClient = client
 
+	key := data.Get("encryption_key").(string)
+	if key == "" {
+		key = os.Getenv("GOOGLE_ENCRYPTION_KEY")
+	}
+
+	if key != "" {
+		kc, _, err := pathorcontents.Read(key)
+		if err != nil {
+			return fmt.Errorf("Error loading encryption key: %s", err)
+		}
+
+		// The GCS client expects a customer supplied encryption key to be
+		// passed in as a 32 byte long byte slice. The byte slice is base64
+		// encoded before being passed to the API. We take a base64 encoded key
+		// to remain consistent with the GCS docs.
+		// https://cloud.google.com/storage/docs/encryption#customer-supplied
+		// https://github.com/GoogleCloudPlatform/google-cloud-go/blob/def681/storage/storage.go#L1181
+		k, err := base64.StdEncoding.DecodeString(kc)
+		if err != nil {
+			return fmt.Errorf("Error decoding encryption key: %s", err)
+		}
+		b.encryptionKey = k
+	}
+
 	return nil
 }
 

backend/remote-state/gcs/backend_state.go

@@ -79,6 +79,7 @@ func (b *gcsBackend) client(name string) (*remoteClient, error) {
 		bucketName:     b.bucketName,
 		stateFilePath:  b.stateFile(name),
 		lockFilePath:   b.lockFile(name),
+		encryptionKey:  b.encryptionKey,
 	}, nil
 }
 

backend/remote-state/gcs/backend_test.go

@@ -13,7 +13,13 @@ import (
 	"github.com/hashicorp/terraform/state/remote"
 )
 
-const noPrefix = ""
+const (
+	noPrefix        = ""
+	noEncryptionKey = ""
+)
+
+// See https://cloud.google.com/storage/docs/using-encryption-keys#generating_your_own_encryption_key
+var encryptionKey = "yRyCOikXi1ZDNE0xN3yiFsJjg7LGimoLrGFcLZgQoVk="
 
 func TestStateFile(t *testing.T) {
 	t.Parallel()
@@ -52,7 +58,26 @@ func TestRemoteClient(t *testing.T) {
 	t.Parallel()
 
 	bucket := bucketName(t)
-	be := setupBackend(t, bucket, noPrefix)
+	be := setupBackend(t, bucket, noPrefix, noEncryptionKey)
+	defer teardownBackend(t, be, noPrefix)
+
+	ss, err := be.State(backend.DefaultStateName)
+	if err != nil {
+		t.Fatalf("be.State(%q) = %v", backend.DefaultStateName, err)
+	}
+
+	rs, ok := ss.(*remote.State)
+	if !ok {
+		t.Fatalf("be.State(): got a %T, want a *remote.State", ss)
+	}
+
+	remote.TestClient(t, rs.Client)
+}
+func TestRemoteClientWithEncryption(t *testing.T) {
+	t.Parallel()
+
+	bucket := bucketName(t)
+	be := setupBackend(t, bucket, noPrefix, encryptionKey)
 	defer teardownBackend(t, be, noPrefix)
 
 	ss, err := be.State(backend.DefaultStateName)
@@ -72,7 +97,7 @@ func TestRemoteLocks(t *testing.T) {
 	t.Parallel()
 
 	bucket := bucketName(t)
-	be := setupBackend(t, bucket, noPrefix)
+	be := setupBackend(t, bucket, noPrefix, noEncryptionKey)
 	defer teardownBackend(t, be, noPrefix)
 
 	remoteClient := func() (remote.Client, error) {
@@ -106,10 +131,10 @@ func TestBackend(t *testing.T) {
 
 	bucket := bucketName(t)
 
-	be0 := setupBackend(t, bucket, noPrefix)
+	be0 := setupBackend(t, bucket, noPrefix, noEncryptionKey)
 	defer teardownBackend(t, be0, noPrefix)
 
-	be1 := setupBackend(t, bucket, noPrefix)
+	be1 := setupBackend(t, bucket, noPrefix, noEncryptionKey)
 
 	backend.TestBackend(t, be0, be1)
 }
@@ -119,16 +144,28 @@ func TestBackendWithPrefix(t *testing.T) {
 	prefix := "test/prefix"
 	bucket := bucketName(t)
 
-	be0 := setupBackend(t, bucket, prefix)
+	be0 := setupBackend(t, bucket, prefix, noEncryptionKey)
 	defer teardownBackend(t, be0, prefix)
 
-	be1 := setupBackend(t, bucket, prefix+"/")
+	be1 := setupBackend(t, bucket, prefix+"/", noEncryptionKey)
+
+	backend.TestBackend(t, be0, be1)
+}
+func TestBackendWithEncryption(t *testing.T) {
+	t.Parallel()
+
+	bucket := bucketName(t)
+
+	be0 := setupBackend(t, bucket, noPrefix, encryptionKey)
+	defer teardownBackend(t, be0, noPrefix)
+
+	be1 := setupBackend(t, bucket, noPrefix, encryptionKey)
 
 	backend.TestBackend(t, be0, be1)
 }
 
 // setupBackend returns a new GCS backend.
-func setupBackend(t *testing.T, bucket, prefix string) backend.Backend {
+func setupBackend(t *testing.T, bucket, prefix, key string) backend.Backend {
 	t.Helper()
 
 	projectID := os.Getenv("GOOGLE_PROJECT")
@@ -139,9 +176,10 @@ func setupBackend(t *testing.T, bucket, prefix string) backend.Backend {
 	}
 
 	config := map[string]interface{}{
-		"project": projectID,
+		"project":        projectID,
-		"bucket":  bucket,
+		"bucket":         bucket,
-		"prefix":  prefix,
+		"prefix":         prefix,
+		"encryption_key": key,
 	}
 
 	b := backend.TestBackendConfig(t, New(), config)

backend/remote-state/gcs/client.go

@@ -22,6 +22,7 @@ type remoteClient struct {
 	bucketName     string
 	stateFilePath  string
 	lockFilePath   string
+	encryptionKey  []byte
 }
 
 func (c *remoteClient) Get() (payload *remote.Payload, err error) {
@@ -152,7 +153,11 @@ func (c *remoteClient) lockInfo() (*state.LockInfo, error) {
 }
 
 func (c *remoteClient) stateFile() *storage.ObjectHandle {
-	return c.storageClient.Bucket(c.bucketName).Object(c.stateFilePath)
+	h := c.storageClient.Bucket(c.bucketName).Object(c.stateFilePath)
+	if len(c.encryptionKey) > 0 {
+		return h.Key(c.encryptionKey)
+	}
+	return h
 }
 
 func (c *remoteClient) stateFileURL() string {

website/docs/backends/types/gcs.html.md

@@ -59,3 +59,4 @@ The following configuration options are supported:
     Since buckets have globally unique names, the project ID is not required to access the bucket during normal operation.
  *  `region` / `GOOGLE_REGION` - (Optional) The region in which a new bucket is created.
     For more information, see [Bucket Locations](https://cloud.google.com/storage/docs/bucket-locations).
+ *  `encryption_key` / `GOOGLE_ENCRYPTION_KEY` - (Optional) A 32 byte base64 encoded 'customer supplied encryption key' used to encrypt all state. For more information see [Customer Supplied Encryption Keys](https://cloud.google.com/storage/docs/encryption#customer-supplied).