From 4980fa20e778ecb989eae98077873ce76630ca48 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Wed, 22 Mar 2017 15:52:55 -0400 Subject: [PATCH] move s3 config from client to backend The RemoteClient needs to be configured for the named state, so move the general config to the backend. Rename some fields for consistency. --- backend/remote-state/consul/backend_state.go | 2 +- backend/remote-state/s3/backend.go | 43 +++++++++--------- backend/remote-state/s3/backend_state.go | 48 +++++++++++++++++++- backend/remote-state/s3/backend_test.go | 40 ++++++++-------- backend/remote-state/s3/client.go | 38 ++++++++-------- backend/remote-state/s3/client_test.go | 16 +++---- 6 files changed, 115 insertions(+), 72 deletions(-) diff --git a/backend/remote-state/consul/backend_state.go b/backend/remote-state/consul/backend_state.go index 4c0851871..74f30c842 100644 --- a/backend/remote-state/consul/backend_state.go +++ b/backend/remote-state/consul/backend_state.go @@ -56,7 +56,7 @@ func (b *Backend) States() ([]string, error) { } func (b *Backend) DeleteState(name string) error { - if name == backend.DefaultStateName { + if name == backend.DefaultStateName || name == "" { return fmt.Errorf("can't delete default state") } diff --git a/backend/remote-state/s3/backend.go b/backend/remote-state/s3/backend.go index c210ee5fe..8265d7f25 100644 --- a/backend/remote-state/s3/backend.go +++ b/backend/remote-state/s3/backend.go @@ -128,25 +128,31 @@ type Backend struct { *schema.Backend // The fields below are set from configure - client *S3Client + s3Client *s3.S3 + dynClient *dynamodb.DynamoDB + + bucketName string + keyName string + serverSideEncryption bool + acl string + kmsKeyID string + lockTable string } func (b *Backend) configure(ctx context.Context) error { - if b.client != nil { + if b.s3Client != nil { return nil } // Grab the resource data data := schema.FromContextBackendConfig(ctx) - bucketName := data.Get("bucket").(string) - keyName := data.Get("key").(string) - endpoint := data.Get("endpoint").(string) - region := data.Get("region").(string) - serverSideEncryption := data.Get("encrypt").(bool) - acl := data.Get("acl").(string) - kmsKeyID := data.Get("kms_key_id").(string) - lockTable := data.Get("lock_table").(string) + b.bucketName = data.Get("bucket").(string) + b.keyName = data.Get("key").(string) + b.serverSideEncryption = data.Get("encrypt").(bool) + b.acl = data.Get("acl").(string) + b.kmsKeyID = data.Get("kms_key_id").(string) + b.lockTable = data.Get("lock_table").(string) var errs []error creds, err := terraformAWS.GetCredentials(&terraformAWS.Config{ @@ -175,6 +181,9 @@ providing credentials for the AWS S3 remote`)) return &multierror.Error{Errors: errs} } + endpoint := data.Get("endpoint").(string) + region := data.Get("region").(string) + awsConfig := &aws.Config{ Credentials: creds, Endpoint: aws.String(endpoint), @@ -182,18 +191,8 @@ providing credentials for the AWS S3 remote`)) HTTPClient: cleanhttp.DefaultClient(), } sess := session.New(awsConfig) - nativeClient := s3.New(sess) - dynClient := dynamodb.New(sess) + b.s3Client = s3.New(sess) + b.dynClient = dynamodb.New(sess) - b.client = &S3Client{ - nativeClient: nativeClient, - bucketName: bucketName, - keyName: keyName, - serverSideEncryption: serverSideEncryption, - acl: acl, - kmsKeyID: kmsKeyID, - dynClient: dynClient, - lockTable: lockTable, - } return nil } diff --git a/backend/remote-state/s3/backend_state.go b/backend/remote-state/s3/backend_state.go index 83cbc4ca7..6e9252861 100644 --- a/backend/remote-state/s3/backend_state.go +++ b/backend/remote-state/s3/backend_state.go @@ -1,13 +1,18 @@ package s3 import ( + "fmt" + "strings" + "github.com/hashicorp/terraform/backend" "github.com/hashicorp/terraform/state" "github.com/hashicorp/terraform/state/remote" ) const ( - keyEnvPrefix = "-env:" + // This will be used a directory name, the odd looking colon is to reduce + // the chance of name conflicts with existing deployments. + keyEnvPrefix = "env:" ) func (b *Backend) States() ([]string, error) { @@ -16,6 +21,20 @@ func (b *Backend) States() ([]string, error) { func (b *Backend) DeleteState(name string) error { return backend.ErrNamedStatesNotSupported + if name == backend.DefaultStateName || name == "" { + return fmt.Errorf("can't delete default state") + } + + //params := &s3.ListObjectsInput{ + // Bucket: &b.client.bucketName, + // Delimiter: aws.String("Delimiter"), + // EncodingType: aws.String("EncodingType"), + // Marker: aws.String("Marker"), + // MaxKeys: aws.Int64(1), + // Prefix: aws.String("env"), + // RequestPayer: aws.String("RequestPayer"), + //} + return nil } func (b *Backend) State(name string) (state.State, error) { @@ -23,5 +42,30 @@ func (b *Backend) State(name string) (state.State, error) { return nil, backend.ErrNamedStatesNotSupported } - return &remote.State{Client: b.client}, nil + client := &RemoteClient{ + s3Client: b.s3Client, + dynClient: b.dynClient, + bucketName: b.bucketName, + path: b.path(name), + serverSideEncryption: b.serverSideEncryption, + acl: b.acl, + kmsKeyID: b.kmsKeyID, + lockTable: b.lockTable, + } + + // TODO: create new state if it doesn't exist + + return &remote.State{Client: client}, nil +} + +func (b *Backend) client() *RemoteClient { + return &RemoteClient{} +} + +func (b *Backend) path(name string) string { + if name == backend.DefaultStateName { + return b.keyName + } + + return strings.Join([]string{keyEnvPrefix, name, b.keyName}, "/") } diff --git a/backend/remote-state/s3/backend_test.go b/backend/remote-state/s3/backend_test.go index 3d2e8ec2f..0838fa159 100644 --- a/backend/remote-state/s3/backend_test.go +++ b/backend/remote-state/s3/backend_test.go @@ -44,17 +44,17 @@ func TestBackendConfig(t *testing.T) { b := backend.TestBackendConfig(t, New(), config).(*Backend) - if *b.client.nativeClient.Config.Region != "us-west-1" { + if *b.s3Client.Config.Region != "us-west-1" { t.Fatalf("Incorrect region was populated") } - if b.client.bucketName != "tf-test" { + if b.bucketName != "tf-test" { t.Fatalf("Incorrect bucketName was populated") } - if b.client.keyName != "state" { + if b.keyName != "state" { t.Fatalf("Incorrect keyName was populated") } - credentials, err := b.client.nativeClient.Config.Credentials.Get() + credentials, err := b.s3Client.Config.Credentials.Get() if err != nil { t.Fatalf("Error when requesting credentials") } @@ -78,8 +78,8 @@ func TestBackend(t *testing.T) { "encrypt": true, }).(*Backend) - createS3Bucket(t, b.client, bucketName) - defer deleteS3Bucket(t, b.client, bucketName) + createS3Bucket(t, b.s3Client, bucketName) + defer deleteS3Bucket(t, b.s3Client, bucketName) backend.TestBackend(t, b, nil) } @@ -104,41 +104,41 @@ func TestBackendLocked(t *testing.T) { "lock_table": bucketName, }).(*Backend) - createS3Bucket(t, b1.client, bucketName) - defer deleteS3Bucket(t, b1.client, bucketName) - createDynamoDBTable(t, b1.client, bucketName) - defer deleteDynamoDBTable(t, b1.client, bucketName) + createS3Bucket(t, b1.s3Client, bucketName) + defer deleteS3Bucket(t, b1.s3Client, bucketName) + createDynamoDBTable(t, b1.dynClient, bucketName) + defer deleteDynamoDBTable(t, b1.dynClient, bucketName) backend.TestBackend(t, b1, b2) } -func createS3Bucket(t *testing.T, c *S3Client, bucketName string) { +func createS3Bucket(t *testing.T, s3Client *s3.S3, bucketName string) { createBucketReq := &s3.CreateBucketInput{ Bucket: &bucketName, } // Be clear about what we're doing in case the user needs to clean // this up later. - t.Logf("creating S3 bucket %s in %s", bucketName, *c.nativeClient.Config.Region) - _, err := c.nativeClient.CreateBucket(createBucketReq) + t.Logf("creating S3 bucket %s in %s", bucketName, *s3Client.Config.Region) + _, err := s3Client.CreateBucket(createBucketReq) if err != nil { t.Fatal("failed to create test S3 bucket:", err) } } -func deleteS3Bucket(t *testing.T, c *S3Client, bucketName string) { +func deleteS3Bucket(t *testing.T, s3Client *s3.S3, bucketName string) { deleteBucketReq := &s3.DeleteBucketInput{ Bucket: &bucketName, } - _, err := c.nativeClient.DeleteBucket(deleteBucketReq) + _, err := s3Client.DeleteBucket(deleteBucketReq) if err != nil { t.Logf("WARNING: Failed to delete the test S3 bucket. It may have been left in your AWS account and may incur storage charges. (error was %s)", err) } } // create the dynamoDB table, and wait until we can query it. -func createDynamoDBTable(t *testing.T, c *S3Client, tableName string) { +func createDynamoDBTable(t *testing.T, dynClient *dynamodb.DynamoDB, tableName string) { createInput := &dynamodb.CreateTableInput{ AttributeDefinitions: []*dynamodb.AttributeDefinition{ { @@ -159,7 +159,7 @@ func createDynamoDBTable(t *testing.T, c *S3Client, tableName string) { TableName: aws.String(tableName), } - _, err := c.dynClient.CreateTable(createInput) + _, err := dynClient.CreateTable(createInput) if err != nil { t.Fatal(err) } @@ -173,7 +173,7 @@ func createDynamoDBTable(t *testing.T, c *S3Client, tableName string) { } for { - resp, err := c.dynClient.DescribeTable(describeInput) + resp, err := dynClient.DescribeTable(describeInput) if err != nil { t.Fatal(err) } @@ -191,11 +191,11 @@ func createDynamoDBTable(t *testing.T, c *S3Client, tableName string) { } -func deleteDynamoDBTable(t *testing.T, c *S3Client, tableName string) { +func deleteDynamoDBTable(t *testing.T, dynClient *dynamodb.DynamoDB, tableName string) { params := &dynamodb.DeleteTableInput{ TableName: aws.String(tableName), } - _, err := c.dynClient.DeleteTable(params) + _, err := dynClient.DeleteTable(params) if err != nil { t.Logf("WARNING: Failed to delete the test DynamoDB table %q. It has been left in your AWS account and may incur charges. (error was %s)", tableName, err) } diff --git a/backend/remote-state/s3/client.go b/backend/remote-state/s3/client.go index 0a37d5b46..735180ba9 100644 --- a/backend/remote-state/s3/client.go +++ b/backend/remote-state/s3/client.go @@ -17,21 +17,21 @@ import ( "github.com/hashicorp/terraform/state/remote" ) -type S3Client struct { - nativeClient *s3.S3 +type RemoteClient struct { + s3Client *s3.S3 + dynClient *dynamodb.DynamoDB bucketName string - keyName string + path string serverSideEncryption bool acl string kmsKeyID string - dynClient *dynamodb.DynamoDB lockTable string } -func (c *S3Client) Get() (*remote.Payload, error) { - output, err := c.nativeClient.GetObject(&s3.GetObjectInput{ +func (c *RemoteClient) Get() (*remote.Payload, error) { + output, err := c.s3Client.GetObject(&s3.GetObjectInput{ Bucket: &c.bucketName, - Key: &c.keyName, + Key: &c.path, }) if err != nil { @@ -65,7 +65,7 @@ func (c *S3Client) Get() (*remote.Payload, error) { return payload, nil } -func (c *S3Client) Put(data []byte) error { +func (c *RemoteClient) Put(data []byte) error { contentType := "application/json" contentLength := int64(len(data)) @@ -74,7 +74,7 @@ func (c *S3Client) Put(data []byte) error { ContentLength: &contentLength, Body: bytes.NewReader(data), Bucket: &c.bucketName, - Key: &c.keyName, + Key: &c.path, } if c.serverSideEncryption { @@ -92,28 +92,28 @@ func (c *S3Client) Put(data []byte) error { log.Printf("[DEBUG] Uploading remote state to S3: %#v", i) - if _, err := c.nativeClient.PutObject(i); err == nil { + if _, err := c.s3Client.PutObject(i); err == nil { return nil } else { return fmt.Errorf("Failed to upload state: %v", err) } } -func (c *S3Client) Delete() error { - _, err := c.nativeClient.DeleteObject(&s3.DeleteObjectInput{ +func (c *RemoteClient) Delete() error { + _, err := c.s3Client.DeleteObject(&s3.DeleteObjectInput{ Bucket: &c.bucketName, - Key: &c.keyName, + Key: &c.path, }) return err } -func (c *S3Client) Lock(info *state.LockInfo) (string, error) { +func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) { if c.lockTable == "" { return "", nil } - stateName := fmt.Sprintf("%s/%s", c.bucketName, c.keyName) + stateName := fmt.Sprintf("%s/%s", c.bucketName, c.path) info.Path = stateName if info.ID == "" { @@ -150,10 +150,10 @@ func (c *S3Client) Lock(info *state.LockInfo) (string, error) { return info.ID, nil } -func (c *S3Client) getLockInfo() (*state.LockInfo, error) { +func (c *RemoteClient) getLockInfo() (*state.LockInfo, error) { getParams := &dynamodb.GetItemInput{ Key: map[string]*dynamodb.AttributeValue{ - "LockID": {S: aws.String(fmt.Sprintf("%s/%s", c.bucketName, c.keyName))}, + "LockID": {S: aws.String(fmt.Sprintf("%s/%s", c.bucketName, c.path))}, }, ProjectionExpression: aws.String("LockID, Info"), TableName: aws.String(c.lockTable), @@ -178,7 +178,7 @@ func (c *S3Client) getLockInfo() (*state.LockInfo, error) { return lockInfo, nil } -func (c *S3Client) Unlock(id string) error { +func (c *RemoteClient) Unlock(id string) error { if c.lockTable == "" { return nil } @@ -202,7 +202,7 @@ func (c *S3Client) Unlock(id string) error { params := &dynamodb.DeleteItemInput{ Key: map[string]*dynamodb.AttributeValue{ - "LockID": {S: aws.String(fmt.Sprintf("%s/%s", c.bucketName, c.keyName))}, + "LockID": {S: aws.String(fmt.Sprintf("%s/%s", c.bucketName, c.path))}, }, TableName: aws.String(c.lockTable), } diff --git a/backend/remote-state/s3/client_test.go b/backend/remote-state/s3/client_test.go index 3cd99b6fc..0cef7c9ed 100644 --- a/backend/remote-state/s3/client_test.go +++ b/backend/remote-state/s3/client_test.go @@ -10,8 +10,8 @@ import ( ) func TestRemoteClient_impl(t *testing.T) { - var _ remote.Client = new(S3Client) - var _ remote.ClientLocker = new(S3Client) + var _ remote.Client = new(RemoteClient) + var _ remote.ClientLocker = new(RemoteClient) } func TestRemoteClient(t *testing.T) { @@ -31,8 +31,8 @@ func TestRemoteClient(t *testing.T) { t.Fatal(err) } - createS3Bucket(t, b.client, bucketName) - defer deleteS3Bucket(t, b.client, bucketName) + createS3Bucket(t, b.s3Client, bucketName) + defer deleteS3Bucket(t, b.s3Client, bucketName) remote.TestClient(t, state.(*remote.State).Client) } @@ -67,10 +67,10 @@ func TestRemoteClientLocks(t *testing.T) { t.Fatal(err) } - createS3Bucket(t, b1.client, bucketName) - defer deleteS3Bucket(t, b1.client, bucketName) - createDynamoDBTable(t, b1.client, bucketName) - defer deleteDynamoDBTable(t, b1.client, bucketName) + createS3Bucket(t, b1.s3Client, bucketName) + defer deleteS3Bucket(t, b1.s3Client, bucketName) + createDynamoDBTable(t, b1.dynClient, bucketName) + defer deleteDynamoDBTable(t, b1.dynClient, bucketName) remote.TestRemoteLocks(t, s1.(*remote.State).Client, s2.(*remote.State).Client) }