diff --git a/backend/remote-state/consul/backend_state.go b/backend/remote-state/consul/backend_state.go index 4c0851871..74f30c842 100644 --- a/backend/remote-state/consul/backend_state.go +++ b/backend/remote-state/consul/backend_state.go @@ -56,7 +56,7 @@ func (b *Backend) States() ([]string, error) { } func (b *Backend) DeleteState(name string) error { - if name == backend.DefaultStateName { + if name == backend.DefaultStateName || name == "" { return fmt.Errorf("can't delete default state") } diff --git a/backend/remote-state/s3/backend.go b/backend/remote-state/s3/backend.go index c210ee5fe..8265d7f25 100644 --- a/backend/remote-state/s3/backend.go +++ b/backend/remote-state/s3/backend.go @@ -128,25 +128,31 @@ type Backend struct { *schema.Backend // The fields below are set from configure - client *S3Client + s3Client *s3.S3 + dynClient *dynamodb.DynamoDB + + bucketName string + keyName string + serverSideEncryption bool + acl string + kmsKeyID string + lockTable string } func (b *Backend) configure(ctx context.Context) error { - if b.client != nil { + if b.s3Client != nil { return nil } // Grab the resource data data := schema.FromContextBackendConfig(ctx) - bucketName := data.Get("bucket").(string) - keyName := data.Get("key").(string) - endpoint := data.Get("endpoint").(string) - region := data.Get("region").(string) - serverSideEncryption := data.Get("encrypt").(bool) - acl := data.Get("acl").(string) - kmsKeyID := data.Get("kms_key_id").(string) - lockTable := data.Get("lock_table").(string) + b.bucketName = data.Get("bucket").(string) + b.keyName = data.Get("key").(string) + b.serverSideEncryption = data.Get("encrypt").(bool) + b.acl = data.Get("acl").(string) + b.kmsKeyID = data.Get("kms_key_id").(string) + b.lockTable = data.Get("lock_table").(string) var errs []error creds, err := terraformAWS.GetCredentials(&terraformAWS.Config{ @@ -175,6 +181,9 @@ providing credentials for the AWS S3 remote`)) return &multierror.Error{Errors: errs} } + endpoint := data.Get("endpoint").(string) + region := data.Get("region").(string) + awsConfig := &aws.Config{ Credentials: creds, Endpoint: aws.String(endpoint), @@ -182,18 +191,8 @@ providing credentials for the AWS S3 remote`)) HTTPClient: cleanhttp.DefaultClient(), } sess := session.New(awsConfig) - nativeClient := s3.New(sess) - dynClient := dynamodb.New(sess) + b.s3Client = s3.New(sess) + b.dynClient = dynamodb.New(sess) - b.client = &S3Client{ - nativeClient: nativeClient, - bucketName: bucketName, - keyName: keyName, - serverSideEncryption: serverSideEncryption, - acl: acl, - kmsKeyID: kmsKeyID, - dynClient: dynClient, - lockTable: lockTable, - } return nil } diff --git a/backend/remote-state/s3/backend_state.go b/backend/remote-state/s3/backend_state.go index 83cbc4ca7..3166cbfb9 100644 --- a/backend/remote-state/s3/backend_state.go +++ b/backend/remote-state/s3/backend_state.go @@ -1,27 +1,112 @@ package s3 import ( + "fmt" + "sort" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" "github.com/hashicorp/terraform/backend" "github.com/hashicorp/terraform/state" "github.com/hashicorp/terraform/state/remote" ) const ( - keyEnvPrefix = "-env:" + // This will be used as directory name, the odd looking colon is simply to + // reduce the chance of name conflicts with existing objects. + keyEnvPrefix = "env:" ) func (b *Backend) States() ([]string, error) { - return nil, backend.ErrNamedStatesNotSupported + params := &s3.ListObjectsInput{ + Bucket: &b.bucketName, + Prefix: aws.String(keyEnvPrefix + "/"), + } + + resp, err := b.s3Client.ListObjects(params) + if err != nil { + return nil, err + } + + var envs []string + for _, obj := range resp.Contents { + env := keyEnv(*obj.Key) + if env != "" { + envs = append(envs, env) + } + } + + sort.Strings(envs) + envs = append([]string{backend.DefaultStateName}, envs...) + return envs, nil +} + +// extract the env name from the S3 key +func keyEnv(key string) string { + parts := strings.Split(key, "/") + if len(parts) < 3 { + // no env here + return "" + } + + if parts[0] != keyEnvPrefix { + // not our key, so ignore + return "" + } + + return parts[1] } func (b *Backend) DeleteState(name string) error { - return backend.ErrNamedStatesNotSupported + if name == backend.DefaultStateName || name == "" { + return fmt.Errorf("can't delete default state") + } + + params := &s3.DeleteObjectInput{ + Bucket: &b.bucketName, + Key: aws.String(b.path(name)), + } + + _, err := b.s3Client.DeleteObject(params) + if err != nil { + return err + } + + return nil } func (b *Backend) State(name string) (state.State, error) { - if name != backend.DefaultStateName { - return nil, backend.ErrNamedStatesNotSupported + client := &RemoteClient{ + s3Client: b.s3Client, + dynClient: b.dynClient, + bucketName: b.bucketName, + path: b.path(name), + serverSideEncryption: b.serverSideEncryption, + acl: b.acl, + kmsKeyID: b.kmsKeyID, + lockTable: b.lockTable, } - return &remote.State{Client: b.client}, nil + // if this isn't the default state name, we need to create the object so + // it's listed by States. + if name != backend.DefaultStateName { + if err := client.Put([]byte{}); err != nil { + return nil, err + } + } + + return &remote.State{Client: client}, nil +} + +func (b *Backend) client() *RemoteClient { + return &RemoteClient{} +} + +func (b *Backend) path(name string) string { + if name == backend.DefaultStateName { + return b.keyName + } + + return strings.Join([]string{keyEnvPrefix, name, b.keyName}, "/") } diff --git a/backend/remote-state/s3/backend_test.go b/backend/remote-state/s3/backend_test.go index 3d2e8ec2f..f8b664b80 100644 --- a/backend/remote-state/s3/backend_test.go +++ b/backend/remote-state/s3/backend_test.go @@ -44,17 +44,17 @@ func TestBackendConfig(t *testing.T) { b := backend.TestBackendConfig(t, New(), config).(*Backend) - if *b.client.nativeClient.Config.Region != "us-west-1" { + if *b.s3Client.Config.Region != "us-west-1" { t.Fatalf("Incorrect region was populated") } - if b.client.bucketName != "tf-test" { + if b.bucketName != "tf-test" { t.Fatalf("Incorrect bucketName was populated") } - if b.client.keyName != "state" { + if b.keyName != "state" { t.Fatalf("Incorrect keyName was populated") } - credentials, err := b.client.nativeClient.Config.Credentials.Get() + credentials, err := b.s3Client.Config.Credentials.Get() if err != nil { t.Fatalf("Error when requesting credentials") } @@ -78,8 +78,8 @@ func TestBackend(t *testing.T) { "encrypt": true, }).(*Backend) - createS3Bucket(t, b.client, bucketName) - defer deleteS3Bucket(t, b.client, bucketName) + createS3Bucket(t, b.s3Client, bucketName) + defer deleteS3Bucket(t, b.s3Client, bucketName) backend.TestBackend(t, b, nil) } @@ -104,41 +104,52 @@ func TestBackendLocked(t *testing.T) { "lock_table": bucketName, }).(*Backend) - createS3Bucket(t, b1.client, bucketName) - defer deleteS3Bucket(t, b1.client, bucketName) - createDynamoDBTable(t, b1.client, bucketName) - defer deleteDynamoDBTable(t, b1.client, bucketName) + createS3Bucket(t, b1.s3Client, bucketName) + defer deleteS3Bucket(t, b1.s3Client, bucketName) + createDynamoDBTable(t, b1.dynClient, bucketName) + defer deleteDynamoDBTable(t, b1.dynClient, bucketName) backend.TestBackend(t, b1, b2) } -func createS3Bucket(t *testing.T, c *S3Client, bucketName string) { +func createS3Bucket(t *testing.T, s3Client *s3.S3, bucketName string) { createBucketReq := &s3.CreateBucketInput{ Bucket: &bucketName, } // Be clear about what we're doing in case the user needs to clean // this up later. - t.Logf("creating S3 bucket %s in %s", bucketName, *c.nativeClient.Config.Region) - _, err := c.nativeClient.CreateBucket(createBucketReq) + t.Logf("creating S3 bucket %s in %s", bucketName, *s3Client.Config.Region) + _, err := s3Client.CreateBucket(createBucketReq) if err != nil { t.Fatal("failed to create test S3 bucket:", err) } } -func deleteS3Bucket(t *testing.T, c *S3Client, bucketName string) { - deleteBucketReq := &s3.DeleteBucketInput{ - Bucket: &bucketName, +func deleteS3Bucket(t *testing.T, s3Client *s3.S3, bucketName string) { + warning := "WARNING: Failed to delete the test S3 bucket. It may have been left in your AWS account and may incur storage charges. (error was %s)" + + // first we have to get rid of the env objects, or we can't delete the bucket + resp, err := s3Client.ListObjects(&s3.ListObjectsInput{Bucket: &bucketName}) + if err != nil { + t.Logf(warning, err) + return + } + for _, obj := range resp.Contents { + if _, err := s3Client.DeleteObject(&s3.DeleteObjectInput{Bucket: &bucketName, Key: obj.Key}); err != nil { + // this will need cleanup no matter what, so just warn and exit + t.Logf(warning, err) + return + } } - _, err := c.nativeClient.DeleteBucket(deleteBucketReq) - if err != nil { - t.Logf("WARNING: Failed to delete the test S3 bucket. It may have been left in your AWS account and may incur storage charges. (error was %s)", err) + if _, err := s3Client.DeleteBucket(&s3.DeleteBucketInput{Bucket: &bucketName}); err != nil { + t.Logf(warning, err) } } // create the dynamoDB table, and wait until we can query it. -func createDynamoDBTable(t *testing.T, c *S3Client, tableName string) { +func createDynamoDBTable(t *testing.T, dynClient *dynamodb.DynamoDB, tableName string) { createInput := &dynamodb.CreateTableInput{ AttributeDefinitions: []*dynamodb.AttributeDefinition{ { @@ -159,7 +170,7 @@ func createDynamoDBTable(t *testing.T, c *S3Client, tableName string) { TableName: aws.String(tableName), } - _, err := c.dynClient.CreateTable(createInput) + _, err := dynClient.CreateTable(createInput) if err != nil { t.Fatal(err) } @@ -173,7 +184,7 @@ func createDynamoDBTable(t *testing.T, c *S3Client, tableName string) { } for { - resp, err := c.dynClient.DescribeTable(describeInput) + resp, err := dynClient.DescribeTable(describeInput) if err != nil { t.Fatal(err) } @@ -191,11 +202,11 @@ func createDynamoDBTable(t *testing.T, c *S3Client, tableName string) { } -func deleteDynamoDBTable(t *testing.T, c *S3Client, tableName string) { +func deleteDynamoDBTable(t *testing.T, dynClient *dynamodb.DynamoDB, tableName string) { params := &dynamodb.DeleteTableInput{ TableName: aws.String(tableName), } - _, err := c.dynClient.DeleteTable(params) + _, err := dynClient.DeleteTable(params) if err != nil { t.Logf("WARNING: Failed to delete the test DynamoDB table %q. It has been left in your AWS account and may incur charges. (error was %s)", tableName, err) } diff --git a/backend/remote-state/s3/client.go b/backend/remote-state/s3/client.go index 0a37d5b46..735180ba9 100644 --- a/backend/remote-state/s3/client.go +++ b/backend/remote-state/s3/client.go @@ -17,21 +17,21 @@ import ( "github.com/hashicorp/terraform/state/remote" ) -type S3Client struct { - nativeClient *s3.S3 +type RemoteClient struct { + s3Client *s3.S3 + dynClient *dynamodb.DynamoDB bucketName string - keyName string + path string serverSideEncryption bool acl string kmsKeyID string - dynClient *dynamodb.DynamoDB lockTable string } -func (c *S3Client) Get() (*remote.Payload, error) { - output, err := c.nativeClient.GetObject(&s3.GetObjectInput{ +func (c *RemoteClient) Get() (*remote.Payload, error) { + output, err := c.s3Client.GetObject(&s3.GetObjectInput{ Bucket: &c.bucketName, - Key: &c.keyName, + Key: &c.path, }) if err != nil { @@ -65,7 +65,7 @@ func (c *S3Client) Get() (*remote.Payload, error) { return payload, nil } -func (c *S3Client) Put(data []byte) error { +func (c *RemoteClient) Put(data []byte) error { contentType := "application/json" contentLength := int64(len(data)) @@ -74,7 +74,7 @@ func (c *S3Client) Put(data []byte) error { ContentLength: &contentLength, Body: bytes.NewReader(data), Bucket: &c.bucketName, - Key: &c.keyName, + Key: &c.path, } if c.serverSideEncryption { @@ -92,28 +92,28 @@ func (c *S3Client) Put(data []byte) error { log.Printf("[DEBUG] Uploading remote state to S3: %#v", i) - if _, err := c.nativeClient.PutObject(i); err == nil { + if _, err := c.s3Client.PutObject(i); err == nil { return nil } else { return fmt.Errorf("Failed to upload state: %v", err) } } -func (c *S3Client) Delete() error { - _, err := c.nativeClient.DeleteObject(&s3.DeleteObjectInput{ +func (c *RemoteClient) Delete() error { + _, err := c.s3Client.DeleteObject(&s3.DeleteObjectInput{ Bucket: &c.bucketName, - Key: &c.keyName, + Key: &c.path, }) return err } -func (c *S3Client) Lock(info *state.LockInfo) (string, error) { +func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) { if c.lockTable == "" { return "", nil } - stateName := fmt.Sprintf("%s/%s", c.bucketName, c.keyName) + stateName := fmt.Sprintf("%s/%s", c.bucketName, c.path) info.Path = stateName if info.ID == "" { @@ -150,10 +150,10 @@ func (c *S3Client) Lock(info *state.LockInfo) (string, error) { return info.ID, nil } -func (c *S3Client) getLockInfo() (*state.LockInfo, error) { +func (c *RemoteClient) getLockInfo() (*state.LockInfo, error) { getParams := &dynamodb.GetItemInput{ Key: map[string]*dynamodb.AttributeValue{ - "LockID": {S: aws.String(fmt.Sprintf("%s/%s", c.bucketName, c.keyName))}, + "LockID": {S: aws.String(fmt.Sprintf("%s/%s", c.bucketName, c.path))}, }, ProjectionExpression: aws.String("LockID, Info"), TableName: aws.String(c.lockTable), @@ -178,7 +178,7 @@ func (c *S3Client) getLockInfo() (*state.LockInfo, error) { return lockInfo, nil } -func (c *S3Client) Unlock(id string) error { +func (c *RemoteClient) Unlock(id string) error { if c.lockTable == "" { return nil } @@ -202,7 +202,7 @@ func (c *S3Client) Unlock(id string) error { params := &dynamodb.DeleteItemInput{ Key: map[string]*dynamodb.AttributeValue{ - "LockID": {S: aws.String(fmt.Sprintf("%s/%s", c.bucketName, c.keyName))}, + "LockID": {S: aws.String(fmt.Sprintf("%s/%s", c.bucketName, c.path))}, }, TableName: aws.String(c.lockTable), } diff --git a/backend/remote-state/s3/client_test.go b/backend/remote-state/s3/client_test.go index 3cd99b6fc..0cef7c9ed 100644 --- a/backend/remote-state/s3/client_test.go +++ b/backend/remote-state/s3/client_test.go @@ -10,8 +10,8 @@ import ( ) func TestRemoteClient_impl(t *testing.T) { - var _ remote.Client = new(S3Client) - var _ remote.ClientLocker = new(S3Client) + var _ remote.Client = new(RemoteClient) + var _ remote.ClientLocker = new(RemoteClient) } func TestRemoteClient(t *testing.T) { @@ -31,8 +31,8 @@ func TestRemoteClient(t *testing.T) { t.Fatal(err) } - createS3Bucket(t, b.client, bucketName) - defer deleteS3Bucket(t, b.client, bucketName) + createS3Bucket(t, b.s3Client, bucketName) + defer deleteS3Bucket(t, b.s3Client, bucketName) remote.TestClient(t, state.(*remote.State).Client) } @@ -67,10 +67,10 @@ func TestRemoteClientLocks(t *testing.T) { t.Fatal(err) } - createS3Bucket(t, b1.client, bucketName) - defer deleteS3Bucket(t, b1.client, bucketName) - createDynamoDBTable(t, b1.client, bucketName) - defer deleteDynamoDBTable(t, b1.client, bucketName) + createS3Bucket(t, b1.s3Client, bucketName) + defer deleteS3Bucket(t, b1.s3Client, bucketName) + createDynamoDBTable(t, b1.dynClient, bucketName) + defer deleteDynamoDBTable(t, b1.dynClient, bucketName) remote.TestRemoteLocks(t, s1.(*remote.State).Client, s2.(*remote.State).Client) }