resilien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #12987 from hashicorp/jbardin/s3-backend 

Add named state support to the s3 backend
This commit is contained in:
James Bardin 2017-03-22 17:50:33 -04:00 committed by GitHub
parent 02aa1ea8a1 fa4dc01cf4
commit df1fc8f2c3
6 changed files with 175 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/remote-state/consul/backend_state.go
View File

@ -56,7 +56,7 @@ func (b *Backend) States() ([]string, error) {
}
func (b *Backend) DeleteState(name string) error {
if name == backend.DefaultStateName {
if name == backend.DefaultStateName || name == "" {
return fmt.Errorf("can't delete default state")
}

43
backend/remote-state/s3/backend.go
View File

@ -128,25 +128,31 @@ type Backend struct {
*schema.Backend
// The fields below are set from configure
client *S3Client
s3Client *s3.S3
dynClient *dynamodb.DynamoDB
bucketName string
keyName string
serverSideEncryption bool
acl string
kmsKeyID string
lockTable string
}
func (b *Backend) configure(ctx context.Context) error {
if b.client != nil {
if b.s3Client != nil {
return nil
}
// Grab the resource data
data := schema.FromContextBackendConfig(ctx)
bucketName := data.Get("bucket").(string)
keyName := data.Get("key").(string)
endpoint := data.Get("endpoint").(string)
region := data.Get("region").(string)
serverSideEncryption := data.Get("encrypt").(bool)
acl := data.Get("acl").(string)
kmsKeyID := data.Get("kms_key_id").(string)
lockTable := data.Get("lock_table").(string)
b.bucketName = data.Get("bucket").(string)
b.keyName = data.Get("key").(string)
b.serverSideEncryption = data.Get("encrypt").(bool)
b.acl = data.Get("acl").(string)
b.kmsKeyID = data.Get("kms_key_id").(string)
b.lockTable = data.Get("lock_table").(string)
var errs []error
creds, err := terraformAWS.GetCredentials(&terraformAWS.Config{
@ -175,6 +181,9 @@ providing credentials for the AWS S3 remote`))
return &multierror.Error{Errors: errs}
}
endpoint := data.Get("endpoint").(string)
region := data.Get("region").(string)
awsConfig := &aws.Config{
Credentials: creds,
Endpoint: aws.String(endpoint),
@ -182,18 +191,8 @@ providing credentials for the AWS S3 remote`))
HTTPClient: cleanhttp.DefaultClient(),
}
sess := session.New(awsConfig)
nativeClient := s3.New(sess)
dynClient := dynamodb.New(sess)
b.s3Client = s3.New(sess)
b.dynClient = dynamodb.New(sess)
b.client = &S3Client{
nativeClient: nativeClient,
bucketName: bucketName,
keyName: keyName,
serverSideEncryption: serverSideEncryption,
acl: acl,
kmsKeyID: kmsKeyID,
dynClient: dynClient,
lockTable: lockTable,
}
return nil
}

97
backend/remote-state/s3/backend_state.go
View File

@ -1,27 +1,112 @@
package s3
import (
"fmt"
"sort"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/state/remote"
)
const (
keyEnvPrefix = "-env:"
// This will be used as directory name, the odd looking colon is simply to
// reduce the chance of name conflicts with existing objects.
keyEnvPrefix = "env:"
)
func (b *Backend) States() ([]string, error) {
return nil, backend.ErrNamedStatesNotSupported
params := &s3.ListObjectsInput{
Bucket: &b.bucketName,
Prefix: aws.String(keyEnvPrefix + "/"),
}
resp, err := b.s3Client.ListObjects(params)
if err != nil {
return nil, err
}
var envs []string
for _, obj := range resp.Contents {
env := keyEnv(*obj.Key)
if env != "" {
envs = append(envs, env)
}
}
sort.Strings(envs)
envs = append([]string{backend.DefaultStateName}, envs...)
return envs, nil
}
// extract the env name from the S3 key
func keyEnv(key string) string {
parts := strings.Split(key, "/")
if len(parts) < 3 {
// no env here
return ""
}
if parts[0] != keyEnvPrefix {
// not our key, so ignore
return ""
}
return parts[1]
}
func (b *Backend) DeleteState(name string) error {
return backend.ErrNamedStatesNotSupported
if name == backend.DefaultStateName || name == "" {
return fmt.Errorf("can't delete default state")
}
params := &s3.DeleteObjectInput{
Bucket: &b.bucketName,
Key: aws.String(b.path(name)),
}
_, err := b.s3Client.DeleteObject(params)
if err != nil {
return err
}
return nil
}
func (b *Backend) State(name string) (state.State, error) {
if name != backend.DefaultStateName {
return nil, backend.ErrNamedStatesNotSupported
client := &RemoteClient{
s3Client: b.s3Client,
dynClient: b.dynClient,
bucketName: b.bucketName,
path: b.path(name),
serverSideEncryption: b.serverSideEncryption,
acl: b.acl,
kmsKeyID: b.kmsKeyID,
lockTable: b.lockTable,
}
return &remote.State{Client: b.client}, nil
// if this isn't the default state name, we need to create the object so
// it's listed by States.
if name != backend.DefaultStateName {
if err := client.Put([]byte{}); err != nil {
return nil, err
}
}
return &remote.State{Client: client}, nil
}
func (b *Backend) client() *RemoteClient {
return &RemoteClient{}
}
func (b *Backend) path(name string) string {
if name == backend.DefaultStateName {
return b.keyName
}
return strings.Join([]string{keyEnvPrefix, name, b.keyName}, "/")
}

59
backend/remote-state/s3/backend_test.go
View File

@ -44,17 +44,17 @@ func TestBackendConfig(t *testing.T) {
b := backend.TestBackendConfig(t, New(), config).(*Backend)
if *b.client.nativeClient.Config.Region != "us-west-1" {
if *b.s3Client.Config.Region != "us-west-1" {
t.Fatalf("Incorrect region was populated")
}
if b.client.bucketName != "tf-test" {
if b.bucketName != "tf-test" {
t.Fatalf("Incorrect bucketName was populated")
}
if b.client.keyName != "state" {
if b.keyName != "state" {
t.Fatalf("Incorrect keyName was populated")
}
credentials, err := b.client.nativeClient.Config.Credentials.Get()
credentials, err := b.s3Client.Config.Credentials.Get()
if err != nil {
t.Fatalf("Error when requesting credentials")
}
@ -78,8 +78,8 @@ func TestBackend(t *testing.T) {
"encrypt": true,
}).(*Backend)
createS3Bucket(t, b.client, bucketName)
defer deleteS3Bucket(t, b.client, bucketName)
createS3Bucket(t, b.s3Client, bucketName)
defer deleteS3Bucket(t, b.s3Client, bucketName)
backend.TestBackend(t, b, nil)
}
@ -104,41 +104,52 @@ func TestBackendLocked(t *testing.T) {
"lock_table": bucketName,
}).(*Backend)
createS3Bucket(t, b1.client, bucketName)
defer deleteS3Bucket(t, b1.client, bucketName)
createDynamoDBTable(t, b1.client, bucketName)
defer deleteDynamoDBTable(t, b1.client, bucketName)
createS3Bucket(t, b1.s3Client, bucketName)
defer deleteS3Bucket(t, b1.s3Client, bucketName)
createDynamoDBTable(t, b1.dynClient, bucketName)
defer deleteDynamoDBTable(t, b1.dynClient, bucketName)
backend.TestBackend(t, b1, b2)
}
func createS3Bucket(t *testing.T, c *S3Client, bucketName string) {
func createS3Bucket(t *testing.T, s3Client *s3.S3, bucketName string) {
createBucketReq := &s3.CreateBucketInput{
Bucket: &bucketName,
}
// Be clear about what we're doing in case the user needs to clean
// this up later.
t.Logf("creating S3 bucket %s in %s", bucketName, *c.nativeClient.Config.Region)
_, err := c.nativeClient.CreateBucket(createBucketReq)
t.Logf("creating S3 bucket %s in %s", bucketName, *s3Client.Config.Region)
_, err := s3Client.CreateBucket(createBucketReq)
if err != nil {
t.Fatal("failed to create test S3 bucket:", err)
}
}
func deleteS3Bucket(t *testing.T, c *S3Client, bucketName string) {
deleteBucketReq := &s3.DeleteBucketInput{
Bucket: &bucketName,
func deleteS3Bucket(t *testing.T, s3Client *s3.S3, bucketName string) {
warning := "WARNING: Failed to delete the test S3 bucket. It may have been left in your AWS account and may incur storage charges. (error was %s)"
// first we have to get rid of the env objects, or we can't delete the bucket
resp, err := s3Client.ListObjects(&s3.ListObjectsInput{Bucket: &bucketName})
if err != nil {
t.Logf(warning, err)
return
}
for _, obj := range resp.Contents {
if _, err := s3Client.DeleteObject(&s3.DeleteObjectInput{Bucket: &bucketName, Key: obj.Key}); err != nil {
// this will need cleanup no matter what, so just warn and exit
t.Logf(warning, err)
return
}
}
_, err := c.nativeClient.DeleteBucket(deleteBucketReq)
if err != nil {
t.Logf("WARNING: Failed to delete the test S3 bucket. It may have been left in your AWS account and may incur storage charges. (error was %s)", err)
if _, err := s3Client.DeleteBucket(&s3.DeleteBucketInput{Bucket: &bucketName}); err != nil {
t.Logf(warning, err)
}
}
// create the dynamoDB table, and wait until we can query it.
func createDynamoDBTable(t *testing.T, c *S3Client, tableName string) {
func createDynamoDBTable(t *testing.T, dynClient *dynamodb.DynamoDB, tableName string) {
createInput := &dynamodb.CreateTableInput{
AttributeDefinitions: []*dynamodb.AttributeDefinition{
{
@ -159,7 +170,7 @@ func createDynamoDBTable(t *testing.T, c *S3Client, tableName string) {
TableName: aws.String(tableName),
}
_, err := c.dynClient.CreateTable(createInput)
_, err := dynClient.CreateTable(createInput)
if err != nil {
t.Fatal(err)
}
@ -173,7 +184,7 @@ func createDynamoDBTable(t *testing.T, c *S3Client, tableName string) {
}
for {
resp, err := c.dynClient.DescribeTable(describeInput)
resp, err := dynClient.DescribeTable(describeInput)
if err != nil {
t.Fatal(err)
}
@ -191,11 +202,11 @@ func createDynamoDBTable(t *testing.T, c *S3Client, tableName string) {
}
func deleteDynamoDBTable(t *testing.T, c *S3Client, tableName string) {
func deleteDynamoDBTable(t *testing.T, dynClient *dynamodb.DynamoDB, tableName string) {
params := &dynamodb.DeleteTableInput{
TableName: aws.String(tableName),
}
_, err := c.dynClient.DeleteTable(params)
_, err := dynClient.DeleteTable(params)
if err != nil {
t.Logf("WARNING: Failed to delete the test DynamoDB table %q. It has been left in your AWS account and may incur charges. (error was %s)", tableName, err)
}

38
backend/remote-state/s3/client.go
View File

@ -17,21 +17,21 @@ import (
"github.com/hashicorp/terraform/state/remote"
)
type S3Client struct {
nativeClient *s3.S3
type RemoteClient struct {
s3Client *s3.S3
dynClient *dynamodb.DynamoDB
bucketName string
keyName string
path string
serverSideEncryption bool
acl string
kmsKeyID string
dynClient *dynamodb.DynamoDB
lockTable string
}
func (c *S3Client) Get() (*remote.Payload, error) {
output, err := c.nativeClient.GetObject(&s3.GetObjectInput{
func (c *RemoteClient) Get() (*remote.Payload, error) {
output, err := c.s3Client.GetObject(&s3.GetObjectInput{
Bucket: &c.bucketName,
Key: &c.keyName,
Key: &c.path,
})
if err != nil {
@ -65,7 +65,7 @@ func (c *S3Client) Get() (*remote.Payload, error) {
return payload, nil
}
func (c *S3Client) Put(data []byte) error {
func (c *RemoteClient) Put(data []byte) error {
contentType := "application/json"
contentLength := int64(len(data))
@ -74,7 +74,7 @@ func (c *S3Client) Put(data []byte) error {
ContentLength: &contentLength,
Body: bytes.NewReader(data),
Bucket: &c.bucketName,
Key: &c.keyName,
Key: &c.path,
}
if c.serverSideEncryption {
@ -92,28 +92,28 @@ func (c *S3Client) Put(data []byte) error {
log.Printf("[DEBUG] Uploading remote state to S3: %#v", i)
if _, err := c.nativeClient.PutObject(i); err == nil {
if _, err := c.s3Client.PutObject(i); err == nil {
return nil
} else {
return fmt.Errorf("Failed to upload state: %v", err)
}
}
func (c *S3Client) Delete() error {
_, err := c.nativeClient.DeleteObject(&s3.DeleteObjectInput{
func (c *RemoteClient) Delete() error {
_, err := c.s3Client.DeleteObject(&s3.DeleteObjectInput{
Bucket: &c.bucketName,
Key: &c.keyName,
Key: &c.path,
})
return err
}
func (c *S3Client) Lock(info *state.LockInfo) (string, error) {
func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) {
if c.lockTable == "" {
return "", nil
}
stateName := fmt.Sprintf("%s/%s", c.bucketName, c.keyName)
stateName := fmt.Sprintf("%s/%s", c.bucketName, c.path)
info.Path = stateName
if info.ID == "" {
@ -150,10 +150,10 @@ func (c *S3Client) Lock(info *state.LockInfo) (string, error) {
return info.ID, nil
}
func (c *S3Client) getLockInfo() (*state.LockInfo, error) {
func (c *RemoteClient) getLockInfo() (*state.LockInfo, error) {
getParams := &dynamodb.GetItemInput{
Key: map[string]*dynamodb.AttributeValue{
"LockID": {S: aws.String(fmt.Sprintf("%s/%s", c.bucketName, c.keyName))},
"LockID": {S: aws.String(fmt.Sprintf("%s/%s", c.bucketName, c.path))},
},
ProjectionExpression: aws.String("LockID, Info"),
TableName: aws.String(c.lockTable),
@ -178,7 +178,7 @@ func (c *S3Client) getLockInfo() (*state.LockInfo, error) {
return lockInfo, nil
}
func (c *S3Client) Unlock(id string) error {
func (c *RemoteClient) Unlock(id string) error {
if c.lockTable == "" {
return nil
}
@ -202,7 +202,7 @@ func (c *S3Client) Unlock(id string) error {
params := &dynamodb.DeleteItemInput{
Key: map[string]*dynamodb.AttributeValue{
"LockID": {S: aws.String(fmt.Sprintf("%s/%s", c.bucketName, c.keyName))},
"LockID": {S: aws.String(fmt.Sprintf("%s/%s", c.bucketName, c.path))},
},
TableName: aws.String(c.lockTable),
}

16
backend/remote-state/s3/client_test.go
View File

@ -10,8 +10,8 @@ import (
)
func TestRemoteClient_impl(t *testing.T) {
var _ remote.Client = new(S3Client)
var _ remote.ClientLocker = new(S3Client)
var _ remote.Client = new(RemoteClient)
var _ remote.ClientLocker = new(RemoteClient)
}
func TestRemoteClient(t *testing.T) {
@ -31,8 +31,8 @@ func TestRemoteClient(t *testing.T) {
t.Fatal(err)
}
createS3Bucket(t, b.client, bucketName)
defer deleteS3Bucket(t, b.client, bucketName)
createS3Bucket(t, b.s3Client, bucketName)
defer deleteS3Bucket(t, b.s3Client, bucketName)
remote.TestClient(t, state.(*remote.State).Client)
}
@ -67,10 +67,10 @@ func TestRemoteClientLocks(t *testing.T) {
t.Fatal(err)
}
createS3Bucket(t, b1.client, bucketName)
defer deleteS3Bucket(t, b1.client, bucketName)
createDynamoDBTable(t, b1.client, bucketName)
defer deleteDynamoDBTable(t, b1.client, bucketName)
createS3Bucket(t, b1.s3Client, bucketName)
defer deleteS3Bucket(t, b1.s3Client, bucketName)
createDynamoDBTable(t, b1.dynClient, bucketName)
defer deleteDynamoDBTable(t, b1.dynClient, bucketName)
remote.TestRemoteLocks(t, s1.(*remote.State).Client, s2.(*remote.State).Client)
}