Merge pull request #1723 from apparentlymart/s3remotestate
S3 Remote State Backend
This commit is contained in:
Mitchell Hashimoto
commit
af5ac59188
|
|
@ -39,6 +39,7 @@ var BuiltinClients = map[string]Factory{
|
|||
"atlas": atlasFactory,
|
||||
"consul": consulFactory,
|
||||
"http": httpFactory,
|
||||
"s3": s3Factory,
|
||||
|
||||
// This is used for development purposes only.
|
||||
"_local": fileFactory,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,125 @@
|
|||
package remote
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/awslabs/aws-sdk-go/aws"
|
||||
"github.com/awslabs/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
func s3Factory(conf map[string]string) (Client, error) {
|
||||
bucketName, ok := conf["bucket"]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("missing 'bucket' configuration")
|
||||
}
|
||||
|
||||
keyName, ok := conf["key"]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("missing 'key' configuration")
|
||||
}
|
||||
|
||||
regionName, ok := conf["region"]
|
||||
if !ok {
|
||||
regionName = os.Getenv("AWS_DEFAULT_REGION")
|
||||
if regionName == "" {
|
||||
return nil, fmt.Errorf("missing 'region' configuration or AWS_DEFAULT_REGION environment variable")
|
||||
}
|
||||
}
|
||||
|
||||
accessKeyId := conf["access_key"]
|
||||
secretAccessKey := conf["secret_key"]
|
||||
|
||||
credentialsProvider := aws.DetectCreds(accessKeyId, secretAccessKey, "")
|
||||
|
||||
// Make sure we got some sort of working credentials.
|
||||
_, err := credentialsProvider.Credentials()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Unable to determine AWS credentials. Set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.\n(error was: %s)", err)
|
||||
}
|
||||
|
||||
awsConfig := &aws.Config{
|
||||
Credentials: credentialsProvider,
|
||||
Region: regionName,
|
||||
}
|
||||
nativeClient := s3.New(awsConfig)
|
||||
|
||||
return &S3Client{
|
||||
nativeClient: nativeClient,
|
||||
bucketName: bucketName,
|
||||
keyName: keyName,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type S3Client struct {
|
||||
nativeClient *s3.S3
|
||||
bucketName string
|
||||
keyName string
|
||||
}
|
||||
|
||||
func (c *S3Client) Get() (*Payload, error) {
|
||||
output, err := c.nativeClient.GetObject(&s3.GetObjectInput{
|
||||
Bucket: &c.bucketName,
|
||||
Key: &c.keyName,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if awserr := aws.Error(err); awserr != nil {
|
||||
if awserr.Code == "NoSuchKey" {
|
||||
return nil, nil
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
defer output.Body.Close()
|
||||
|
||||
buf := bytes.NewBuffer(nil)
|
||||
if _, err := io.Copy(buf, output.Body); err != nil {
|
||||
return nil, fmt.Errorf("Failed to read remote state: %s", err)
|
||||
}
|
||||
|
||||
payload := &Payload{
|
||||
Data: buf.Bytes(),
|
||||
}
|
||||
|
||||
// If there was no data, then return nil
|
||||
if len(payload.Data) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func (c *S3Client) Put(data []byte) error {
|
||||
contentType := "application/octet-stream"
|
||||
contentLength := int64(len(data))
|
||||
|
||||
_, err := c.nativeClient.PutObject(&s3.PutObjectInput{
|
||||
ContentType: &contentType,
|
||||
ContentLength: &contentLength,
|
||||
Body: bytes.NewReader(data),
|
||||
Bucket: &c.bucketName,
|
||||
Key: &c.keyName,
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
return nil
|
||||
} else {
|
||||
return fmt.Errorf("Failed to upload state: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *S3Client) Delete() error {
|
||||
_, err := c.nativeClient.DeleteObject(&s3.DeleteObjectInput{
|
||||
Bucket: &c.bucketName,
|
||||
Key: &c.keyName,
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
package remote
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/awslabs/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
func TestS3Client_impl(t *testing.T) {
|
||||
var _ Client = new(S3Client)
|
||||
}
|
||||
|
||||
func TestS3Factory(t *testing.T) {
|
||||
// This test just instantiates the client. Shouldn't make any actual
|
||||
// requests nor incur any costs.
|
||||
|
||||
config := make(map[string]string)
|
||||
|
||||
// Empty config is an error
|
||||
_, err := s3Factory(config)
|
||||
if err == nil {
|
||||
t.Fatalf("Empty config should be error")
|
||||
}
|
||||
|
||||
config["region"] = "us-west-1"
|
||||
config["bucket"] = "foo"
|
||||
config["key"] = "bar"
|
||||
// For this test we'll provide the credentials as config. The
|
||||
// acceptance tests implicitly test passing credentials as
|
||||
// environment variables.
|
||||
config["access_key"] = "bazkey"
|
||||
config["secret_key"] = "bazsecret"
|
||||
|
||||
client, err := s3Factory(config)
|
||||
if err != nil {
|
||||
t.Fatalf("Error for valid config")
|
||||
}
|
||||
|
||||
s3Client := client.(*S3Client)
|
||||
|
||||
if s3Client.nativeClient.Config.Region != "us-west-1" {
|
||||
t.Fatalf("Incorrect region was populated")
|
||||
}
|
||||
if s3Client.bucketName != "foo" {
|
||||
t.Fatalf("Incorrect bucketName was populated")
|
||||
}
|
||||
if s3Client.keyName != "bar" {
|
||||
t.Fatalf("Incorrect keyName was populated")
|
||||
}
|
||||
|
||||
credentials, err := s3Client.nativeClient.Config.Credentials.Credentials()
|
||||
if err != nil {
|
||||
t.Fatalf("Error when requesting credentials")
|
||||
}
|
||||
if credentials.AccessKeyID != "bazkey" {
|
||||
t.Fatalf("Incorrect Access Key Id was populated")
|
||||
}
|
||||
if credentials.SecretAccessKey != "bazsecret" {
|
||||
t.Fatalf("Incorrect Secret Access Key was populated")
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3Client(t *testing.T) {
|
||||
// This test creates a bucket in S3 and populates it.
|
||||
// It may incur costs, so it will only run if AWS credential environment
|
||||
// variables are present.
|
||||
|
||||
accessKeyId := os.Getenv("AWS_ACCESS_KEY_ID")
|
||||
if accessKeyId == "" {
|
||||
t.Skipf("skipping; AWS_ACCESS_KEY_ID must be set")
|
||||
}
|
||||
|
||||
regionName := os.Getenv("AWS_DEFAULT_REGION")
|
||||
if regionName == "" {
|
||||
regionName = "us-west-2"
|
||||
}
|
||||
|
||||
bucketName := fmt.Sprintf("terraform-remote-s3-test-%x", time.Now().Unix())
|
||||
keyName := "testState"
|
||||
|
||||
config := make(map[string]string)
|
||||
config["region"] = regionName
|
||||
config["bucket"] = bucketName
|
||||
config["key"] = keyName
|
||||
|
||||
client, err := s3Factory(config)
|
||||
if err != nil {
|
||||
t.Fatalf("Error for valid config")
|
||||
}
|
||||
|
||||
s3Client := client.(*S3Client)
|
||||
nativeClient := s3Client.nativeClient
|
||||
|
||||
createBucketReq := &s3.CreateBucketInput{
|
||||
Bucket: &bucketName,
|
||||
}
|
||||
|
||||
// Be clear about what we're doing in case the user needs to clean
|
||||
// this up later.
|
||||
t.Logf("Creating S3 bucket %s in %s", bucketName, regionName)
|
||||
_, err = nativeClient.CreateBucket(createBucketReq)
|
||||
if err != nil {
|
||||
t.Skipf("Failed to create test S3 bucket, so skipping")
|
||||
}
|
||||
defer func () {
|
||||
deleteBucketReq := &s3.DeleteBucketInput{
|
||||
Bucket: &bucketName,
|
||||
}
|
||||
|
||||
_, err := nativeClient.DeleteBucket(deleteBucketReq)
|
||||
if err != nil {
|
||||
t.Logf("WARNING: Failed to delete the test S3 bucket. It has been left in your AWS account and may incur storage charges. (error was %s)", err)
|
||||
}
|
||||
}()
|
||||
|
||||
testClient(t, client)
|
||||
}
|
||||
|
|
@ -50,6 +50,14 @@ The following backends are supported:
|
|||
variables can optionally be provided. Address is assumed to be the
|
||||
local agent if not provided.
|
||||
|
||||
* S3 - Stores the state as a given key in a given bucket on Amazon S3.
|
||||
Requires the `bucket` and `key` variables. Supports and honors the standard
|
||||
AWS environment variables `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`
|
||||
and `AWS_DEFAULT_REGION`. These can optionally be provided as parameters
|
||||
in the `aws_access_key`, `aws_secret_key` and `region` variables
|
||||
respectively, but passing credentials this way is not recommended since they
|
||||
will be included in cleartext inside the persisted state.
|
||||
|
||||
* HTTP - Stores the state using a simple REST client. State will be fetched
|
||||
via GET, updated via POST, and purged with DELETE. Requires the `address` variable.
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue