resilien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #16865 from hashicorp/jbardin/gcs 

gcs provider fixes
This commit is contained in:
James Bardin 2017-12-07 09:39:55 -05:00 committed by GitHub
parent f461c1750c 52eced589f
commit bbc20cfec9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 101 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
backend/remote-state/gcs/backend.go
View File

@ -3,14 +3,17 @@ package gcs
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"cloud.google.com/go/storage"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/helper/pathorcontents"
"github.com/hashicorp/terraform/helper/schema"
"github.com/hashicorp/terraform/terraform"
"golang.org/x/oauth2/jwt"
"google.golang.org/api/option"
)
@ -111,16 +114,39 @@ func (b *gcsBackend) configure(ctx context.Context) error {
b.region = r
}
opts := []option.ClientOption{
option.WithScopes(storage.ScopeReadWrite),
option.WithUserAgent(terraform.UserAgentString()),
}
if credentialsFile := data.Get("credentials").(string); credentialsFile != "" {
opts = append(opts, option.WithCredentialsFile(credentialsFile))
} else if credentialsFile := os.Getenv("GOOGLE_CREDENTIALS"); credentialsFile != "" {
opts = append(opts, option.WithCredentialsFile(credentialsFile))
var opts []option.ClientOption
creds := data.Get("credentials").(string)
if creds == "" {
creds = os.Getenv("GOOGLE_CREDENTIALS")
}
if creds != "" {
var account accountFile
// to mirror how the provider works, we accept the file path or the contents
contents, _, err := pathorcontents.Read(creds)
if err != nil {
return fmt.Errorf("Error loading credentials: %s", err)
}
if err := json.Unmarshal([]byte(contents), &account); err != nil {
return fmt.Errorf("Error parsing credentials '%s': %s", contents, err)
}
conf := jwt.Config{
Email: account.ClientEmail,
PrivateKey: []byte(account.PrivateKey),
Scopes: []string{storage.ScopeReadWrite},
TokenURL: "https://accounts.google.com/o/oauth2/token",
}
opts = append(opts, option.WithHTTPClient(conf.Client(ctx)))
} else {
opts = append(opts, option.WithScopes(storage.ScopeReadWrite))
}
opts = append(opts, option.WithUserAgent(terraform.UserAgentString()))
client, err := storage.NewClient(b.storageContext, opts...)
if err != nil {
return fmt.Errorf("storage.NewClient() failed: %v", err)
@ -128,22 +154,13 @@ func (b *gcsBackend) configure(ctx context.Context) error {
b.storageClient = client
return b.ensureBucketExists()
return nil
}
func (b *gcsBackend) ensureBucketExists() error {
_, err := b.storageClient.Bucket(b.bucketName).Attrs(b.storageContext)
if err != storage.ErrBucketNotExist {
return err
}
if b.projectID == "" {
return fmt.Errorf("bucket %q does not exist; specify the \"project\" option or create the bucket manually using `gsutil mb gs://%s`", b.bucketName, b.bucketName)
}
attrs := &storage.BucketAttrs{
Location: b.region,
}
return b.storageClient.Bucket(b.bucketName).Create(b.storageContext, b.projectID, attrs)
// accountFile represents the structure of the account file JSON file.
type accountFile struct {
PrivateKeyId string `json:"private_key_id"`
PrivateKey string `json:"private_key"`
ClientEmail string `json:"client_email"`
ClientId string `json:"client_id"`
}

121
backend/remote-state/gcs/backend_test.go
View File

@ -2,10 +2,13 @@ package gcs
import (
"fmt"
"log"
"os"
"strings"
"testing"
"time"
"cloud.google.com/go/storage"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/state/remote"
)
@ -48,7 +51,8 @@ func TestStateFile(t *testing.T) {
func TestRemoteClient(t *testing.T) {
t.Parallel()
be := setupBackend(t, noPrefix)
bucket := bucketName(t)
be := setupBackend(t, bucket, noPrefix)
defer teardownBackend(t, be, noPrefix)
ss, err := be.State(backend.DefaultStateName)
@ -67,7 +71,8 @@ func TestRemoteClient(t *testing.T) {
func TestRemoteLocks(t *testing.T) {
t.Parallel()
be := setupBackend(t, noPrefix)
bucket := bucketName(t)
be := setupBackend(t, bucket, noPrefix)
defer teardownBackend(t, be, noPrefix)
remoteClient := func() (remote.Client, error) {
@ -99,10 +104,12 @@ func TestRemoteLocks(t *testing.T) {
func TestBackend(t *testing.T) {
t.Parallel()
be0 := setupBackend(t, noPrefix)
bucket := bucketName(t)
be0 := setupBackend(t, bucket, noPrefix)
defer teardownBackend(t, be0, noPrefix)
be1 := setupBackend(t, noPrefix)
be1 := setupBackend(t, bucket, noPrefix)
backend.TestBackend(t, be0, be1)
}
@ -110,17 +117,18 @@ func TestBackendWithPrefix(t *testing.T) {
t.Parallel()
prefix := "test/prefix"
bucket := bucketName(t)
be0 := setupBackend(t, prefix)
be0 := setupBackend(t, bucket, prefix)
defer teardownBackend(t, be0, prefix)
be1 := setupBackend(t, prefix+"/")
be1 := setupBackend(t, bucket, prefix+"/")
backend.TestBackend(t, be0, be1)
}
// setupBackend returns a new GCS backend.
func setupBackend(t *testing.T, prefix string) backend.Backend {
func setupBackend(t *testing.T, bucket, prefix string) backend.Backend {
t.Helper()
projectID := os.Getenv("GOOGLE_PROJECT")
@ -132,92 +140,67 @@ func setupBackend(t *testing.T, prefix string) backend.Backend {
config := map[string]interface{}{
"project": projectID,
"bucket": toBucketName(projectID + "-" + t.Name()),
"bucket": bucket,
"prefix": prefix,
}
if creds := os.Getenv("GOOGLE_CREDENTIALS"); creds != "" {
config["credentials"] = creds
t.Logf("using credentials from %q", creds)
} else {
t.Log("using default credentials; set GOOGLE_CREDENTIALS for custom credentials")
b := backend.TestBackendConfig(t, New(), config)
be := b.(*gcsBackend)
// create the bucket if it doesn't exist
bkt := be.storageClient.Bucket(bucket)
_, err := bkt.Attrs(be.storageContext)
if err != nil {
if err != storage.ErrBucketNotExist {
t.Fatal(err)
}
attrs := &storage.BucketAttrs{
Location: be.region,
}
err := bkt.Create(be.storageContext, be.projectID, attrs)
if err != nil {
t.Fatal(err)
}
}
return backend.TestBackendConfig(t, New(), config)
return b
}
// teardownBackend deletes all states from be except the default state.
func teardownBackend(t *testing.T, be backend.Backend, prefix string) {
t.Helper()
// Delete all states. The bucket must be empty before it can be deleted.
states, err := be.States()
if err != nil {
t.Fatalf("be.States() = %v; manual clean-up may be required", err)
}
for _, st := range states {
if st == backend.DefaultStateName {
continue
}
if err := be.DeleteState(st); err != nil {
t.Fatalf("be.DeleteState(%q) = %v; manual clean-up may be required", st, err)
}
}
gcsBE, ok := be.(*gcsBackend)
if !ok {
t.Fatalf("be is a %T, want a *gcsBackend", be)
}
ctx := gcsBE.storageContext
// Delete the default state, which DeleteState() will refuse to do.
// It's okay if this fails, not all tests create a default state.
ds := "default.tfstate"
if prefix != "" {
ds = fmt.Sprintf("%s/%s", prefix, ds)
}
if err := gcsBE.storageClient.Bucket(gcsBE.bucketName).Object(ds).Delete(ctx); err != nil {
t.Logf("deleting \"%s\": %v; manual clean-up may be required", ds, err)
bucket := gcsBE.storageClient.Bucket(gcsBE.bucketName)
objs := bucket.Objects(ctx, nil)
for o, err := objs.Next(); err == nil; o, err = objs.Next() {
if err := bucket.Object(o.Name).Delete(ctx); err != nil {
log.Printf("Error trying to delete object: %s %s\n\n", o.Name, err)
} else {
log.Printf("Object deleted: %s", o.Name)
}
}
// Delete the bucket itself.
if err := gcsBE.storageClient.Bucket(gcsBE.bucketName).Delete(ctx); err != nil {
t.Fatalf("deleting bucket failed: %v; manual cleanup may be required, though later test runs will happily reuse an existing bucket", err)
if err := bucket.Delete(ctx); err != nil {
t.Errorf("deleting bucket %q failed, manual cleanup may be required: %v", gcsBE.bucketName, err)
}
}
// toBucketName returns a copy of in that is suitable for use as a bucket name.
// All upper case characters are converted to lower case, other invalid
// characters are replaced by '_'.
func toBucketName(in string) string {
// Bucket names must contain only lowercase letters, numbers, dashes
// (-), and underscores (_).
isValid := func(r rune) bool {
switch {
case r >= 'a' && r <= 'z':
return true
case r >= '0' && r <= '9':
return true
case r == '-' || r == '_':
return true
default:
return false
}
}
out := make([]rune, 0, len(in))
for _, r := range strings.ToLower(in) {
if !isValid(r) {
r = '_'
}
out = append(out, r)
}
// bucketName returns a valid bucket name for this test.
func bucketName(t *testing.T) string {
name := fmt.Sprintf("tf-%x-%s", time.Now().UnixNano(), t.Name())
// Bucket names must contain 3 to 63 characters.
if len(out) > 63 {
out = out[:63]
if len(name) > 63 {
name = name[:63]
}
return string(out)
return strings.ToLower(name)
}

8
backend/testing.go
View File

@ -281,6 +281,14 @@ func testBackendStateLock(t *testing.T, b1, b2 Backend) {
t.Fatal("unable to get initial lock:", err)
}
// Make sure we can still get the state.State from another instance even
// when locked. This should only happen when a state is loaded via the
// backend, and as a remote state.
_, err = b2.State(DefaultStateName)
if err != nil {
t.Fatalf("failed to read locked state from another backend instance: %s", err)
}
// If the lock ID is blank, assume locking is disabled
if lockIDA == "" {
t.Logf("TestBackend: %T: empty string returned for lock, assuming disabled", b1)