diff --git a/backend/init/init.go b/backend/init/init.go index 685276dde..eb74ebeb5 100644 --- a/backend/init/init.go +++ b/backend/init/init.go @@ -13,6 +13,7 @@ import ( backendconsul "github.com/hashicorp/terraform/backend/remote-state/consul" backendinmem "github.com/hashicorp/terraform/backend/remote-state/inmem" backendS3 "github.com/hashicorp/terraform/backend/remote-state/s3" + backendSwift "github.com/hashicorp/terraform/backend/remote-state/swift" ) // backends is the list of available backends. This is a global variable @@ -37,6 +38,7 @@ func init() { "local": func() backend.Backend { return &backendlocal.Local{} }, "consul": func() backend.Backend { return backendconsul.New() }, "inmem": func() backend.Backend { return backendinmem.New() }, + "swift": func() backend.Backend { return backendSwift.New() }, "s3": func() backend.Backend { return backendS3.New() }, } diff --git a/backend/remote-state/swift/backend.go b/backend/remote-state/swift/backend.go new file mode 100644 index 000000000..22f49edae --- /dev/null +++ b/backend/remote-state/swift/backend.go @@ -0,0 +1,325 @@ +package swift + +import ( + "context" + "fmt" + "log" + "strconv" + "strings" + "time" + + "github.com/gophercloud/gophercloud" + "github.com/gophercloud/gophercloud/openstack" + + "github.com/hashicorp/terraform/backend" + tf_openstack "github.com/hashicorp/terraform/builtin/providers/openstack" + "github.com/hashicorp/terraform/helper/schema" +) + +// New creates a new backend for Swift remote state. +func New() backend.Backend { + s := &schema.Backend{ + Schema: map[string]*schema.Schema{ + "auth_url": &schema.Schema{ + Type: schema.TypeString, + Required: true, + DefaultFunc: schema.EnvDefaultFunc("OS_AUTH_URL", nil), + Description: descriptions["auth_url"], + }, + + "user_id": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + DefaultFunc: schema.EnvDefaultFunc("OS_USER_ID", ""), + Description: descriptions["user_name"], + }, + + "user_name": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + DefaultFunc: schema.EnvDefaultFunc("OS_USERNAME", ""), + Description: descriptions["user_name"], + }, + + "tenant_id": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + DefaultFunc: schema.MultiEnvDefaultFunc([]string{ + "OS_TENANT_ID", + "OS_PROJECT_ID", + }, ""), + Description: descriptions["tenant_id"], + }, + + "tenant_name": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + DefaultFunc: schema.MultiEnvDefaultFunc([]string{ + "OS_TENANT_NAME", + "OS_PROJECT_NAME", + }, ""), + Description: descriptions["tenant_name"], + }, + + "password": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Sensitive: true, + DefaultFunc: schema.EnvDefaultFunc("OS_PASSWORD", ""), + Description: descriptions["password"], + }, + + "token": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + DefaultFunc: schema.EnvDefaultFunc("OS_AUTH_TOKEN", ""), + Description: descriptions["token"], + }, + + "domain_id": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + DefaultFunc: schema.MultiEnvDefaultFunc([]string{ + "OS_USER_DOMAIN_ID", + "OS_PROJECT_DOMAIN_ID", + "OS_DOMAIN_ID", + }, ""), + Description: descriptions["domain_id"], + }, + + "domain_name": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + DefaultFunc: schema.MultiEnvDefaultFunc([]string{ + "OS_USER_DOMAIN_NAME", + "OS_PROJECT_DOMAIN_NAME", + "OS_DOMAIN_NAME", + "OS_DEFAULT_DOMAIN", + }, ""), + Description: descriptions["domain_name"], + }, + + "region_name": &schema.Schema{ + Type: schema.TypeString, + Required: true, + DefaultFunc: schema.EnvDefaultFunc("OS_REGION_NAME", ""), + Description: descriptions["region_name"], + }, + + "insecure": &schema.Schema{ + Type: schema.TypeBool, + Optional: true, + DefaultFunc: schema.EnvDefaultFunc("OS_INSECURE", ""), + Description: descriptions["insecure"], + }, + + "endpoint_type": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + DefaultFunc: schema.EnvDefaultFunc("OS_ENDPOINT_TYPE", ""), + }, + + "cacert_file": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + DefaultFunc: schema.EnvDefaultFunc("OS_CACERT", ""), + Description: descriptions["cacert_file"], + }, + + "cert": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + DefaultFunc: schema.EnvDefaultFunc("OS_CERT", ""), + Description: descriptions["cert"], + }, + + "key": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + DefaultFunc: schema.EnvDefaultFunc("OS_KEY", ""), + Description: descriptions["key"], + }, + + "path": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Description: descriptions["path"], + Deprecated: "Use container instead", + ConflictsWith: []string{"container"}, + }, + + "container": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Description: descriptions["container"], + }, + + "archive_path": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Description: descriptions["archive_path"], + Deprecated: "Use archive_container instead", + ConflictsWith: []string{"archive_container"}, + }, + + "archive_container": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Description: descriptions["archive_container"], + }, + + "expire_after": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Description: descriptions["expire_after"], + }, + }, + } + + result := &Backend{Backend: s} + result.Backend.ConfigureFunc = result.configure + return result +} + +var descriptions map[string]string + +func init() { + descriptions = map[string]string{ + "auth_url": "The Identity authentication URL.", + + "user_name": "Username to login with.", + + "user_id": "User ID to login with.", + + "tenant_id": "The ID of the Tenant (Identity v2) or Project (Identity v3)\n" + + "to login with.", + + "tenant_name": "The name of the Tenant (Identity v2) or Project (Identity v3)\n" + + "to login with.", + + "password": "Password to login with.", + + "token": "Authentication token to use as an alternative to username/password.", + + "domain_id": "The ID of the Domain to scope to (Identity v3).", + + "domain_name": "The name of the Domain to scope to (Identity v3).", + + "region_name": "The name of the Region to use.", + + "insecure": "Trust self-signed certificates.", + + "cacert_file": "A Custom CA certificate.", + + "endpoint_type": "The catalog endpoint type to use.", + + "cert": "A client certificate to authenticate with.", + + "key": "A client private key to authenticate with.", + + "path": "Swift container path to use.", + + "container": "Swift container to create", + + "archive_path": "Swift container path to archive state to.", + + "archive_container": "Swift container to archive state to.", + + "expire_after": "Archive object expiry duration.", + } +} + +type Backend struct { + *schema.Backend + + // Fields below are set from configure + client *gophercloud.ServiceClient + archive bool + archiveContainer string + expireSecs int + container string +} + +func (b *Backend) configure(ctx context.Context) error { + if b.client != nil { + return nil + } + + // Grab the resource data + data := schema.FromContextBackendConfig(ctx) + + config := &tf_openstack.Config{ + CACertFile: data.Get("cacert_file").(string), + ClientCertFile: data.Get("cert").(string), + ClientKeyFile: data.Get("key").(string), + DomainID: data.Get("domain_id").(string), + DomainName: data.Get("domain_name").(string), + EndpointType: data.Get("endpoint_type").(string), + IdentityEndpoint: data.Get("auth_url").(string), + Insecure: data.Get("insecure").(bool), + Password: data.Get("password").(string), + Token: data.Get("token").(string), + TenantID: data.Get("tenant_id").(string), + TenantName: data.Get("tenant_name").(string), + Username: data.Get("user_name").(string), + UserID: data.Get("user_id").(string), + } + + if err := config.LoadAndValidate(); err != nil { + return err + } + + // Assign Container + b.container = data.Get("container").(string) + if b.container == "" { + // Check deprecated field + b.container = data.Get("path").(string) + } + + // Enable object archiving? + if archiveContainer, ok := data.GetOk("archive_container"); ok { + log.Printf("[DEBUG] Archive_container set, enabling object versioning") + b.archive = true + b.archiveContainer = archiveContainer.(string) + } else if archivePath, ok := data.GetOk("archive_path"); ok { + log.Printf("[DEBUG] Archive_path set, enabling object versioning") + b.archive = true + b.archiveContainer = archivePath.(string) + } + + // Enable object expiry? + if expireRaw, ok := data.GetOk("expire_after"); ok { + expire := expireRaw.(string) + log.Printf("[DEBUG] Requested that remote state expires after %s", expire) + + if strings.HasSuffix(expire, "d") { + log.Printf("[DEBUG] Got a days expire after duration. Converting to hours") + days, err := strconv.Atoi(expire[:len(expire)-1]) + if err != nil { + return fmt.Errorf("Error converting expire_after value %s to int: %s", expire, err) + } + + expire = fmt.Sprintf("%dh", days*24) + log.Printf("[DEBUG] Expire after %s hours", expire) + } + + expireDur, err := time.ParseDuration(expire) + if err != nil { + log.Printf("[DEBUG] Error parsing duration %s: %s", expire, err) + return fmt.Errorf("Error parsing expire_after duration '%s': %s", expire, err) + } + log.Printf("[DEBUG] Seconds duration = %d", int(expireDur.Seconds())) + b.expireSecs = int(expireDur.Seconds()) + } + + objClient, err := openstack.NewObjectStorageV1(config.OsClient, gophercloud.EndpointOpts{ + Region: data.Get("region_name").(string), + }) + if err != nil { + return err + } + + b.client = objClient + + return nil +} diff --git a/backend/remote-state/swift/backend_state.go b/backend/remote-state/swift/backend_state.go new file mode 100644 index 000000000..b8ab98107 --- /dev/null +++ b/backend/remote-state/swift/backend_state.go @@ -0,0 +1,31 @@ +package swift + +import ( + "github.com/hashicorp/terraform/backend" + "github.com/hashicorp/terraform/state" + "github.com/hashicorp/terraform/state/remote" +) + +func (b *Backend) States() ([]string, error) { + return nil, backend.ErrNamedStatesNotSupported +} + +func (b *Backend) DeleteState(name string) error { + return backend.ErrNamedStatesNotSupported +} + +func (b *Backend) State(name string) (state.State, error) { + if name != backend.DefaultStateName { + return nil, backend.ErrNamedStatesNotSupported + } + + client := &RemoteClient{ + client: b.client, + container: b.container, + archive: b.archive, + archiveContainer: b.archiveContainer, + expireSecs: b.expireSecs, + } + + return &remote.State{Client: client}, nil +} diff --git a/backend/remote-state/swift/backend_test.go b/backend/remote-state/swift/backend_test.go new file mode 100644 index 000000000..d3dbe5acd --- /dev/null +++ b/backend/remote-state/swift/backend_test.go @@ -0,0 +1,259 @@ +package swift + +import ( + "fmt" + "io" + "os" + "testing" + "time" + + "github.com/gophercloud/gophercloud" + "github.com/gophercloud/gophercloud/openstack/objectstorage/v1/containers" + "github.com/gophercloud/gophercloud/openstack/objectstorage/v1/objects" + "github.com/gophercloud/gophercloud/pagination" + "github.com/hashicorp/terraform/backend" + "github.com/hashicorp/terraform/state/remote" + "github.com/hashicorp/terraform/terraform" +) + +// verify that we are doing ACC tests or the Swift tests specifically +func testACC(t *testing.T) { + skip := os.Getenv("TF_ACC") == "" && os.Getenv("TF_SWIFT_TEST") == "" + if skip { + t.Log("swift backend tests require setting TF_ACC or TF_SWIFT_TEST") + t.Skip() + } + t.Log("swift backend acceptance tests enabled") +} + +func TestBackend_impl(t *testing.T) { + var _ backend.Backend = new(Backend) +} + +func testAccPreCheck(t *testing.T) { + v := os.Getenv("OS_AUTH_URL") + if v == "" { + t.Fatal("OS_AUTH_URL must be set for acceptance tests") + } +} + +func TestBackendConfig(t *testing.T) { + testACC(t) + + // Build config + config := map[string]interface{}{ + "archive_container": "test-tfstate-archive", + "container": "test-tfstate", + } + + b := backend.TestBackendConfig(t, New(), config).(*Backend) + + if b.container != "test-tfstate" { + t.Fatal("Incorrect path was provided.") + } + if b.archiveContainer != "test-tfstate-archive" { + t.Fatal("Incorrect archivepath was provided.") + } +} + +func TestBackend(t *testing.T) { + testACC(t) + + container := fmt.Sprintf("terraform-state-swift-test-%x", time.Now().Unix()) + + b := backend.TestBackendConfig(t, New(), map[string]interface{}{ + "container": container, + }).(*Backend) + + defer deleteSwiftContainer(t, b.client, container) + + backend.TestBackend(t, b, nil) +} + +func TestBackendPath(t *testing.T) { + testACC(t) + + path := fmt.Sprintf("terraform-state-swift-test-%x", time.Now().Unix()) + t.Logf("[DEBUG] Generating backend config") + b := backend.TestBackendConfig(t, New(), map[string]interface{}{ + "path": path, + }).(*Backend) + t.Logf("[DEBUG] Backend configured") + + defer deleteSwiftContainer(t, b.client, path) + + t.Logf("[DEBUG] Testing Backend") + + // Generate some state + state1 := terraform.NewState() + // state1Lineage := state1.Lineage + t.Logf("state1 lineage = %s, serial = %d", state1.Lineage, state1.Serial) + + // RemoteClient to test with + client := &RemoteClient{ + client: b.client, + archive: b.archive, + archiveContainer: b.archiveContainer, + container: b.container, + } + + stateMgr := &remote.State{Client: client} + stateMgr.WriteState(state1) + if err := stateMgr.PersistState(); err != nil { + t.Fatal(err) + } + + if err := stateMgr.RefreshState(); err != nil { + t.Fatal(err) + } + + // Add some state + state1.AddModuleState(&terraform.ModuleState{ + Path: []string{"root"}, + Outputs: map[string]*terraform.OutputState{ + "bar": &terraform.OutputState{ + Type: "string", + Sensitive: false, + Value: "baz", + }, + }, + }) + stateMgr.WriteState(state1) + if err := stateMgr.PersistState(); err != nil { + t.Fatal(err) + } + +} + +func TestBackendArchive(t *testing.T) { + testACC(t) + + container := fmt.Sprintf("terraform-state-swift-test-%x", time.Now().Unix()) + archiveContainer := fmt.Sprintf("%s_archive", container) + + b := backend.TestBackendConfig(t, New(), map[string]interface{}{ + "archive_container": archiveContainer, + "container": container, + }).(*Backend) + + defer deleteSwiftContainer(t, b.client, container) + defer deleteSwiftContainer(t, b.client, archiveContainer) + + // Generate some state + state1 := terraform.NewState() + // state1Lineage := state1.Lineage + t.Logf("state1 lineage = %s, serial = %d", state1.Lineage, state1.Serial) + + // RemoteClient to test with + client := &RemoteClient{ + client: b.client, + archive: b.archive, + archiveContainer: b.archiveContainer, + container: b.container, + } + + stateMgr := &remote.State{Client: client} + stateMgr.WriteState(state1) + if err := stateMgr.PersistState(); err != nil { + t.Fatal(err) + } + + if err := stateMgr.RefreshState(); err != nil { + t.Fatal(err) + } + + // Add some state + state1.AddModuleState(&terraform.ModuleState{ + Path: []string{"root"}, + Outputs: map[string]*terraform.OutputState{ + "bar": &terraform.OutputState{ + Type: "string", + Sensitive: false, + Value: "baz", + }, + }, + }) + stateMgr.WriteState(state1) + if err := stateMgr.PersistState(); err != nil { + t.Fatal(err) + } + + archiveObjects := getSwiftObjectNames(t, b.client, archiveContainer) + t.Logf("archiveObjects len = %d. Contents = %+v", len(archiveObjects), archiveObjects) + if len(archiveObjects) != 1 { + t.Fatalf("Invalid number of archive objects. Expected 1, got %d", len(archiveObjects)) + } + + // Download archive state to validate + archiveData := downloadSwiftObject(t, b.client, archiveContainer, archiveObjects[0]) + t.Logf("Archive data downloaded... Looks like: %+v", archiveData) + archiveState, err := terraform.ReadState(archiveData) + if err != nil { + t.Fatalf("Error Reading State: %s", err) + } + + t.Logf("Archive state lineage = %s, serial = %d, lineage match = %t", archiveState.Lineage, archiveState.Serial, stateMgr.State().SameLineage(archiveState)) + if !stateMgr.State().SameLineage(archiveState) { + t.Fatal("Got a different lineage") + } + +} + +// Helper function to download an object in a Swift container +func downloadSwiftObject(t *testing.T, osClient *gophercloud.ServiceClient, container, object string) (data io.Reader) { + t.Logf("Attempting to download object %s from container %s", object, container) + res := objects.Download(osClient, container, object, nil) + if res.Err != nil { + t.Fatalf("Error downloading object: %s", res.Err) + } + data = res.Body + return +} + +// Helper function to get a list of objects in a Swift container +func getSwiftObjectNames(t *testing.T, osClient *gophercloud.ServiceClient, container string) (objectNames []string) { + _ = objects.List(osClient, container, nil).EachPage(func(page pagination.Page) (bool, error) { + + // Get a slice of object names + names, err := objects.ExtractNames(page) + if err != nil { + t.Fatalf("Error extracting object names from page: %s", err) + } + for _, object := range names { + objectNames = append(objectNames, object) + } + + return true, nil + }) + return +} + +// Helper function to delete Swift container +func deleteSwiftContainer(t *testing.T, osClient *gophercloud.ServiceClient, container string) { + warning := "WARNING: Failed to delete the test Swift container. It may have been left in your Openstack account and may incur storage charges. (error was %s)" + + // Remove any objects + deleteSwiftObjects(t, osClient, container) + + // Delete the container + deleteResult := containers.Delete(osClient, container) + if deleteResult.Err != nil { + if _, ok := deleteResult.Err.(gophercloud.ErrDefault404); !ok { + t.Fatalf(warning, deleteResult.Err) + } + } +} + +// Helper function to delete Swift objects within a container +func deleteSwiftObjects(t *testing.T, osClient *gophercloud.ServiceClient, container string) { + // Get a slice of object names + objectNames := getSwiftObjectNames(t, osClient, container) + + for _, object := range objectNames { + result := objects.Delete(osClient, container, object, nil) + if result.Err != nil { + t.Fatalf("Error deleting object %s from container %s: %s", object, container, result.Err) + } + } + +} diff --git a/backend/remote-state/swift/client.go b/backend/remote-state/swift/client.go new file mode 100644 index 000000000..1f8bf4649 --- /dev/null +++ b/backend/remote-state/swift/client.go @@ -0,0 +1,115 @@ +package swift + +import ( + "bytes" + "crypto/md5" + "log" + "os" + + "github.com/gophercloud/gophercloud" + "github.com/gophercloud/gophercloud/openstack/objectstorage/v1/containers" + "github.com/gophercloud/gophercloud/openstack/objectstorage/v1/objects" + + "github.com/hashicorp/terraform/state/remote" +) + +const ( + TFSTATE_NAME = "tfstate.tf" + TFSTATE_LOCK_NAME = "tfstate.lock" +) + +// RemoteClient implements the Client interface for an Openstack Swift server. +type RemoteClient struct { + client *gophercloud.ServiceClient + container string + archive bool + archiveContainer string + expireSecs int +} + +func (c *RemoteClient) Get() (*remote.Payload, error) { + log.Printf("[DEBUG] Getting object %s in container %s", TFSTATE_NAME, c.container) + result := objects.Download(c.client, c.container, TFSTATE_NAME, nil) + + // Extract any errors from result + _, err := result.Extract() + + // 404 response is to be expected if the object doesn't already exist! + if _, ok := err.(gophercloud.ErrDefault404); ok { + log.Println("[DEBUG] Object doesn't exist to download.") + return nil, nil + } + + bytes, err := result.ExtractContent() + if err != nil { + return nil, err + } + + hash := md5.Sum(bytes) + payload := &remote.Payload{ + Data: bytes, + MD5: hash[:md5.Size], + } + + return payload, nil +} + +func (c *RemoteClient) Put(data []byte) error { + if err := c.ensureContainerExists(); err != nil { + return err + } + + log.Printf("[DEBUG] Putting object %s in container %s", TFSTATE_NAME, c.container) + reader := bytes.NewReader(data) + createOpts := objects.CreateOpts{ + Content: reader, + } + + if c.expireSecs != 0 { + log.Printf("[DEBUG] ExpireSecs = %d", c.expireSecs) + createOpts.DeleteAfter = c.expireSecs + } + + result := objects.Create(c.client, c.container, TFSTATE_NAME, createOpts) + + return result.Err +} + +func (c *RemoteClient) Delete() error { + log.Printf("[DEBUG] Deleting object %s in container %s", TFSTATE_NAME, c.container) + result := objects.Delete(c.client, c.container, TFSTATE_NAME, nil) + return result.Err +} + +func (c *RemoteClient) ensureContainerExists() error { + containerOpts := &containers.CreateOpts{} + + if c.archive { + log.Printf("[DEBUG] Creating archive container %s", c.archiveContainer) + result := containers.Create(c.client, c.archiveContainer, nil) + if result.Err != nil { + log.Printf("[DEBUG] Error creating archive container %s: %s", c.archiveContainer, result.Err) + return result.Err + } + + log.Printf("[DEBUG] Enabling Versioning on container %s", c.container) + containerOpts.VersionsLocation = c.archiveContainer + } + + log.Printf("[DEBUG] Creating container %s", c.container) + result := containers.Create(c.client, c.container, containerOpts) + if result.Err != nil { + return result.Err + } + + return nil +} + +func multiEnv(ks []string) string { + for _, k := range ks { + if v := os.Getenv(k); v != "" { + return v + } + } + return "" +} diff --git a/backend/remote-state/swift/client_test.go b/backend/remote-state/swift/client_test.go new file mode 100644 index 000000000..a550af08a --- /dev/null +++ b/backend/remote-state/swift/client_test.go @@ -0,0 +1,33 @@ +package swift + +import ( + "fmt" + "testing" + "time" + + "github.com/hashicorp/terraform/backend" + "github.com/hashicorp/terraform/state/remote" +) + +func TestRemoteClient_impl(t *testing.T) { + var _ remote.Client = new(RemoteClient) +} + +func TestRemoteClient(t *testing.T) { + testACC(t) + + container := fmt.Sprintf("terraform-state-swift-test-%x", time.Now().Unix()) + + b := backend.TestBackendConfig(t, New(), map[string]interface{}{ + "container": container, + }).(*Backend) + + state, err := b.State(backend.DefaultStateName) + if err != nil { + t.Fatal(err) + } + + defer deleteSwiftContainer(t, b.client, container) + + remote.TestClient(t, state.(*remote.State).Client) +} diff --git a/state/remote/remote.go b/state/remote/remote.go index b99703201..03506ae1b 100644 --- a/state/remote/remote.go +++ b/state/remote/remote.go @@ -51,6 +51,5 @@ var BuiltinClients = map[string]Factory{ "gcs": gcsFactory, "http": httpFactory, "local": fileFactory, - "swift": swiftFactory, "manta": mantaFactory, } diff --git a/state/remote/swift_test.go b/state/remote/swift_test.go deleted file mode 100644 index c27de8e73..000000000 --- a/state/remote/swift_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package remote - -import ( - "net/http" - "os" - "testing" -) - -func TestSwiftClient_impl(t *testing.T) { - var _ Client = new(SwiftClient) -} - -func TestSwiftClient(t *testing.T) { - os_auth_url := os.Getenv("OS_AUTH_URL") - if os_auth_url == "" { - t.Skipf("skipping, OS_AUTH_URL and friends must be set") - } - - if _, err := http.Get(os_auth_url); err != nil { - t.Skipf("skipping, unable to reach %s: %s", os_auth_url, err) - } - - client, err := swiftFactory(map[string]string{ - "path": "swift_test", - }) - if err != nil { - t.Fatalf("bad: %s", err) - } - - testClient(t, client) -}