From c659a8305ea3ddad8814f5e69e3fdeb10a99c808 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Wed, 3 Dec 2014 20:05:29 -0800 Subject: [PATCH] remote: Moving to flexible factory + interface model --- remote/atlas.go | 223 ++++++++++++++++++++++++++++++++++++++++++ remote/atlas_test.go | 14 +++ remote/client.go | 186 ++++------------------------------- remote/consul.go | 77 +++++++++++++++ remote/consul_test.go | 14 +++ remote/http.go | 157 +++++++++++++++++++++++++++++ remote/http_test.go | 14 +++ remote/remote.go | 29 +++--- 8 files changed, 534 insertions(+), 180 deletions(-) create mode 100644 remote/atlas.go create mode 100644 remote/atlas_test.go create mode 100644 remote/consul.go create mode 100644 remote/consul_test.go create mode 100644 remote/http.go create mode 100644 remote/http_test.go diff --git a/remote/atlas.go b/remote/atlas.go new file mode 100644 index 000000000..c600942d5 --- /dev/null +++ b/remote/atlas.go @@ -0,0 +1,223 @@ +package remote + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" +) + +const ( + // defaultAtlasServer is used when no address is given + defaultAtlasServer = "https://atlas.hashicorp.com/" +) + +// AtlasRemoteClient implements the RemoteClient interface +// for an Atlas compatible server. +type AtlasRemoteClient struct { + server string + serverURL *url.URL + user string + name string + accessToken string +} + +func NewAtlasRemoteClient(conf map[string]string) (*AtlasRemoteClient, error) { + client := &AtlasRemoteClient{} + if err := client.validateConfig(conf); err != nil { + return nil, err + } + return nil, nil +} + +func (c *AtlasRemoteClient) validateConfig(conf map[string]string) error { + server, ok := conf["server"] + if !ok || server == "" { + server = defaultAtlasServer + } + url, err := url.Parse(server) + if err != nil { + return err + } + c.server = server + c.serverURL = url + + token, ok := conf["access_token"] + if !ok || token == "" { + return fmt.Errorf("missing 'access_token' configuration") + } + c.accessToken = token + + name, ok := conf["access_token"] + if !ok || name == "" { + return fmt.Errorf("missing 'name' configuration") + } + + parts := strings.Split(name, "/") + if len(parts) != 2 { + return fmt.Errorf("malformed slug '%s'", name) + } + c.user = parts[0] + c.name = parts[1] + return nil +} + +func (c *AtlasRemoteClient) GetState() (*RemoteStatePayload, error) { + // Make the HTTP request + req, err := http.NewRequest("GET", c.url("show").String(), nil) + if err != nil { + return nil, fmt.Errorf("Failed to make HTTP request: %v", err) + } + + // Request the url + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Handle the common status codes + switch resp.StatusCode { + case http.StatusOK: + // Handled after + case http.StatusNoContent: + return nil, nil + case http.StatusNotFound: + return nil, nil + case http.StatusUnauthorized: + return nil, ErrRequireAuth + case http.StatusForbidden: + return nil, ErrInvalidAuth + case http.StatusInternalServerError: + return nil, ErrRemoteInternal + default: + return nil, fmt.Errorf("Unexpected HTTP response code %d", resp.StatusCode) + } + + // Read in the body + buf := bytes.NewBuffer(nil) + if _, err := io.Copy(buf, resp.Body); err != nil { + return nil, fmt.Errorf("Failed to read remote state: %v", err) + } + + // Create the payload + payload := &RemoteStatePayload{ + State: buf.Bytes(), + } + + // Check for the MD5 + if raw := resp.Header.Get("Content-MD5"); raw != "" { + md5, err := base64.StdEncoding.DecodeString(raw) + if err != nil { + return nil, fmt.Errorf("Failed to decode Content-MD5 '%s': %v", raw, err) + } + payload.MD5 = md5 + + } else { + // Generate the MD5 + hash := md5.Sum(payload.State) + payload.MD5 = hash[:md5.Size] + } + + return payload, nil +} + +func (c *AtlasRemoteClient) PutState(state []byte, force bool) error { + // Get the target URL + base := c.url("update") + + // Generate the MD5 + hash := md5.Sum(state) + b64 := base64.StdEncoding.EncodeToString(hash[:md5.Size]) + + // Set the force query parameter if needed + if force { + values := base.Query() + values.Set("force", "true") + base.RawQuery = values.Encode() + } + + // Make the HTTP client and request + req, err := http.NewRequest("PUT", base.String(), bytes.NewReader(state)) + if err != nil { + return fmt.Errorf("Failed to make HTTP request: %v", err) + } + + // Prepare the request + req.Header.Set("Content-MD5", b64) + req.Header.Set("Content-Type", "application/json") + req.ContentLength = int64(len(state)) + + // Make the request + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("Failed to upload state: %v", err) + } + defer resp.Body.Close() + + // Handle the error codes + switch resp.StatusCode { + case http.StatusOK: + return nil + case http.StatusConflict: + return ErrConflict + case http.StatusPreconditionFailed: + return ErrServerNewer + case http.StatusUnauthorized: + return ErrRequireAuth + case http.StatusForbidden: + return ErrInvalidAuth + case http.StatusInternalServerError: + return ErrRemoteInternal + default: + return fmt.Errorf("Unexpected HTTP response code %d", resp.StatusCode) + } +} + +func (c *AtlasRemoteClient) DeleteState() error { + // Make the HTTP request + req, err := http.NewRequest("DELETE", c.url("destroy").String(), nil) + if err != nil { + return fmt.Errorf("Failed to make HTTP request: %v", err) + } + + // Make the request + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("Failed to delete state: %v", err) + } + defer resp.Body.Close() + + // Handle the error codes + switch resp.StatusCode { + case http.StatusOK: + return nil + case http.StatusNoContent: + return nil + case http.StatusNotFound: + return nil + case http.StatusUnauthorized: + return ErrRequireAuth + case http.StatusForbidden: + return ErrInvalidAuth + case http.StatusInternalServerError: + return ErrRemoteInternal + default: + return fmt.Errorf("Unexpected HTTP response code %d", resp.StatusCode) + } + return nil +} + +func (c *AtlasRemoteClient) url(route string) *url.URL { + return &url.URL{ + Scheme: c.serverURL.Scheme, + Host: c.serverURL.Host, + Path: path.Join("api/v1/state", c.user, c.name, route), + RawQuery: fmt.Sprintf("access_token=%s", c.accessToken), + } +} diff --git a/remote/atlas_test.go b/remote/atlas_test.go new file mode 100644 index 000000000..be8e02fe2 --- /dev/null +++ b/remote/atlas_test.go @@ -0,0 +1,14 @@ +package remote + +import "testing" + +func TestAtlasRemote_Interface(t *testing.T) { + var client interface{} = &AtlasRemoteClient{} + if _, ok := client.(RemoteClient); !ok { + t.Fatalf("does not implement interface") + } +} + +func TestAtlasRemote(t *testing.T) { + // TODO +} diff --git a/remote/client.go b/remote/client.go index 812c24122..bf9986500 100644 --- a/remote/client.go +++ b/remote/client.go @@ -1,17 +1,8 @@ package remote import ( - "bytes" - "crypto/md5" - "encoding/base64" - "encoding/json" "fmt" - "io" - "net/http" - "net/url" - "path" - - "github.com/hashicorp/terraform/terraform" + "strings" ) var ( @@ -36,6 +27,12 @@ var ( ErrRemoteInternal = fmt.Errorf("Remote server reporting internal error") ) +type RemoteClient interface { + GetState() (*RemoteStatePayload, error) + PutState(state []byte, force bool) error + DeleteState() error +} + // RemoteStatePayload is used to return the remote state // along with associated meta data when we do a remote fetch. type RemoteStatePayload struct { @@ -43,163 +40,18 @@ type RemoteStatePayload struct { State []byte } -// remoteStateClient is used to interact with a remote state store -// using the API -type remoteStateClient struct { - conf *terraform.RemoteState -} - -// URL is used to return an appropriate URL to hit for the -// given server and remote name -func (r *remoteStateClient) URL() (*url.URL, error) { - // Get the base URL configuration - base, err := url.Parse(r.conf.Server) - if err != nil { - return nil, fmt.Errorf("Failed to parse remote server '%s': %v", r.conf.Server, err) - } - - // Compute the full path by just appending the name - base.Path = path.Join(base.Path, r.conf.Name) - - // Add the request token if any - if r.conf.AuthToken != "" { - values := base.Query() - values.Set("access_token", r.conf.AuthToken) - base.RawQuery = values.Encode() - } - return base, nil -} - -// GetState is used to read the remote state -func (r *remoteStateClient) GetState() (*RemoteStatePayload, error) { - // Get the target URL - base, err := r.URL() - if err != nil { - return nil, err - } - - // Request the url - resp, err := http.Get(base.String()) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - // Handle the common status codes - switch resp.StatusCode { - case http.StatusOK: - // Handled after - case http.StatusNoContent: - return nil, nil - case http.StatusNotFound: - return nil, nil - case http.StatusUnauthorized: - return nil, ErrRequireAuth - case http.StatusForbidden: - return nil, ErrInvalidAuth - case http.StatusInternalServerError: - return nil, ErrRemoteInternal +// NewClientByType is used to construct a RemoteClient +// based on the configured type. +func NewClientByType(ctype string, conf map[string]string) (RemoteClient, error) { + ctype = strings.ToLower(ctype) + switch ctype { + case "atlas": + return NewAtlasRemoteClient(conf) + case "consul": + return NewConsulRemoteClient(conf) + case "http": + return NewHTTPRemoteClient(conf) default: - return nil, fmt.Errorf("Unexpected HTTP response code %d", resp.StatusCode) - } - - // Read in the body - buf := bytes.NewBuffer(nil) - if _, err := io.Copy(buf, resp.Body); err != nil { - return nil, fmt.Errorf("Failed to read remote state: %v", err) - } - - // Create the payload - payload := &RemoteStatePayload{ - State: buf.Bytes(), - } - - // Check if this is Consul - if raw := resp.Header.Get("X-Consul-Index"); raw != "" { - // Check if we used the ?raw query param, otherwise decode - if _, ok := base.Query()["raw"]; !ok { - type kv struct { - Value []byte - } - var values []*kv - if err := json.Unmarshal(buf.Bytes(), &values); err != nil { - return nil, fmt.Errorf("Failed to decode Consul response: %v", err) - } - - // Setup the reader to pull the value from Consul - payload.State = values[0].Value - } - } - - // Check for the MD5 - if raw := resp.Header.Get("Content-MD5"); raw != "" { - md5, err := base64.StdEncoding.DecodeString(raw) - if err != nil { - return nil, fmt.Errorf("Failed to decode Content-MD5 '%s': %v", raw, err) - } - payload.MD5 = md5 - - } else { - // Generate the MD5 - hash := md5.Sum(payload.State) - payload.MD5 = hash[:md5.Size] - } - - return payload, nil -} - -// Put is used to update the remote state -func (r *remoteStateClient) PutState(state []byte, force bool) error { - // Get the target URL - base, err := r.URL() - if err != nil { - return err - } - - // Generate the MD5 - hash := md5.Sum(state) - b64 := base64.StdEncoding.EncodeToString(hash[:md5.Size]) - - // Set the force query parameter if needed - if force { - values := base.Query() - values.Set("force", "true") - base.RawQuery = values.Encode() - } - - // Make the HTTP client and request - client := http.Client{} - req, err := http.NewRequest("PUT", base.String(), bytes.NewReader(state)) - if err != nil { - return fmt.Errorf("Failed to make HTTP request: %v", err) - } - - // Prepare the request - req.Header.Set("Content-MD5", b64) - req.ContentLength = int64(len(state)) - - // Make the request - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("Failed to upload state: %v", err) - } - defer resp.Body.Close() - - // Handle the error codes - switch resp.StatusCode { - case http.StatusOK: - return nil - case http.StatusConflict: - return ErrConflict - case http.StatusPreconditionFailed: - return ErrServerNewer - case http.StatusUnauthorized: - return ErrRequireAuth - case http.StatusForbidden: - return ErrInvalidAuth - case http.StatusInternalServerError: - return ErrRemoteInternal - default: - return fmt.Errorf("Unexpected HTTP response code %d", resp.StatusCode) + return nil, fmt.Errorf("Unknown remote client type '%s'", ctype) } } diff --git a/remote/consul.go b/remote/consul.go new file mode 100644 index 000000000..bbd51c4a4 --- /dev/null +++ b/remote/consul.go @@ -0,0 +1,77 @@ +package remote + +import ( + "crypto/md5" + "fmt" + + "github.com/armon/consul-api" +) + +// ConsulRemoteClient implements the RemoteClient interface +// for an Consul compatible server. +type ConsulRemoteClient struct { + client *consulapi.Client + path string // KV path +} + +func NewConsulRemoteClient(conf map[string]string) (*ConsulRemoteClient, error) { + client := &ConsulRemoteClient{} + if err := client.validateConfig(conf); err != nil { + return nil, err + } + return client, nil +} + +func (c *ConsulRemoteClient) validateConfig(conf map[string]string) (err error) { + config := consulapi.DefaultConfig() + if token, ok := conf["token"]; ok && token != "" { + config.Token = token + } + if addr, ok := conf["address"]; ok && addr != "" { + config.Address = addr + } + path, ok := conf["path"] + if !ok || path == "" { + return fmt.Errorf("missing 'path' configuration") + } + c.path = path + c.client, err = consulapi.NewClient(config) + return err +} + +func (c *ConsulRemoteClient) GetState() (*RemoteStatePayload, error) { + kv := c.client.KV() + pair, _, err := kv.Get(c.path, nil) + if err != nil { + return nil, err + } + if pair == nil { + return nil, nil + } + + // Create the payload + payload := &RemoteStatePayload{ + State: pair.Value, + } + + // Generate the MD5 + hash := md5.Sum(payload.State) + payload.MD5 = hash[:md5.Size] + return payload, nil +} + +func (c *ConsulRemoteClient) PutState(state []byte, force bool) error { + pair := &consulapi.KVPair{ + Key: c.path, + Value: state, + } + kv := c.client.KV() + _, err := kv.Put(pair, nil) + return err +} + +func (c *ConsulRemoteClient) DeleteState() error { + kv := c.client.KV() + _, err := kv.Delete(c.path, nil) + return err +} diff --git a/remote/consul_test.go b/remote/consul_test.go new file mode 100644 index 000000000..9da39b8c6 --- /dev/null +++ b/remote/consul_test.go @@ -0,0 +1,14 @@ +package remote + +import "testing" + +func TestConsulRemote_Interface(t *testing.T) { + var client interface{} = &ConsulRemoteClient{} + if _, ok := client.(RemoteClient); !ok { + t.Fatalf("does not implement interface") + } +} + +func TestConsulRemote(t *testing.T) { + // TODO +} diff --git a/remote/http.go b/remote/http.go new file mode 100644 index 000000000..1063b9d13 --- /dev/null +++ b/remote/http.go @@ -0,0 +1,157 @@ +package remote + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "fmt" + "io" + "net/http" + "net/url" +) + +// HTTPRemoteClient implements the RemoteClient interface +// for an HTTP compatible server. +type HTTPRemoteClient struct { + // url is the URL that we GET / POST / DELETE to + url *url.URL +} + +func NewHTTPRemoteClient(conf map[string]string) (*HTTPRemoteClient, error) { + client := &HTTPRemoteClient{} + if err := client.validateConfig(conf); err != nil { + return nil, err + } + return client, nil +} + +func (c *HTTPRemoteClient) validateConfig(conf map[string]string) error { + urlRaw, ok := conf["url"] + if !ok || urlRaw == "" { + return fmt.Errorf("missing 'url' configuration") + } + url, err := url.Parse(urlRaw) + if err != nil { + return fmt.Errorf("failed to parse url: %v", err) + } + c.url = url + return nil +} + +func (c *HTTPRemoteClient) GetState() (*RemoteStatePayload, error) { + // Request the url + resp, err := http.Get(c.url.String()) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Handle the common status codes + switch resp.StatusCode { + case http.StatusOK: + // Handled after + case http.StatusNoContent: + return nil, nil + case http.StatusNotFound: + return nil, nil + default: + return nil, fmt.Errorf("Unexpected HTTP response code %d", resp.StatusCode) + } + + // Read in the body + buf := bytes.NewBuffer(nil) + if _, err := io.Copy(buf, resp.Body); err != nil { + return nil, fmt.Errorf("Failed to read remote state: %v", err) + } + + // Create the payload + payload := &RemoteStatePayload{ + State: buf.Bytes(), + } + + // Check for the MD5 + if raw := resp.Header.Get("Content-MD5"); raw != "" { + md5, err := base64.StdEncoding.DecodeString(raw) + if err != nil { + return nil, fmt.Errorf("Failed to decode Content-MD5 '%s': %v", raw, err) + } + payload.MD5 = md5 + + } else { + // Generate the MD5 + hash := md5.Sum(payload.State) + payload.MD5 = hash[:md5.Size] + } + + return payload, nil +} + +func (c *HTTPRemoteClient) PutState(state []byte, force bool) error { + // Copy the target URL + base := new(url.URL) + *base = *c.url + + // Generate the MD5 + hash := md5.Sum(state) + b64 := base64.StdEncoding.EncodeToString(hash[:md5.Size]) + + // Set the force query parameter if needed + if force { + values := base.Query() + values.Set("force", "true") + base.RawQuery = values.Encode() + } + + // Make the HTTP client and request + req, err := http.NewRequest("POST", base.String(), bytes.NewReader(state)) + if err != nil { + return fmt.Errorf("Failed to make HTTP request: %v", err) + } + + // Prepare the request + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-MD5", b64) + req.ContentLength = int64(len(state)) + + // Make the request + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("Failed to upload state: %v", err) + } + defer resp.Body.Close() + + // Handle the error codes + switch resp.StatusCode { + case http.StatusOK: + return nil + default: + return fmt.Errorf("Unexpected HTTP response code %d", resp.StatusCode) + } +} + +func (c *HTTPRemoteClient) DeleteState() error { + // Make the HTTP request + req, err := http.NewRequest("DELETE", c.url.String(), nil) + if err != nil { + return fmt.Errorf("Failed to make HTTP request: %v", err) + } + + // Make the request + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("Failed to delete state: %v", err) + } + defer resp.Body.Close() + + // Handle the error codes + switch resp.StatusCode { + case http.StatusOK: + return nil + case http.StatusNoContent: + return nil + case http.StatusNotFound: + return nil + default: + return fmt.Errorf("Unexpected HTTP response code %d", resp.StatusCode) + } +} diff --git a/remote/http_test.go b/remote/http_test.go new file mode 100644 index 000000000..ce12e376b --- /dev/null +++ b/remote/http_test.go @@ -0,0 +1,14 @@ +package remote + +import "testing" + +func TestHTTPRemote_Interface(t *testing.T) { + var client interface{} = &HTTPRemoteClient{} + if _, ok := client.(RemoteClient); !ok { + t.Fatalf("does not implement interface") + } +} + +func TestHTTPRemote(t *testing.T) { + // TODO +} diff --git a/remote/remote.go b/remote/remote.go index 4edb86fb3..1651f6263 100644 --- a/remote/remote.go +++ b/remote/remote.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "io/ioutil" - "net/url" "os" "path/filepath" @@ -185,17 +184,13 @@ func ExistsFile(path string) (bool, error) { // ValidConfig does a purely logical validation of the remote config func ValidConfig(conf *terraform.RemoteState) error { - // Verify the remote server configuration is sane - if conf.Name == "" { - return fmt.Errorf("Name must be provided for remote state storage") + // Default the type to Atlas + if conf.Type == "" { + conf.Type = "atlas" } - if conf.Server != "" { - if _, err := url.Parse(conf.Server); err != nil { - return fmt.Errorf("Remote Server URL invalid: %v", err) - } - } else { - // Fill in the default server - conf.Server = DefaultServer + _, err := NewClientByType(conf.Type, conf.Config) + if err != nil { + return err } return nil } @@ -233,7 +228,11 @@ func RefreshState(conf *terraform.RemoteState) (StateChangeResult, error) { } // Read the state from the server - client := &remoteStateClient{conf: conf} + client, err := NewClientByType(conf.Type, conf.Config) + if err != nil { + return StateChangeNoop, + fmt.Errorf("Failed to create remote client: %v", err) + } payload, err := client.GetState() if err != nil { return StateChangeNoop, @@ -335,7 +334,11 @@ func PushState(conf *terraform.RemoteState, force bool) (StateChangeResult, erro } // Push the state to the server - client := &remoteStateClient{conf: conf} + client, err := NewClientByType(conf.Type, conf.Config) + if err != nil { + return StateChangeNoop, + fmt.Errorf("Failed to create remote client: %v", err) + } err = client.PutState(raw, force) // Handle the various edge cases