resilien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge #15793: Locking support in HTTP remote backend

This commit is contained in:
Martin Atkins 2017-08-22 09:43:38 -07:00 committed by GitHub
parent 4e8ccb3127 e5029de439
commit 2e0c1d07ae
3 changed files with 300 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

221
state/remote/http.go
View File

@ -5,11 +5,15 @@ import (
"crypto/md5"
"crypto/tls"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"github.com/hashicorp/terraform/state"
)
func httpFactory(conf map[string]string) (Client, error) {
@ -18,13 +22,53 @@ func httpFactory(conf map[string]string) (Client, error) {
return nil, fmt.Errorf("missing 'address' configuration")
}
url, err := url.Parse(address)
updateURL, err := url.Parse(address)
if err != nil {
return nil, fmt.Errorf("failed to parse HTTP URL: %s", err)
return nil, fmt.Errorf("failed to parse address URL: %s", err)
}
if url.Scheme != "http" && url.Scheme != "https" {
if updateURL.Scheme != "http" && updateURL.Scheme != "https" {
return nil, fmt.Errorf("address must be HTTP or HTTPS")
}
updateMethod, ok := conf["update_method"]
if !ok {
updateMethod = "POST"
}
var lockURL *url.URL
if lockAddress, ok := conf["lock_address"]; ok {
var err error
lockURL, err = url.Parse(lockAddress)
if err != nil {
return nil, fmt.Errorf("failed to parse lockAddress URL: %s", err)
}
if lockURL.Scheme != "http" && lockURL.Scheme != "https" {
return nil, fmt.Errorf("lockAddress must be HTTP or HTTPS")
}
} else {
lockURL = nil
}
lockMethod, ok := conf["lock_method"]
if !ok {
lockMethod = "LOCK"
}
var unlockURL *url.URL
if unlockAddress, ok := conf["unlock_address"]; ok {
var err error
unlockURL, err = url.Parse(unlockAddress)
if err != nil {
return nil, fmt.Errorf("failed to parse unlockAddress URL: %s", err)
}
if unlockURL.Scheme != "http" && unlockURL.Scheme != "https" {
return nil, fmt.Errorf("unlockAddress must be HTTP or HTTPS")
}
} else {
unlockURL = nil
}
unlockMethod, ok := conf["unlock_method"]
if !ok {
unlockMethod = "UNLOCK"
}
client := &http.Client{}
if skipRaw, ok := conf["skip_cert_verification"]; ok {
@ -45,39 +89,142 @@ func httpFactory(conf map[string]string) (Client, error) {
}
ret := &HTTPClient{
URL: url,
URL: updateURL,
UpdateMethod: updateMethod,
LockURL: lockURL,
LockMethod: lockMethod,
UnlockURL: unlockURL,
UnlockMethod: unlockMethod,
Username: conf["username"],
Password: conf["password"],
// accessible only for testing use
Client: client,
}
if username, ok := conf["username"]; ok && username != "" {
ret.Username = username
}
if password, ok := conf["password"]; ok && password != "" {
ret.Password = password
}
return ret, nil
}
// HTTPClient is a remote client that stores data in Consul or HTTP REST.
type HTTPClient struct {
URL *url.URL
// Update & Retrieve
URL *url.URL
UpdateMethod string
// Locking
LockURL *url.URL
LockMethod string
UnlockURL *url.URL
UnlockMethod string
// HTTP
Client *http.Client
Username string
Password string
lockID string
jsonLockInfo []byte
}
func (c *HTTPClient) Get() (*Payload, error) {
req, err := http.NewRequest("GET", c.URL.String(), nil)
if err != nil {
return nil, err
func (c *HTTPClient) httpRequest(method string, url *url.URL, data *[]byte, what string) (*http.Response, error) {
// If we have data we need a reader
var reader io.Reader = nil
if data != nil {
reader = bytes.NewReader(*data)
}
// Prepare the request
// Create the request
req, err := http.NewRequest(method, url.String(), reader)
if err != nil {
return nil, fmt.Errorf("Failed to make %s HTTP request: %s", what, err)
}
// Setup basic auth
if c.Username != "" {
req.SetBasicAuth(c.Username, c.Password)
}
// Work with data/body
if data != nil {
req.Header.Set("Content-Type", "application/json")
req.ContentLength = int64(len(*data))
// Generate the MD5
hash := md5.Sum(*data)
b64 := base64.StdEncoding.EncodeToString(hash[:])
req.Header.Set("Content-MD5", b64)
}
// Make the request
resp, err := c.Client.Do(req)
if err != nil {
return nil, fmt.Errorf("Failed to %s: %v", what, err)
}
return resp, nil
}
func (c *HTTPClient) Lock(info *state.LockInfo) (string, error) {
if c.LockURL == nil {
return "", nil
}
c.lockID = ""
jsonLockInfo := info.Marshal()
resp, err := c.httpRequest(c.LockMethod, c.LockURL, &jsonLockInfo, "lock")
if err != nil {
return "", err
}
defer resp.Body.Close()
switch resp.StatusCode {
case http.StatusOK:
c.lockID = info.ID
c.jsonLockInfo = jsonLockInfo
return info.ID, nil
case http.StatusUnauthorized:
return "", fmt.Errorf("HTTP remote state endpoint requires auth")
case http.StatusForbidden:
return "", fmt.Errorf("HTTP remote state endpoint invalid auth")
case http.StatusConflict, http.StatusLocked:
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("HTTP remote state already locked, failed to read body")
}
existing := state.LockInfo{}
err = json.Unmarshal(body, &existing)
if err != nil {
return "", fmt.Errorf("HTTP remote state already locked, failed to unmarshal body")
}
return "", fmt.Errorf("HTTP remote state already locked: ID=%s", existing.ID)
default:
return "", fmt.Errorf("Unexpected HTTP response code %d", resp.StatusCode)
}
}
func (c *HTTPClient) Unlock(id string) error {
if c.UnlockURL == nil {
return nil
}
resp, err := c.httpRequest(c.UnlockMethod, c.UnlockURL, &c.jsonLockInfo, "unlock")
if err != nil {
return err
}
defer resp.Body.Close()
switch resp.StatusCode {
case http.StatusOK:
return nil
default:
return fmt.Errorf("Unexpected HTTP response code %d", resp.StatusCode)
}
}
func (c *HTTPClient) Get() (*Payload, error) {
resp, err := c.httpRequest("GET", c.URL, nil, "get state")
if err != nil {
return nil, err
}
@ -139,9 +286,11 @@ func (c *HTTPClient) Put(data []byte) error {
// Copy the target URL
base := *c.URL
// Generate the MD5
hash := md5.Sum(data)
b64 := base64.StdEncoding.EncodeToString(hash[:])
if c.lockID != "" {
query := base.Query()
query.Set("ID", c.lockID)
base.RawQuery = query.Encode()
}
/*
// Set the force query parameter if needed
@ -152,23 +301,13 @@ func (c *HTTPClient) Put(data []byte) error {
}
*/
req, err := http.NewRequest("POST", base.String(), bytes.NewReader(data))
if err != nil {
return fmt.Errorf("Failed to make HTTP request: %s", err)
var method string = "POST"
if c.UpdateMethod != "" {
method = c.UpdateMethod
}
// Prepare the request
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-MD5", b64)
req.ContentLength = int64(len(data))
if c.Username != "" {
req.SetBasicAuth(c.Username, c.Password)
}
// Make the request
resp, err := c.Client.Do(req)
resp, err := c.httpRequest(method, &base, &data, "upload state")
if err != nil {
return fmt.Errorf("Failed to upload state: %v", err)
return err
}
defer resp.Body.Close()
@ -182,20 +321,10 @@ func (c *HTTPClient) Put(data []byte) error {
}
func (c *HTTPClient) Delete() error {
req, err := http.NewRequest("DELETE", c.URL.String(), nil)
if err != nil {
return fmt.Errorf("Failed to make HTTP request: %s", err)
}
// Prepare the request
if c.Username != "" {
req.SetBasicAuth(c.Username, c.Password)
}
// Make the request
resp, err := c.Client.Do(req)
resp, err := c.httpRequest("DELETE", c.URL, nil, "delete state")
if err != nil {
return fmt.Errorf("Failed to delete state: %s", err)
return err
}
defer resp.Body.Close()

108
state/remote/http_test.go
View File

@ -14,6 +14,7 @@ import (
func TestHTTPClient_impl(t *testing.T) {
var _ Client = new(HTTPClient)
var _ ClientLocker = new(HTTPClient)
}
func TestHTTPClient(t *testing.T) {
@ -26,25 +27,128 @@ func TestHTTPClient(t *testing.T) {
t.Fatalf("err: %s", err)
}
// Test basic get/update
client := &HTTPClient{URL: url, Client: cleanhttp.DefaultClient()}
testClient(t, client)
// Test locking and alternative UpdateMethod
a := &HTTPClient{
URL: url,
UpdateMethod: "PUT",
LockURL: url,
LockMethod: "LOCK",
UnlockURL: url,
UnlockMethod: "UNLOCK",
Client: cleanhttp.DefaultClient(),
}
b := &HTTPClient{
URL: url,
UpdateMethod: "PUT",
LockURL: url,
LockMethod: "LOCK",
UnlockURL: url,
UnlockMethod: "UNLOCK",
Client: cleanhttp.DefaultClient(),
}
TestRemoteLocks(t, a, b)
}
func assertError(t *testing.T, err error, expected string) {
if err == nil {
t.Fatalf("Expected empty config to err")
} else if err.Error() != expected {
t.Fatalf("Expected err.Error() to be \"%s\", got \"%s\"", expected, err.Error())
}
}
func TestHTTPClientFactory(t *testing.T) {
// missing address
_, err := httpFactory(map[string]string{})
assertError(t, err, "missing 'address' configuration")
// defaults
conf := map[string]string{
"address": "http://127.0.0.1:8888/foo",
}
c, err := httpFactory(conf)
client, _ := c.(*HTTPClient)
if client == nil || err != nil {
t.Fatal("Unexpected failure, address")
}
if client.URL.String() != conf["address"] {
t.Fatalf("Expected address \"%s\", got \"%s\"", conf["address"], client.URL.String())
}
if client.UpdateMethod != "POST" {
t.Fatalf("Expected update_method \"%s\", got \"%s\"", "POST", client.UpdateMethod)
}
if client.LockURL != nil || client.LockMethod != "LOCK" {
t.Fatal("Unexpected lock_address or lock_method")
}
if client.UnlockURL != nil || client.UnlockMethod != "UNLOCK" {
t.Fatal("Unexpected unlock_address or unlock_method")
}
if client.Username != "" || client.Password != "" {
t.Fatal("Unexpected username or password")
}
// custom
conf = map[string]string{
"address": "http://127.0.0.1:8888/foo",
"update_method": "BLAH",
"lock_address": "http://127.0.0.1:8888/bar",
"lock_method": "BLIP",
"unlock_address": "http://127.0.0.1:8888/baz",
"unlock_method": "BLOOP",
"username": "user",
"password": "pass",
}
c, err = httpFactory(conf)
client, _ = c.(*HTTPClient)
if client == nil || err != nil {
t.Fatal("Unexpected failure, update_method")
}
if client.UpdateMethod != "BLAH" {
t.Fatalf("Expected update_method \"%s\", got \"%s\"", "BLAH", client.UpdateMethod)
}
if client.LockURL.String() != conf["lock_address"] || client.LockMethod != "BLIP" {
t.Fatalf("Unexpected lock_address \"%s\" vs \"%s\" or lock_method \"%s\" vs \"%s\"", client.LockURL.String(),
conf["lock_address"], client.LockMethod, conf["lock_method"])
}
if client.UnlockURL.String() != conf["unlock_address"] || client.UnlockMethod != "BLOOP" {
t.Fatalf("Unexpected unlock_address \"%s\" vs \"%s\" or unlock_method \"%s\" vs \"%s\"", client.UnlockURL.String(),
conf["unlock_address"], client.UnlockMethod, conf["unlock_method"])
}
if client.Username != "user" || client.Password != "pass" {
t.Fatalf("Unexpected username \"%s\" vs \"%s\" or password \"%s\" vs \"%s\"", client.Username, conf["username"],
client.Password, conf["password"])
}
}
type testHTTPHandler struct {
Data []byte
Data []byte
Locked bool
}
func (h *testHTTPHandler) Handle(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "GET":
w.Write(h.Data)
case "POST":
case "POST", "PUT":
buf := new(bytes.Buffer)
if _, err := io.Copy(buf, r.Body); err != nil {
w.WriteHeader(500)
}
h.Data = buf.Bytes()
case "LOCK":
if h.Locked {
w.WriteHeader(423)
} else {
h.Locked = true
}
case "UNLOCK":
h.Locked = false
case "DELETE":
h.Data = nil
w.WriteHeader(200)

22
website/docs/backends/types/http.html.md
View File

@ -8,18 +8,24 @@ description: |-
# http
**Kind: Standard (with no locking)**
**Kind: Standard (with optional locking)**
Stores the state using a simple [REST](https://en.wikipedia.org/wiki/Representational_state_transfer) client.
State will be fetched via GET, updated via POST, and purged with DELETE.
State will be fetched via GET, updated via POST, and purged with DELETE. The method used for updating is configurable.
When locking support is enabled it will use LOCK and UNLOCK requests providing the lock info in the body. The endpoint should
return a 423: Locked or 409: Conflict with the holding lock info when it's already taken, 200: OK for success. Any other status
will be considered an error. The ID of the holding lock info will be added as a query parameter to state updates requests.
## Example Usage
```hcl
terraform {
backend "http" {
address = "http://myrest.api.com"
address = "http://myrest.api.com/foo"
lock_address = "http://myrest.api.com/foo"
unlock_address = "http://myrest.api.com/foo"
}
}
```
@ -40,6 +46,16 @@ data "terraform_remote_state" "foo" {
The following configuration options are supported:
* `address` - (Required) The address of the REST endpoint
* `update_method` - (Optional) HTTP method to use when updating state.
Defaults to `POST`.
* `lock_address` - (Optional) The address of the lock REST endpoint.
Defaults to disabled.
* `lock_method` - (Optional) The HTTP method to use when locking.
Defaults to `LOCK`.
* `unlock_address` - (Optional) The address of the unlock REST endpoint.
Defaults to disabled.
* `unlock_method` - (Optional) The HTTP method to use when unlocking.
Defaults to `UNLOCK`.
* `username` - (Optional) The username for HTTP basic authentication
* `password` - (Optional) The password for HTTP basic authentication
* `skip_cert_verification` - (Optional) Whether to skip TLS verification.