consul_key_prefix resource

This new resource is an alternative to consul_keys that manages all keys
under a given prefix, rather than arbitrary single keys across the entire
store.

The key advantage of this resource over consul_keys is that it is able to
detect and delete keys that are added outside of Terraform, whereas
consul_keys is only able to detect changes to keys it is explicitly
managing.
This commit is contained in:
Martin Atkins 2016-04-02 20:37:11 -07:00
parent f7d1e0390d
commit d706130a51
6 changed files with 489 additions and 1 deletions

View File

@ -42,6 +42,25 @@ func (c *keyClient) Get(path string) (string, error) {
return value, nil return value, nil
} }
func (c *keyClient) GetUnderPrefix(pathPrefix string) (map[string]string, error) {
log.Printf(
"[DEBUG] Listing keys under '%s' in %s",
pathPrefix, c.qOpts.Datacenter,
)
pairs, _, err := c.client.List(pathPrefix, c.qOpts)
if err != nil {
return nil, fmt.Errorf(
"Failed to list Consul keys under prefix '%s': %s", pathPrefix, err,
)
}
value := map[string]string{}
for _, pair := range pairs {
subKey := pair.Key[len(pathPrefix):]
value[subKey] = string(pair.Value)
}
return value, nil
}
func (c *keyClient) Put(path, value string) error { func (c *keyClient) Put(path, value string) error {
log.Printf( log.Printf(
"[DEBUG] Setting key '%s' to '%v' in %s", "[DEBUG] Setting key '%s' to '%v' in %s",
@ -64,3 +83,14 @@ func (c *keyClient) Delete(path string) error {
} }
return nil return nil
} }
func (c *keyClient) DeleteUnderPrefix(pathPrefix string) error {
log.Printf(
"[DEBUG] Deleting all keys under prefix '%s' in %s",
pathPrefix, c.wOpts.Datacenter,
)
if _, err := c.client.DeleteTree(pathPrefix, c.wOpts); err != nil {
return fmt.Errorf("Failed to delete Consul keys under '%s': %s", pathPrefix, err)
}
return nil
}

View File

@ -0,0 +1,221 @@
package consul
import (
"fmt"
consulapi "github.com/hashicorp/consul/api"
"github.com/hashicorp/terraform/helper/schema"
)
func resourceConsulKeyPrefix() *schema.Resource {
return &schema.Resource{
Create: resourceConsulKeyPrefixCreate,
Update: resourceConsulKeyPrefixUpdate,
Read: resourceConsulKeyPrefixRead,
Delete: resourceConsulKeyPrefixDelete,
Schema: map[string]*schema.Schema{
"datacenter": &schema.Schema{
Type: schema.TypeString,
Optional: true,
Computed: true,
ForceNew: true,
},
"token": &schema.Schema{
Type: schema.TypeString,
Optional: true,
},
"path_prefix": &schema.Schema{
Type: schema.TypeString,
Required: true,
ForceNew: true,
},
"subkeys": &schema.Schema{
Type: schema.TypeMap,
Required: true,
Elem: &schema.Schema{
Type: schema.TypeString,
},
},
},
}
}
func resourceConsulKeyPrefixCreate(d *schema.ResourceData, meta interface{}) error {
client := meta.(*consulapi.Client)
kv := client.KV()
token := d.Get("token").(string)
dc, err := getDC(d, client)
if err != nil {
return err
}
keyClient := newKeyClient(kv, dc, token)
pathPrefix := d.Get("path_prefix").(string)
subKeys := map[string]string{}
for k, vI := range d.Get("subkeys").(map[string]interface{}) {
subKeys[k] = vI.(string)
}
// To reduce the impact of mistakes, we will only "create" a prefix that
// is currently empty. This way we are less likely to accidentally
// conflict with other mechanisms managing the same prefix.
currentSubKeys, err := keyClient.GetUnderPrefix(pathPrefix)
if err != nil {
return err
}
if len(currentSubKeys) > 0 {
return fmt.Errorf(
"%d keys already exist under %s; delete them before managing this prefix with Terraform",
len(currentSubKeys), pathPrefix,
)
}
// Ideally we'd use d.Partial(true) here so we can correctly record
// a partial write, but that mechanism doesn't work for individual map
// members, so we record that the resource was created before we
// do anything and that way we can recover from errors by doing an
// Update on subsequent runs, rather than re-attempting Create with
// some keys possibly already present.
d.SetId(pathPrefix)
// Store the datacenter on this resource, which can be helpful for reference
// in case it was read from the provider
d.Set("datacenter", dc)
// Now we can just write in all the initial values, since we can expect
// that nothing should need deleting yet, as long as there isn't some
// other program racing us to write values... which we'll catch on a
// subsequent Read.
for k, v := range subKeys {
fullPath := pathPrefix + k
err := keyClient.Put(fullPath, v)
if err != nil {
return fmt.Errorf("error while writing %s: %s", fullPath, err)
}
}
return nil
}
func resourceConsulKeyPrefixUpdate(d *schema.ResourceData, meta interface{}) error {
client := meta.(*consulapi.Client)
kv := client.KV()
token := d.Get("token").(string)
dc, err := getDC(d, client)
if err != nil {
return err
}
keyClient := newKeyClient(kv, dc, token)
pathPrefix := d.Id()
if d.HasChange("subkeys") {
o, n := d.GetChange("subkeys")
if o == nil {
o = map[string]interface{}{}
}
if n == nil {
n = map[string]interface{}{}
}
om := o.(map[string]interface{})
nm := n.(map[string]interface{})
// First we'll write all of the stuff in the "new map" nm,
// and then we'll delete any keys that appear in the "old map" om
// and do not also appear in nm. This ordering means that if a subkey
// name is changed we will briefly have both the old and new names in
// Consul, as opposed to briefly having neither.
// Again, we'd ideally use d.Partial(true) here but it doesn't work
// for maps and so we'll just rely on a subsequent Read to tidy up
// after a partial write.
// Write new and changed keys
for k, vI := range nm {
v := vI.(string)
fullPath := pathPrefix + k
err := keyClient.Put(fullPath, v)
if err != nil {
return fmt.Errorf("error while writing %s: %s", fullPath, err)
}
}
// Remove deleted keys
for k, _ := range om {
if _, exists := nm[k]; exists {
continue
}
fullPath := pathPrefix + k
err := keyClient.Delete(fullPath)
if err != nil {
return fmt.Errorf("error while deleting %s: %s", fullPath, err)
}
}
}
// Store the datacenter on this resource, which can be helpful for reference
// in case it was read from the provider
d.Set("datacenter", dc)
return nil
}
func resourceConsulKeyPrefixRead(d *schema.ResourceData, meta interface{}) error {
client := meta.(*consulapi.Client)
kv := client.KV()
token := d.Get("token").(string)
dc, err := getDC(d, client)
if err != nil {
return err
}
keyClient := newKeyClient(kv, dc, token)
pathPrefix := d.Id()
subKeys, err := keyClient.GetUnderPrefix(pathPrefix)
if err != nil {
return err
}
d.Set("subkeys", subKeys)
// Store the datacenter on this resource, which can be helpful for reference
// in case it was read from the provider
d.Set("datacenter", dc)
return nil
}
func resourceConsulKeyPrefixDelete(d *schema.ResourceData, meta interface{}) error {
client := meta.(*consulapi.Client)
kv := client.KV()
token := d.Get("token").(string)
dc, err := getDC(d, client)
if err != nil {
return err
}
keyClient := newKeyClient(kv, dc, token)
pathPrefix := d.Id()
// Delete everything under our prefix, since the entire set of keys under
// the given prefix is considered to be managed exclusively by Terraform.
err = keyClient.DeleteUnderPrefix(pathPrefix)
if err != nil {
return err
}
d.SetId("")
return nil
}

View File

@ -0,0 +1,150 @@
package consul
import (
"fmt"
"testing"
consulapi "github.com/hashicorp/consul/api"
"github.com/hashicorp/terraform/helper/resource"
"github.com/hashicorp/terraform/terraform"
)
func TestAccConsulKeyPrefix_basic(t *testing.T) {
resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: resource.ComposeTestCheckFunc(
testAccCheckConsulKeyPrefixKeyAbsent("species"),
testAccCheckConsulKeyPrefixKeyAbsent("meat"),
testAccCheckConsulKeyPrefixKeyAbsent("cheese"),
testAccCheckConsulKeyPrefixKeyAbsent("bread"),
),
Steps: []resource.TestStep{
resource.TestStep{
Config: testAccConsulKeyPrefixConfig,
Check: resource.ComposeTestCheckFunc(
testAccCheckConsulKeyPrefixKeyValue("cheese", "chevre"),
testAccCheckConsulKeyPrefixKeyValue("bread", "baguette"),
testAccCheckConsulKeyPrefixKeyAbsent("species"),
testAccCheckConsulKeyPrefixKeyAbsent("meat"),
),
},
resource.TestStep{
Config: testAccConsulKeyPrefixConfig,
ExpectNonEmptyPlan: true,
Check: resource.ComposeTestCheckFunc(
// This will add a rogue key that Terraform isn't
// expecting, causing a non-empty plan that wants
// to remove it.
testAccAddConsulKeyPrefixRogue("species", "gorilla"),
),
},
resource.TestStep{
Config: testAccConsulKeyPrefixConfig_Update,
Check: resource.ComposeTestCheckFunc(
testAccCheckConsulKeyPrefixKeyValue("meat", "ham"),
testAccCheckConsulKeyPrefixKeyValue("bread", "batard"),
testAccCheckConsulKeyPrefixKeyAbsent("cheese"),
testAccCheckConsulKeyPrefixKeyAbsent("species"),
),
},
resource.TestStep{
Config: testAccConsulKeyPrefixConfig_Update,
ExpectNonEmptyPlan: true,
Check: resource.ComposeTestCheckFunc(
testAccAddConsulKeyPrefixRogue("species", "gorilla"),
),
},
},
})
}
func testAccCheckConsulKeyPrefixDestroy(s *terraform.State) error {
kv := testAccProvider.Meta().(*consulapi.Client).KV()
opts := &consulapi.QueryOptions{Datacenter: "dc1"}
pair, _, err := kv.Get("test/set", opts)
if err != nil {
return err
}
if pair != nil {
return fmt.Errorf("Key still exists: %#v", pair)
}
return nil
}
func testAccCheckConsulKeyPrefixKeyAbsent(name string) resource.TestCheckFunc {
fullName := "prefix_test/" + name
return func(s *terraform.State) error {
kv := testAccProvider.Meta().(*consulapi.Client).KV()
opts := &consulapi.QueryOptions{Datacenter: "dc1"}
pair, _, err := kv.Get(fullName, opts)
if err != nil {
return err
}
if pair != nil {
return fmt.Errorf("key '%s' exists, but shouldn't", fullName)
}
return nil
}
}
// This one is actually not a check, but rather a mutation step. It writes
// a value directly into Consul, bypassing our Terraform resource.
func testAccAddConsulKeyPrefixRogue(name, value string) resource.TestCheckFunc {
fullName := "prefix_test/" + name
return func(s *terraform.State) error {
kv := testAccProvider.Meta().(*consulapi.Client).KV()
opts := &consulapi.WriteOptions{Datacenter: "dc1"}
pair := &consulapi.KVPair{
Key: fullName,
Value: []byte(value),
}
_, err := kv.Put(pair, opts)
return err
}
}
func testAccCheckConsulKeyPrefixKeyValue(name, value string) resource.TestCheckFunc {
fullName := "prefix_test/" + name
return func(s *terraform.State) error {
kv := testAccProvider.Meta().(*consulapi.Client).KV()
opts := &consulapi.QueryOptions{Datacenter: "dc1"}
pair, _, err := kv.Get(fullName, opts)
if err != nil {
return err
}
if pair == nil {
return fmt.Errorf("key %v doesn't exist, but should", fullName)
}
if string(pair.Value) != value {
return fmt.Errorf("key %v has value %v; want %v", fullName, pair.Value, value)
}
return nil
}
}
const testAccConsulKeyPrefixConfig = `
resource "consul_key_prefix" "app" {
datacenter = "dc1"
path_prefix = "prefix_test/"
subkeys = {
cheese = "chevre"
bread = "baguette"
}
}
`
const testAccConsulKeyPrefixConfig_Update = `
resource "consul_key_prefix" "app" {
datacenter = "dc1"
path_prefix = "prefix_test/"
subkeys = {
bread = "batard"
meat = "ham"
}
}
`

View File

@ -30,6 +30,7 @@ func Provider() terraform.ResourceProvider {
ResourcesMap: map[string]*schema.Resource{ ResourcesMap: map[string]*schema.Resource{
"consul_keys": resourceConsulKeys(), "consul_keys": resourceConsulKeys(),
"consul_key_prefix": resourceConsulKeyPrefix(),
}, },
ConfigureFunc: providerConfigure, ConfigureFunc: providerConfigure,

View File

@ -0,0 +1,79 @@
---
layout: "consul"
page_title: "Consul: consul_key_prefix"
sidebar_current: "docs-consul-resource-key-prefix"
description: |-
Allows Terraform to manage a namespace of Consul keys that share a
common name prefix.
---
# consul\_key\_prefix
Allows Terraform to manage a "namespace" of Consul keys that share a common
name prefix.
Like `consul_keys`, this resource can write values into the Consul key/value
store, but *unlike* `consul_keys` this resource can detect and remove extra
keys that have been added some other way, thus ensuring that rogue data
added outside of Terraform will be removed on the next run.
This resource is thus useful in the case where Terraform is exclusively
managing a set of related keys.
To avoid accidentally clobbering matching data that existed in Consul before
a `consul_key_prefix` resource was created, creation of a key prefix instance
will fail if any matching keys are already present in the key/value store.
If any conflicting data is present, you must first delete it manually.
~> **Warning** After this resource is instantiated, Terraform takes control
over *all* keys with the given path prefix, and will remove any matching keys
that are not present in the configuration. It will also delete *all* keys under
the given prefix when a `consul_key_prefix` resource is destroyed, even if
those keys were created outside of Terraform.
## Example Usage
```
resource "consul_key_prefix" "myapp_config" {
datacenter = "nyc1"
token = "abcd"
# Prefix to add to prepend to all of the subkey names below.
path_prefix = "myapp/config/"
subkeys = {
"elb_cname" = "${aws_elb.app.dns_name}"
"s3_bucket_name" = "${aws_s3_bucket.app.bucket}"
"database/hostname" = "${aws_db_instance.app.address}"
"database/port" = "${aws_db_instance.app.port}"
"database/username" = "${aws_db_instance.app.username}"
"database/password" = "${aws_db_instance.app.password}"
"database/name" = "${aws_db_instance.app.name}"
}
}
```
## Argument Reference
The following arguments are supported:
* `datacenter` - (Optional) The datacenter to use. This overrides the
datacenter in the provider setup and the agent's default datacenter.
* `token` - (Optional) The ACL token to use. This overrides the
token that the agent provides by default.
* `path_prefix` - (Required) Specifies the common prefix shared by all keys
that will be managed by this resource instance. In most cases this will
end with a slash, to manage a "folder" of keys.
* `subkeys` - (Required) A mapping from subkey name (which will be appended
to the give `path_prefix`) to the value that should be stored at that key.
Use slashes as shown in the above example to create "sub-folders" under
the given path prefix.
## Attributes Reference
The following attributes are exported:
* `datacenter` - The datacenter the keys are being read/written to.

View File

@ -13,6 +13,13 @@ to both read keys from Consul, but also to set the value of keys
in Consul. This is a powerful way dynamically set values in templates, in Consul. This is a powerful way dynamically set values in templates,
and to expose infrastructure details to clients. and to expose infrastructure details to clients.
This resource manages individual keys, and thus it can create, update and
delete the keys explicitly given. Howver, It is not able to detect and remove
additional keys that have been added by non-Terraform means. To manage
*all* keys sharing a common prefix, and thus have Terraform remove errant keys
not present in the configuration, consider using the `consul_key_prefix`
resource instead.
## Example Usage ## Example Usage
``` ```