Merge pull request #13 from hashicorp/f-count

Count Meta-Parameter

config/config.go

@@ -31,6 +31,7 @@ type ProviderConfig struct {
 type Resource struct {
 	Name      string
 	Type      string
+	Count     int
 	RawConfig *RawConfig
 }
 

config/loader_libucl.go

@@ -253,6 +253,9 @@ func loadResourcesLibucl(o *libucl.Object) ([]*Resource, error) {
 					err)
 			}
 
+			// Remove the "count" from the config, since we treat that special
+			delete(config, "count")
+
 			rawConfig, err := NewRawConfig(config)
 			if err != nil {
 				return nil, fmt.Errorf(
@@ -262,9 +265,24 @@ func loadResourcesLibucl(o *libucl.Object) ([]*Resource, error) {
 					err)
 			}
 
+			// If we have a count, then figure it out
+			var count int = 1
+			if o := r.Get("count"); o != nil {
+				err = o.Decode(&count)
+				o.Close()
+				if err != nil {
+					return nil, fmt.Errorf(
+						"Error parsing count for %s[%s]: %s",
+						t.Key(),
+						r.Key(),
+						err)
+				}
+			}
+
 			result = append(result, &Resource{
 				Name:      r.Key(),
 				Type:      t.Key(),
+				Count:     count,
 				RawConfig: rawConfig,
 			})
 		}

config/loader_test.go

@@ -145,9 +145,10 @@ func resourcesStr(rs []*Resource) string {
 	result := ""
 	for _, r := range rs {
 		result += fmt.Sprintf(
-			"%s[%s]\n",
+			"%s[%s] (x%d)\n",
 			r.Type,
-			r.Name)
+			r.Name,
+			r.Count)
 
 		ks := make([]string, 0, len(r.RawConfig.Raw))
 		for k, _ := range r.RawConfig.Raw {
@@ -229,14 +230,18 @@ do
 `
 
 const basicResourcesStr = `
-aws_security_group[firewall]
-aws_instance[web]
+aws_security_group[firewall] (x5)
+aws_instance[web] (x1)
   ami
   network_interface
   security_groups
   vars
     resource: aws_security_group.firewall.foo
     user: var.foo
+aws_instance[db] (x1)
+  security_groups
+  vars
+    resource: aws_security_group.firewall.*.id
 `
 
 const basicVariablesStr = `
@@ -251,8 +256,8 @@ aws
 `
 
 const importResourcesStr = `
-aws_security_group[db]
-aws_security_group[web]
+aws_security_group[db] (x1)
+aws_security_group[web] (x1)
 `
 
 const importVariablesStr = `

config/test-fixtures/basic.tf

@@ -13,6 +13,7 @@ provider "do" {
 }
 
 resource "aws_security_group" "firewall" {
+    count = 5
 }
 
 resource aws_instance "web" {
@@ -27,3 +28,7 @@ resource aws_instance "web" {
         description = "Main network interface"
     }
 }
+
+resource "aws_instance" "db" {
+    security_groups = "${aws_security_group.firewall.*.id}"
+}

config/variable.go

@@ -13,7 +13,7 @@ import (
 var varRegexp *regexp.Regexp
 
 func init() {
-	varRegexp = regexp.MustCompile(`(?i)(\$+)\{([-.a-z0-9_]+)\}`)
+	varRegexp = regexp.MustCompile(`(?i)(\$+)\{([*-.a-z0-9_]+)\}`)
 }
 
 // ReplaceVariables takes a configuration and a mapping of variables

config/variable_test.go

@@ -87,6 +87,22 @@ func TestVariableDetectWalker_resource(t *testing.T) {
 	}
 }
 
+func TestVariableDetectWalker_resourceMulti(t *testing.T) {
+	w := new(variableDetectWalker)
+
+	str := `foo ${ec2.foo.*.bar}`
+	if err := w.Primitive(reflect.ValueOf(str)); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if len(w.Variables) != 1 {
+		t.Fatalf("bad: %#v", w.Variables)
+	}
+	if w.Variables["ec2.foo.*.bar"].(*ResourceVariable).FullKey() != "ec2.foo.*.bar" {
+		t.Fatalf("bad: %#v", w.Variables)
+	}
+}
+
 func TestVariableDetectWalker_bad(t *testing.T) {
 	w := new(variableDetectWalker)
 

depgraph/graph.go

@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"fmt"
 	"sort"
+	"strings"
 	"sync"
 
 	"github.com/hashicorp/terraform/digraph"
@@ -42,7 +43,34 @@ type ValidateError struct {
 }
 
 func (v *ValidateError) Error() string {
-	return "The depedency graph is not valid"
+	var msgs []string
+
+	if v.MissingRoot {
+		msgs = append(msgs, "The graph has no single root")
+	}
+
+	for _, n := range v.Unreachable {
+		msgs = append(msgs, fmt.Sprintf(
+			"Unreachable node: %s", n.Name))
+	}
+
+	for _, c := range v.Cycles {
+		cycleNodes := make([]string, len(c))
+		for i, n := range c {
+			cycleNodes[i] = n.Name
+		}
+
+		msgs = append(msgs, fmt.Sprintf(
+			"Cycle: %s", strings.Join(cycleNodes, " -> ")))
+	}
+
+	for i, m := range msgs {
+		msgs[i] = fmt.Sprintf("* %s", m)
+	}
+
+	return fmt.Sprintf(
+		"The dependency graph is not valid:\n\n%s",
+		strings.Join(msgs, "\n"))
 }
 
 // ConstraintError is used to return detailed violation

terraform/context.go

@@ -3,6 +3,7 @@ package terraform
 import (
 	"fmt"
 	"log"
+	"strings"
 	"sync"
 	"sync/atomic"
 
@@ -509,6 +510,9 @@ func (c *Context) genericWalkFn(
 		vars[fmt.Sprintf("var.%s", k)] = v
 	}
 
+	// This will keep track of the counts of multi-count resources
+	counts := make(map[string]int)
+
 	// This will keep track of whether we're stopped or not
 	var stop uint32 = 0
 
@@ -523,8 +527,20 @@ func (c *Context) genericWalkFn(
 			return nil
 		}
 
+		// Calculate any aggregate interpolated variables if we have to.
+		// Aggregate variables (such as "test_instance.foo.*.id") are not
+		// pre-computed since the fanout would be expensive. We calculate
+		// them on-demand here.
+		computeAggregateVars(&l, n, counts, vars)
+
 		switch m := n.Meta.(type) {
 		case *GraphNodeResource:
+		case *GraphNodeResourceMeta:
+			// Record the count and then just ignore
+			l.Lock()
+			counts[m.ID] = m.Count
+			l.Unlock()
+			return nil
 		case *GraphNodeResourceProvider:
 			var rc *ResourceConfig
 			if m.Config != nil {
@@ -543,6 +559,8 @@ func (c *Context) genericWalkFn(
 			}
 
 			return nil
+		default:
+			panic(fmt.Sprintf("unknown graph node: %#v", n.Meta))
 		}
 
 		rn := n.Meta.(*GraphNodeResource)
@@ -603,3 +621,68 @@ func (c *Context) genericWalkFn(
 		return nil
 	}
 }
+
+func computeAggregateVars(
+	l *sync.RWMutex,
+	n *depgraph.Noun,
+	cs map[string]int,
+	vs map[string]string) {
+	var ivars map[string]config.InterpolatedVariable
+	switch m := n.Meta.(type) {
+	case *GraphNodeResource:
+		if m.Config != nil {
+			ivars = m.Config.RawConfig.Variables
+		}
+	case *GraphNodeResourceProvider:
+		if m.Config != nil {
+			ivars = m.Config.RawConfig.Variables
+		}
+	}
+	if len(ivars) == 0 {
+		return
+	}
+
+	for _, v := range ivars {
+		rv, ok := v.(*config.ResourceVariable)
+		if !ok {
+			continue
+		}
+
+		idx := strings.Index(rv.Field, ".")
+		if idx == -1 {
+			// It isn't an aggregated var
+			continue
+		}
+		if rv.Field[:idx] != "*" {
+			// It isn't an aggregated var
+			continue
+		}
+		field := rv.Field[idx+1:]
+
+		// Get the meta node so that we can determine the count
+		key := fmt.Sprintf("%s.%s", rv.Type, rv.Name)
+		l.RLock()
+		count, ok := cs[key]
+		l.RUnlock()
+		if !ok {
+			// This should never happen due to semantic checks
+			panic(fmt.Sprintf(
+				"non-existent resource variable access: %s\n\n%#v", key, rv))
+		}
+
+		var values []string
+		for i := 0; i < count; i++ {
+			key := fmt.Sprintf(
+				"%s.%s.%d.%s",
+				rv.Type,
+				rv.Name,
+				i,
+				field)
+			if v, ok := vs[key]; ok {
+				values = append(values, v)
+			}
+		}
+
+		vs[rv.FullKey()] = strings.Join(values, ",")
+	}
+}

terraform/context_test.go

@@ -641,6 +641,113 @@ func TestContextPlan_computed(t *testing.T) {
 	}
 }
 
+func TestContextPlan_count(t *testing.T) {
+	c := testConfig(t, "plan-count")
+	p := testProvider("aws")
+	p.DiffFn = testDiffFn
+	ctx := testContext(t, &ContextOpts{
+		Config: c,
+		Providers: map[string]ResourceProviderFactory{
+			"aws": testProviderFuncFixed(p),
+		},
+	})
+
+	plan, err := ctx.Plan(nil)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if len(plan.Diff.Resources) < 6 {
+		t.Fatalf("bad: %#v", plan.Diff.Resources)
+	}
+
+	actual := strings.TrimSpace(plan.String())
+	expected := strings.TrimSpace(testTerraformPlanCountStr)
+	if actual != expected {
+		t.Fatalf("bad:\n%s", actual)
+	}
+}
+
+func TestContextPlan_countDecreaseToOne(t *testing.T) {
+	c := testConfig(t, "plan-count-dec")
+	p := testProvider("aws")
+	p.DiffFn = testDiffFn
+	s := &State{
+		Resources: map[string]*ResourceState{
+			"aws_instance.foo.0": &ResourceState{
+				ID:   "bar",
+				Type: "aws_instance",
+				Attributes: map[string]string{
+					"foo":  "foo",
+					"type": "aws_instance",
+				},
+			},
+			"aws_instance.foo.1": &ResourceState{
+				ID:   "bar",
+				Type: "aws_instance",
+			},
+			"aws_instance.foo.2": &ResourceState{
+				ID:   "bar",
+				Type: "aws_instance",
+			},
+		},
+	}
+	ctx := testContext(t, &ContextOpts{
+		Config: c,
+		Providers: map[string]ResourceProviderFactory{
+			"aws": testProviderFuncFixed(p),
+		},
+		State: s,
+	})
+
+	plan, err := ctx.Plan(nil)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	actual := strings.TrimSpace(plan.String())
+	expected := strings.TrimSpace(testTerraformPlanCountDecreaseStr)
+	if actual != expected {
+		t.Fatalf("bad:\n%s", actual)
+	}
+}
+
+func TestContextPlan_countIncreaseFromOne(t *testing.T) {
+	c := testConfig(t, "plan-count-inc")
+	p := testProvider("aws")
+	p.DiffFn = testDiffFn
+	s := &State{
+		Resources: map[string]*ResourceState{
+			"aws_instance.foo": &ResourceState{
+				ID:   "bar",
+				Type: "aws_instance",
+				Attributes: map[string]string{
+					"foo":  "foo",
+					"type": "aws_instance",
+				},
+			},
+		},
+	}
+	ctx := testContext(t, &ContextOpts{
+		Config: c,
+		Providers: map[string]ResourceProviderFactory{
+			"aws": testProviderFuncFixed(p),
+		},
+		State: s,
+	})
+
+	plan, err := ctx.Plan(nil)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	actual := strings.TrimSpace(plan.String())
+	expected := strings.TrimSpace(testTerraformPlanCountIncreaseStr)
+	if actual != expected {
+		t.Fatalf("bad:\n%s", actual)
+	}
+}
+
 func TestContextPlan_destroy(t *testing.T) {
 	c := testConfig(t, "plan-destroy")
 	p := testProvider("aws")
@@ -917,10 +1024,6 @@ func testDiffFn(
 	c *ResourceConfig) (*ResourceDiff, error) {
 	var diff ResourceDiff
 	diff.Attributes = make(map[string]*ResourceAttrDiff)
-	diff.Attributes["type"] = &ResourceAttrDiff{
-		Old: "",
-		New: s.Type,
-	}
 
 	for k, v := range c.Raw {
 		if _, ok := v.(string); !ok {
@@ -982,6 +1085,27 @@ func testDiffFn(
 		}
 	}
 
+	for k, v := range diff.Attributes {
+		if v.NewComputed {
+			continue
+		}
+
+		old, ok := s.Attributes[k]
+		if !ok {
+			continue
+		}
+		if old == v.New {
+			delete(diff.Attributes, k)
+		}
+	}
+
+	if !diff.Empty() {
+		diff.Attributes["type"] = &ResourceAttrDiff{
+			Old: "",
+			New: s.Type,
+		}
+	}
+
 	return &diff, nil
 }
 

terraform/graph.go

@@ -45,8 +45,12 @@ type GraphOpts struct {
 // graph. This node is just a placemarker and has no associated functionality.
 const GraphRootNode = "root"
 
-// GraphNodeResource is a node type in the graph that represents a resource.
+// GraphNodeResource is a node type in the graph that represents a resource
+// that will be created or managed. Unlike the GraphNodeResourceMeta node,
+// this represents a _single_, _resource_ to be managed, not a set of resources
+// or a component of a resource.
 type GraphNodeResource struct {
+	Index              int
 	Type               string
 	Config             *config.Resource
 	Orphan             bool
@@ -54,6 +58,16 @@ type GraphNodeResource struct {
 	ResourceProviderID string
 }
 
+// GraphNodeResourceMeta is a node type in the graph that represents the
+// metadata for a resource. There will be one meta node for every resource
+// in the configuration.
+type GraphNodeResourceMeta struct {
+	ID    string
+	Name  string
+	Type  string
+	Count int
+}
+
 // GraphNodeResourceProvider is a node type in the graph that represents
 // the configuration for a resource provider.
 type GraphNodeResourceProvider struct {
@@ -142,28 +156,85 @@ func graphAddConfigResources(
 	// This tracks all the resource nouns
 	nouns := make(map[string]*depgraph.Noun)
 	for _, r := range c.Resources {
-		var state *ResourceState
-		if s != nil {
-			state = s.Resources[r.Id()]
-		}
-		if state == nil {
-			state = &ResourceState{
-				Type: r.Type,
+		resourceNouns := make([]*depgraph.Noun, r.Count)
+		for i := 0; i < r.Count; i++ {
+			name := r.Id()
+			index := -1
+
+			// If we have a count that is more than one, then make sure
+			// we suffix with the number of the resource that this is.
+			if r.Count > 1 {
+				name = fmt.Sprintf("%s.%d", name, i)
+				index = i
+			}
+
+			var state *ResourceState
+			if s != nil {
+				state = s.Resources[name]
+
+				if state == nil {
+					if r.Count == 1 {
+						// If the count is one, check the state for ".0"
+						// appended, which might exist if we go from
+						// count > 1 to count == 1.
+						state = s.Resources[r.Id()+".0"]
+					} else if i == 0 {
+						// If count is greater than one, check for state
+						// with just the ID, which might exist if we go
+						// from count == 1 to count > 1
+						state = s.Resources[r.Id()]
+					}
+				}
+			}
+			if state == nil {
+				state = &ResourceState{
+					Type: r.Type,
+				}
+			}
+
+			resourceNouns[i] = &depgraph.Noun{
+				Name: name,
+				Meta: &GraphNodeResource{
+					Index:  index,
+					Type:   r.Type,
+					Config: r,
+					Resource: &Resource{
+						Id:    name,
+						State: state,
+					},
+				},
 			}
 		}
 
-		noun := &depgraph.Noun{
-			Name: r.Id(),
-			Meta: &GraphNodeResource{
-				Type:   r.Type,
-				Config: r,
-				Resource: &Resource{
-					Id:    r.Id(),
-					State: state,
+		// If we have more than one, then create a meta node to track
+		// the resources.
+		if r.Count > 1 {
+			metaNoun := &depgraph.Noun{
+				Name: r.Id(),
+				Meta: &GraphNodeResourceMeta{
+					ID:    r.Id(),
+					Name:  r.Name,
+					Type:  r.Type,
+					Count: r.Count,
 				},
-			},
+			}
+
+			// Create the dependencies on this noun
+			for _, n := range resourceNouns {
+				metaNoun.Deps = append(metaNoun.Deps, &depgraph.Dependency{
+					Name:   n.Name,
+					Source: metaNoun,
+					Target: n,
+				})
+			}
+
+			// Assign it to the map so that we have it
+			nouns[metaNoun.Name] = metaNoun
+		}
+
+		for _, n := range resourceNouns {
+			nouns[n.Name] = n
 		}
-		nouns[noun.Name] = noun
 	}
 
 	// Build the list of nouns that we iterate over
@@ -339,6 +410,7 @@ func graphAddOrphans(g *depgraph.Graph, c *config.Config, s *State) {
 		noun := &depgraph.Noun{
 			Name: k,
 			Meta: &GraphNodeResource{
+				Index:  -1,
 				Type:   rs.Type,
 				Orphan: true,
 				Resource: &Resource{
@@ -357,7 +429,10 @@ func graphAddProviderConfigs(g *depgraph.Graph, c *config.Config) {
 	nounsList := make([]*depgraph.Noun, 0, 2)
 	pcNouns := make(map[string]*depgraph.Noun)
 	for _, noun := range g.Nouns {
-		resourceNode := noun.Meta.(*GraphNodeResource)
+		resourceNode, ok := noun.Meta.(*GraphNodeResource)
+		if !ok {
+			continue
+		}
 
 		// Look up the provider config for this resource
 		pcName := config.ProviderConfigName(resourceNode.Type, c.ProviderConfigs)
@@ -401,8 +476,17 @@ func graphAddProviderConfigs(g *depgraph.Graph, c *config.Config) {
 func graphAddRoot(g *depgraph.Graph) {
 	root := &depgraph.Noun{Name: GraphRootNode}
 	for _, n := range g.Nouns {
-		// The root only needs to depend on all the resources
-		if _, ok := n.Meta.(*GraphNodeResource); !ok {
+		switch m := n.Meta.(type) {
+		case *GraphNodeResource:
+			// If the resource is part of a group, we don't need to make a dep
+			if m.Index != -1 {
+				continue
+			}
+		case *GraphNodeResourceMeta:
+			// Always in the graph
+		case *GraphNodeResourceProvider:
+			// ResourceProviders don't need to be in the root deps because
+			// they're always pointed to by some resource.
 			continue
 		}
 

terraform/graph_test.go

@@ -27,6 +27,21 @@ func TestGraph_configRequired(t *testing.T) {
 	}
 }
 
+func TestGraph_count(t *testing.T) {
+	config := testConfig(t, "graph-count")
+
+	g, err := Graph(&GraphOpts{Config: config})
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	actual := strings.TrimSpace(g.String())
+	expected := strings.TrimSpace(testTerraformGraphCountStr)
+	if actual != expected {
+		t.Fatalf("bad:\n\n%s", actual)
+	}
+}
+
 func TestGraph_cycle(t *testing.T) {
 	config := testConfig(t, "graph-cycle")
 
@@ -226,6 +241,22 @@ root
   root -> openstack_floating_ip.random
 `
 
+const testTerraformGraphCountStr = `
+root: root
+aws_instance.web
+  aws_instance.web -> aws_instance.web.0
+  aws_instance.web -> aws_instance.web.1
+  aws_instance.web -> aws_instance.web.2
+aws_instance.web.0
+aws_instance.web.1
+aws_instance.web.2
+aws_load_balancer.weblb
+  aws_load_balancer.weblb -> aws_instance.web
+root
+  root -> aws_instance.web
+  root -> aws_load_balancer.weblb
+`
+
 const testTerraformGraphDiffStr = `
 root: root
 aws_instance.foo

terraform/state.go

@@ -38,6 +38,12 @@ func (s *State) Orphans(c *config.Config) []string {
 
 	for _, r := range c.Resources {
 		delete(keys, r.Id())
+
+		// If there is only one of this instance, then we alias that
+		// to the ".0" version as well so that it can count
+		if r.Count == 1 {
+			delete(keys, r.Id()+".0")
+		}
 	}
 
 	result := make([]string, 0, len(keys))

terraform/terraform_test.go

@@ -155,6 +155,75 @@ STATE:
 <no state>
 `
 
+const testTerraformPlanCountStr = `
+DIFF:
+
+UPDATE: aws_instance.bar
+  foo:  "" => "foo,foo,foo,foo,foo"
+  type: "" => "aws_instance"
+UPDATE: aws_instance.foo.0
+  foo:  "" => "foo"
+  type: "" => "aws_instance"
+UPDATE: aws_instance.foo.1
+  foo:  "" => "foo"
+  type: "" => "aws_instance"
+UPDATE: aws_instance.foo.2
+  foo:  "" => "foo"
+  type: "" => "aws_instance"
+UPDATE: aws_instance.foo.3
+  foo:  "" => "foo"
+  type: "" => "aws_instance"
+UPDATE: aws_instance.foo.4
+  foo:  "" => "foo"
+  type: "" => "aws_instance"
+
+STATE:
+
+<no state>
+`
+
+const testTerraformPlanCountDecreaseStr = `
+DIFF:
+
+UPDATE: aws_instance.bar
+  foo:  "" => "bar"
+  type: "" => "aws_instance"
+DESTROY: aws_instance.foo.1
+DESTROY: aws_instance.foo.2
+
+STATE:
+
+aws_instance.foo.0:
+  ID = bar
+  foo = foo
+  type = aws_instance
+aws_instance.foo.1:
+  ID = bar
+aws_instance.foo.2:
+  ID = bar
+`
+
+const testTerraformPlanCountIncreaseStr = `
+DIFF:
+
+UPDATE: aws_instance.bar
+  foo:  "" => "bar"
+  type: "" => "aws_instance"
+UPDATE: aws_instance.foo.1
+  foo:  "" => "foo"
+  type: "" => "aws_instance"
+UPDATE: aws_instance.foo.2
+  foo:  "" => "foo"
+  type: "" => "aws_instance"
+
+STATE:
+
+aws_instance.foo:
+  ID = bar
+  foo = foo
+  type = aws_instance
+`
+
 const testTerraformPlanDestroyStr = `
 DIFF:
 

terraform/test-fixtures/graph-count/main.tf

@@ -0,0 +1,7 @@
+resource "aws_instance" "web" {
+    count = 3
+}
+
+resource "aws_load_balancer" "weblb" {
+    members = "${aws_instance.web.*.id}"
+}

terraform/test-fixtures/plan-count-dec/main.tf

@@ -0,0 +1,7 @@
+resource "aws_instance" "foo" {
+    foo = "foo"
+}
+
+resource "aws_instance" "bar" {
+    foo = "bar"
+}

terraform/test-fixtures/plan-count-inc/main.tf

@@ -0,0 +1,8 @@
+resource "aws_instance" "foo" {
+    foo = "foo"
+    count = 3
+}
+
+resource "aws_instance" "bar" {
+    foo = "bar"
+}

terraform/test-fixtures/plan-count/main.tf

@@ -0,0 +1,8 @@
+resource "aws_instance" "foo" {
+    count = 5
+    foo = "foo"
+}
+
+resource "aws_instance" "bar" {
+    foo = "${aws_instance.foo.*.foo}"
+}