From cb0e37a870dfcb700cc407061baf4f75c1b42469 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Thu, 12 Oct 2017 13:43:04 -0400 Subject: [PATCH] implement provider inheritence during loading This implements provider inheritance during config loading, rather than during graph evaluation. At this point it's much simpler to find the desired configuration, and once all providers are declared, all the inheritance code in the graph can be removed. The inheritance is dome by simply copying the RawConfig from the parent ProviderConfig into the module. Since this happens before any evaluation, we record the original interpolation scope in the ProviderConfig so that it can be properly resolved later on. --- config/module/module.go | 4 +- .../basic-parent-providers/a/a.tf | 13 ++ .../basic-parent-providers/c/c.tf | 2 + .../basic-parent-providers/main.tf | 11 ++ .../child/grandchild/main.tf | 2 + .../child/main.tf | 9 + .../implicit-grandparent-providers/main.tf | 7 + .../implicit-parent-providers/child/main.tf | 1 + .../implicit-parent-providers/main.tf | 7 + config/module/tree.go | 138 +++++++++++++- config/module/tree_test.go | 172 ++++++++++++++++++ 11 files changed, 362 insertions(+), 4 deletions(-) create mode 100644 config/module/test-fixtures/basic-parent-providers/a/a.tf create mode 100644 config/module/test-fixtures/basic-parent-providers/c/c.tf create mode 100644 config/module/test-fixtures/basic-parent-providers/main.tf create mode 100644 config/module/test-fixtures/implicit-grandparent-providers/child/grandchild/main.tf create mode 100644 config/module/test-fixtures/implicit-grandparent-providers/child/main.tf create mode 100644 config/module/test-fixtures/implicit-grandparent-providers/main.tf create mode 100644 config/module/test-fixtures/implicit-parent-providers/child/main.tf create mode 100644 config/module/test-fixtures/implicit-parent-providers/main.tf diff --git a/config/module/module.go b/config/module/module.go index 8d74715c7..7dc8fccda 100644 --- a/config/module/module.go +++ b/config/module/module.go @@ -1,11 +1,9 @@ package module -import "github.com/hashicorp/terraform/config" - // Module represents the metadata for a single module. type Module struct { Name string Source string Version string - Providers []*config.ProviderConfig + Providers map[string]string } diff --git a/config/module/test-fixtures/basic-parent-providers/a/a.tf b/config/module/test-fixtures/basic-parent-providers/a/a.tf new file mode 100644 index 000000000..0b11ba26a --- /dev/null +++ b/config/module/test-fixtures/basic-parent-providers/a/a.tf @@ -0,0 +1,13 @@ +provider "top" {} + +provider "bottom" { + alias = "foo" + value = "from bottom" +} + +module "b" { + source = "../c" + providers = { + "bottom" = "bottom.foo" + } +} diff --git a/config/module/test-fixtures/basic-parent-providers/c/c.tf b/config/module/test-fixtures/basic-parent-providers/c/c.tf new file mode 100644 index 000000000..d1e2809ce --- /dev/null +++ b/config/module/test-fixtures/basic-parent-providers/c/c.tf @@ -0,0 +1,2 @@ +# Hello +provider "bottom" {} diff --git a/config/module/test-fixtures/basic-parent-providers/main.tf b/config/module/test-fixtures/basic-parent-providers/main.tf new file mode 100644 index 000000000..b9009b6c6 --- /dev/null +++ b/config/module/test-fixtures/basic-parent-providers/main.tf @@ -0,0 +1,11 @@ +provider "top" { + alias = "foo" + value = "from top" +} + +module "a" { + source = "./a" + providers = { + "top" = "top.foo" + } +} diff --git a/config/module/test-fixtures/implicit-grandparent-providers/child/grandchild/main.tf b/config/module/test-fixtures/implicit-grandparent-providers/child/grandchild/main.tf new file mode 100644 index 000000000..492e4d0e4 --- /dev/null +++ b/config/module/test-fixtures/implicit-grandparent-providers/child/grandchild/main.tf @@ -0,0 +1,2 @@ +resource "bar_resource" "in_grandchild" {} + diff --git a/config/module/test-fixtures/implicit-grandparent-providers/child/main.tf b/config/module/test-fixtures/implicit-grandparent-providers/child/main.tf new file mode 100644 index 000000000..1ab8b908e --- /dev/null +++ b/config/module/test-fixtures/implicit-grandparent-providers/child/main.tf @@ -0,0 +1,9 @@ +resource "foo_resource" "in_child" {} + +provider "bar" { + value = "from child" +} + +module "grandchild" { + source = "./grandchild" +} diff --git a/config/module/test-fixtures/implicit-grandparent-providers/main.tf b/config/module/test-fixtures/implicit-grandparent-providers/main.tf new file mode 100644 index 000000000..2c06ab555 --- /dev/null +++ b/config/module/test-fixtures/implicit-grandparent-providers/main.tf @@ -0,0 +1,7 @@ +provider "foo" { + value = "from root" +} + +module "child" { + source = "./child" +} diff --git a/config/module/test-fixtures/implicit-parent-providers/child/main.tf b/config/module/test-fixtures/implicit-parent-providers/child/main.tf new file mode 100644 index 000000000..260124129 --- /dev/null +++ b/config/module/test-fixtures/implicit-parent-providers/child/main.tf @@ -0,0 +1 @@ +resource "foo_instance" "bar" {} diff --git a/config/module/test-fixtures/implicit-parent-providers/main.tf b/config/module/test-fixtures/implicit-parent-providers/main.tf new file mode 100644 index 000000000..2c06ab555 --- /dev/null +++ b/config/module/test-fixtures/implicit-parent-providers/main.tf @@ -0,0 +1,7 @@ +provider "foo" { + value = "from root" +} + +module "child" { + source = "./child" +} diff --git a/config/module/tree.go b/config/module/tree.go index 444219983..f01ed180e 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -190,7 +190,7 @@ func (t *Tree) Load(s getter.Storage, mode GetMode) error { // modules. key := fmt.Sprintf("0.root.%s-%s", strings.Join(path, "."), m.Source) - log.Printf("[TRACE] module source %q", m.Source) + log.Printf("[TRACE] module source: %q", m.Source) // Split out the subdir if we have one. // Terraform keeps the entire requested tree for now, so that modules can // reference sibling modules from the same archive or repo. @@ -301,9 +301,145 @@ func (t *Tree) Load(s getter.Storage, mode GetMode) error { // Set our tree up t.children = children + // if we're the root module, we can now set the provider inheritance + if len(t.path) == 0 { + t.inheritProviderConfigs(nil) + } + return nil } +// Once the tree is loaded, we can resolve all provider config inheritance. +// +// This moves the full responsibility of inheritance to the config loader, +// simplifying locating provider configuration during graph evaluation. +// The algorithm is much simpler now too. If there is a provider block without +// a config, we look in the parent's Module block for a provider, and fetch +// that provider's configuration. If that doesn't exist, we assume a default +// empty config. Implicit providers can still inherit their config all the way +// up from the root, so we walk up the tree and copy the first matching +// provider into the module. +func (t *Tree) inheritProviderConfigs(stack []*Tree) { + stack = append(stack, t) + for _, c := range t.children { + c.inheritProviderConfigs(stack) + } + + providers := make(map[string]*config.ProviderConfig) + missingProviders := make(map[string]bool) + + for _, p := range t.config.ProviderConfigs { + providers[p.FullName()] = p + } + + for _, r := range t.config.Resources { + p := r.ProviderFullName() + if _, ok := providers[p]; !(ok || strings.Contains(p, ".")) { + missingProviders[p] = true + } + } + + // Search for implicit provider configs + // This adds an empty config is no inherited config is found, so that + // there is always a provider config present. + // This is done in the root module as well, just to set the providers. + for missing := range missingProviders { + // first create an empty provider config + pc := &config.ProviderConfig{ + Name: missing, + } + + // walk up the stack looking for matching providers + for i := len(stack) - 2; i >= 0; i-- { + pt := stack[i] + var parentProvider *config.ProviderConfig + for _, p := range pt.config.ProviderConfigs { + if p.FullName() == missing { + parentProvider = p + break + } + } + + if parentProvider == nil { + continue + } + + pc.Scope = pt.Path() + pc.Scope = append([]string{RootName}, pt.path...) + pc.RawConfig = parentProvider.RawConfig + log.Printf("[TRACE] provider %q inheriting config from %q", + strings.Join(append(t.Path(), pc.FullName()), "."), + strings.Join(append(pt.Path(), parentProvider.FullName()), "."), + ) + break + } + + // always set a provider config + if pc.RawConfig == nil { + pc.RawConfig, _ = config.NewRawConfig(map[string]interface{}{}) + } + + t.config.ProviderConfigs = append(t.config.ProviderConfigs, pc) + } + + // After allowing the empty implicit configs to be created in root, there's nothing left to inherit + if len(stack) == 1 { + return + } + + // get our parent's module config block + parent := stack[len(stack)-2] + var parentModule *config.Module + for _, m := range parent.config.Modules { + if m.Name == t.name { + parentModule = m + break + } + } + + if parentModule == nil { + panic("can't be a module without a parent module config") + } + + // now look for providers that need a config + for p, pc := range providers { + if len(pc.RawConfig.RawMap()) > 0 { + log.Printf("[TRACE] provider %q has a config, continuing", p) + continue + } + + // this provider has no config yet, check for one being passed in + parentProviderName, ok := parentModule.Providers[p] + if !ok { + continue + } + + var parentProvider *config.ProviderConfig + // there's a config for us in the parent module + for _, pp := range parent.config.ProviderConfigs { + if pp.FullName() == parentProviderName { + parentProvider = pp + break + } + } + + if parentProvider == nil { + // no config found, assume defaults + continue + } + + // Copy it in, but set an interpolation Scope. + // An interpolation Scope always need to have "root" + pc.Scope = append([]string{RootName}, parent.path...) + pc.RawConfig = parentProvider.RawConfig + log.Printf("[TRACE] provider %q inheriting config from %q", + strings.Join(append(t.Path(), pc.FullName()), "."), + strings.Join(append(parent.Path(), parentProvider.FullName()), "."), + ) + } + +} + func subdirRecordsPath(dir string) string { const filename = "module-subdir.json" // Get the parent directory. diff --git a/config/module/tree_test.go b/config/module/tree_test.go index a197a175b..e71842f3c 100644 --- a/config/module/tree_test.go +++ b/config/module/tree_test.go @@ -3,6 +3,7 @@ package module import ( "fmt" "io/ioutil" + "log" "os" "path/filepath" "reflect" @@ -518,6 +519,177 @@ func TestTreeValidate_unknownModule(t *testing.T) { } } +func TestTreeProviders_basic(t *testing.T) { + storage := testStorage(t) + tree := NewTree("", testConfig(t, "basic-parent-providers")) + + if err := tree.Load(storage, GetModeGet); err != nil { + t.Fatalf("err: %s", err) + } + + var a, b *Tree + for _, child := range tree.Children() { + if child.Name() == "a" { + a = child + } + } + + rootProviders := tree.config.ProviderConfigsByFullName() + topRaw := rootProviders["top.foo"] + + if a == nil { + t.Fatal("could not find module 'a'") + } + + for _, child := range a.Children() { + if child.Name() == "b" { + b = child + } + } + + if b == nil { + t.Fatal("could not find module 'b'") + } + + aProviders := a.config.ProviderConfigsByFullName() + bottomRaw := aProviders["bottom.foo"] + bProviders := b.config.ProviderConfigsByFullName() + bBottom := bProviders["bottom"] + + // compare the configs + // top.foo should have been copied to a.top + aTop := aProviders["top"] + if !reflect.DeepEqual(aTop.RawConfig.RawMap(), topRaw.RawConfig.RawMap()) { + log.Fatalf("expected config %#v, got %#v", + topRaw.RawConfig.RawMap(), + aTop.RawConfig.RawMap(), + ) + } + + if !reflect.DeepEqual(aTop.Scope, []string{RootName}) { + log.Fatalf(`expected scope for "top": {"root"}, got %#v`, aTop.Scope) + } + + if !reflect.DeepEqual(bBottom.RawConfig.RawMap(), bottomRaw.RawConfig.RawMap()) { + t.Fatalf("expected config %#v, got %#v", + bottomRaw.RawConfig.RawMap(), + bBottom.RawConfig.RawMap(), + ) + } + if !reflect.DeepEqual(bBottom.Scope, []string{RootName, "a"}) { + t.Fatalf(`expected scope for "bottom": {"root", "a"}, got %#v`, bBottom.Scope) + } +} + +func TestTreeProviders_implicit(t *testing.T) { + storage := testStorage(t) + tree := NewTree("", testConfig(t, "implicit-parent-providers")) + + if err := tree.Load(storage, GetModeGet); err != nil { + t.Fatalf("err: %s", err) + } + + var child *Tree + for _, c := range tree.Children() { + if c.Name() == "child" { + child = c + } + } + + if child == nil { + t.Fatal("could not find module 'child'") + } + + // child should have inherited foo + providers := child.config.ProviderConfigsByFullName() + foo := providers["foo"] + + if foo == nil { + t.Fatal("could not find provider 'foo' in child module") + } + + if !reflect.DeepEqual([]string{RootName}, foo.Scope) { + t.Fatalf(`expected foo scope of {"root"}, got %#v`, foo.Scope) + } + + expected := map[string]interface{}{ + "value": "from root", + } + + if !reflect.DeepEqual(expected, foo.RawConfig.RawMap()) { + t.Fatalf(`expected "foo" config %#v, got: %#v`, expected, foo.RawConfig.RawMap()) + } +} + +func TestTreeProviders_implicitMultiLevel(t *testing.T) { + storage := testStorage(t) + tree := NewTree("", testConfig(t, "implicit-grandparent-providers")) + + if err := tree.Load(storage, GetModeGet); err != nil { + t.Fatalf("err: %s", err) + } + + var child, grandchild *Tree + for _, c := range tree.Children() { + if c.Name() == "child" { + child = c + } + } + + if child == nil { + t.Fatal("could not find module 'child'") + } + + for _, c := range child.Children() { + if c.Name() == "grandchild" { + grandchild = c + } + } + if grandchild == nil { + t.Fatal("could not find module 'grandchild'") + } + + // child should have inherited foo + providers := child.config.ProviderConfigsByFullName() + foo := providers["foo"] + + if foo == nil { + t.Fatal("could not find provider 'foo' in child module") + } + + if !reflect.DeepEqual([]string{RootName}, foo.Scope) { + t.Fatalf(`expected foo scope of {"root"}, got %#v`, foo.Scope) + } + + expected := map[string]interface{}{ + "value": "from root", + } + + if !reflect.DeepEqual(expected, foo.RawConfig.RawMap()) { + t.Fatalf(`expected "foo" config %#v, got: %#v`, expected, foo.RawConfig.RawMap()) + } + + // grandchild should have inherited bar + providers = grandchild.config.ProviderConfigsByFullName() + bar := providers["bar"] + + if bar == nil { + t.Fatal("could not find provider 'bar' in grandchild module") + } + + if !reflect.DeepEqual([]string{RootName, "child"}, bar.Scope) { + t.Fatalf(`expected bar scope of {"root", "child"}, got %#v`, bar.Scope) + } + + expected = map[string]interface{}{ + "value": "from child", + } + + if !reflect.DeepEqual(expected, bar.RawConfig.RawMap()) { + t.Fatalf(`expected "bar" config %#v, got: %#v`, expected, bar.RawConfig.RawMap()) + } +} + const treeLoadStr = ` root foo (path: foo)