From 49e6ecfd7a27a06edc8975330600002de1e68c2c Mon Sep 17 00:00:00 2001 From: James Bardin Date: Mon, 6 Nov 2017 21:57:06 -0500 Subject: [PATCH] pass providers into modules via config Implement the adding of provider through the module/providers map in the configuration. The way this works is that we start walking the module tree from the top, and for any instance of a provider that can accept a configuration through the parent's module/provider map, we add a proxy node that provides the real name and a pointer to the actual parent provider node. Multiple proxies can be chained back to the original provider. When connecting resources to providers, if that provider is a proxy, we can then connect the resource directly to the proxied node. The proxies are later removed by the DisabledProviderTransformer. This should re-instate the 0.11 beta inheritance behavior, but will allow us to later store the actual concrete provider used by a resource, so that it can be re-connected if it's orphaned by removing its module configuration. --- terraform/eval_context_builtin.go | 7 +- terraform/node_provider_abstract.go | 2 +- terraform/transform_attach_config_provider.go | 62 ----- terraform/transform_config.go | 75 ------ terraform/transform_provider.go | 225 ++++++++++++++++-- terraform/transform_provider_disable.go | 12 +- terraform/transform_provider_test.go | 39 +++ 7 files changed, 259 insertions(+), 163 deletions(-) diff --git a/terraform/eval_context_builtin.go b/terraform/eval_context_builtin.go index 193421b78..1b6ee5a62 100644 --- a/terraform/eval_context_builtin.go +++ b/terraform/eval_context_builtin.go @@ -262,13 +262,8 @@ func (ctx *BuiltinEvalContext) InterpolateProvider( var cfg *config.RawConfig if pc != nil && pc.RawConfig != nil { - path := pc.Path - if len(path) == 0 { - path = ctx.Path() - } - scope := &InterpolationScope{ - Path: path, + Path: ctx.Path(), Resource: r, } diff --git a/terraform/node_provider_abstract.go b/terraform/node_provider_abstract.go index 3230558e8..2b346a44c 100644 --- a/terraform/node_provider_abstract.go +++ b/terraform/node_provider_abstract.go @@ -26,7 +26,7 @@ type NodeAbstractProvider struct { func ResolveProviderName(name string, path []string) string { name = fmt.Sprintf("provider.%s", name) - if len(path) > 1 { + if len(path) >= 1 { name = fmt.Sprintf("%s.%s", modulePrefixStr(path), name) } diff --git a/terraform/transform_attach_config_provider.go b/terraform/transform_attach_config_provider.go index 10506ea06..39cf097ae 100644 --- a/terraform/transform_attach_config_provider.go +++ b/terraform/transform_attach_config_provider.go @@ -1,10 +1,7 @@ package terraform import ( - "log" - "github.com/hashicorp/terraform/config" - "github.com/hashicorp/terraform/config/module" ) // GraphNodeAttachProvider is an interface that must be implemented by nodes @@ -19,62 +16,3 @@ type GraphNodeAttachProvider interface { // Sets the configuration AttachProvider(*config.ProviderConfig) } - -// AttachProviderConfigTransformer goes through the graph and attaches -// provider configuration structures to nodes that implement the interfaces -// above. -// -// The attached configuration structures are directly from the configuration. -// If they're going to be modified, a copy should be made. -type AttachProviderConfigTransformer struct { - Module *module.Tree // Module is the root module for the config -} - -func (t *AttachProviderConfigTransformer) Transform(g *Graph) error { - if err := t.attachProviders(g); err != nil { - return err - } - - return nil -} - -func (t *AttachProviderConfigTransformer) attachProviders(g *Graph) error { - // Go through and find GraphNodeAttachProvider - for _, v := range g.Vertices() { - // Only care about GraphNodeAttachProvider implementations - apn, ok := v.(GraphNodeAttachProvider) - if !ok { - continue - } - - // Determine what we're looking for - path := normalizeModulePath(apn.Path()) - path = path[1:] - name := apn.ProviderName() - log.Printf("[TRACE] Attach provider request: %#v %s", path, name) - - // Get the configuration. - tree := t.Module.Child(path) - if tree == nil { - continue - } - - // Go through the provider configs to find the matching config - for _, p := range tree.Config().ProviderConfigs { - // Build the name, which is "name.alias" if an alias exists - current := p.Name - if p.Alias != "" { - current += "." + p.Alias - } - - // If the configs match then attach! - if current == name { - log.Printf("[TRACE] Attaching provider config: %#v", p) - apn.AttachProvider(p) - break - } - } - } - - return nil -} diff --git a/terraform/transform_config.go b/terraform/transform_config.go index 7ec7744a7..61bce8532 100644 --- a/terraform/transform_config.go +++ b/terraform/transform_config.go @@ -133,78 +133,3 @@ func (t *ConfigTransformer) transformSingle(g *Graph, m *module.Tree) error { return nil } - -type ProviderConfigTransformer struct { - Providers []string - Concrete ConcreteProviderNodeFunc - - // Module is the module to add resources from. - Module *module.Tree -} - -func (t *ProviderConfigTransformer) Transform(g *Graph) error { - // If no module is given, we don't do anything - if t.Module == nil { - return nil - } - - // If the module isn't loaded, that is simply an error - if !t.Module.Loaded() { - return errors.New("module must be loaded for ProviderConfigTransformer") - } - - // Start the transformation process - return t.transform(g, t.Module) -} - -func (t *ProviderConfigTransformer) transform(g *Graph, m *module.Tree) error { - // If no config, do nothing - if m == nil { - return nil - } - - // Add our resources - if err := t.transformSingle(g, m); err != nil { - return err - } - - // Transform all the children. - for _, c := range m.Children() { - if err := t.transform(g, c); err != nil { - return err - } - } - - return nil -} - -func (t *ProviderConfigTransformer) transformSingle(g *Graph, m *module.Tree) error { - log.Printf("[TRACE] ProviderConfigTransformer: Starting for path: %v", m.Path()) - - // Get the configuration for this module - conf := m.Config() - - // Build the path we're at - path := m.Path() - if len(path) > 0 { - path = append([]string{RootModuleName}, path...) - } - - // Write all the resources out - for _, p := range conf.ProviderConfigs { - name := p.Name - if p.Alias != "" { - name += "." + p.Alias - } - - v := t.Concrete(&NodeAbstractProvider{ - NameValue: name, - PathValue: path, - }).(dag.Vertex) - - // Add it to the graph - g.Add(v) - } - - return nil -} diff --git a/terraform/transform_provider.go b/terraform/transform_provider.go index 38821bae3..ac06fb5af 100644 --- a/terraform/transform_provider.go +++ b/terraform/transform_provider.go @@ -1,11 +1,13 @@ package terraform import ( + "errors" "fmt" "log" "strings" "github.com/hashicorp/go-multierror" + "github.com/hashicorp/terraform/config" "github.com/hashicorp/terraform/config/module" "github.com/hashicorp/terraform/dag" ) @@ -18,10 +20,6 @@ func TransformProviders(providers []string, concrete ConcreteProviderNodeFunc, m Providers: providers, Concrete: concrete, }, - // Attach configuration to each provider instance - &AttachProviderConfigTransformer{ - Module: mod, - }, // Add any remaining missing providers &MissingProviderTransformer{ Providers: providers, @@ -107,6 +105,13 @@ func (t *ProviderTransformer) Transform(g *Graph) error { break } + // see if this in an inherited provider + if p, ok := target.(*graphNodeProxyProvider); ok { + g.Remove(p) + target = p.Target + key = p.Target.Name() + } + log.Printf("[DEBUG] resource %s using provider %s", dag.VertexName(pv), key) pv.SetProvider(key) g.Connect(dag.BasicEdge(v, target)) @@ -349,21 +354,209 @@ func (n *graphNodeCloseProvider) RemoveIfNotTargeted() bool { return true } -// graphNodeProviderConsumerDummy is a struct that never enters the real -// graph (though it could to no ill effect). It implements -// GraphNodeProviderConsumer and GraphNodeSubpath as a way to force -// certain transformations. -type graphNodeProviderConsumerDummy struct { - ProviderValue string - PathValue []string +// graphNodeProxyProvider is a GraphNodeProvider implementation that is used to +// store the name and value of a provider node for inheritance between modules. +// These nodes are only used to store the data while loading the provider +// configurations, and are removed after all the resources have been connected +// to their providers. +type graphNodeProxyProvider struct { + NameValue string + Path []string + Target GraphNodeProvider } -func (n *graphNodeProviderConsumerDummy) Path() []string { - return n.PathValue +func (n *graphNodeProxyProvider) ProviderName() string { + return n.Target.ProviderName() } -func (n *graphNodeProviderConsumerDummy) ProvidedBy() string { - return n.ProviderValue +func (n *graphNodeProxyProvider) Name() string { + return ResolveProviderName(n.NameValue, n.Path) } -func (n *graphNodeProviderConsumerDummy) SetProvider(string) {} +// ProviderConfigTransformer adds all provider nodes from the configuration and +// attaches the configs. +type ProviderConfigTransformer struct { + Providers []string + Concrete ConcreteProviderNodeFunc + + // each provider node is stored here so that the proxy nodes can look up + // their targets by name. + providers map[string]GraphNodeProvider + + // Module is the module to add resources from. + Module *module.Tree +} + +func (t *ProviderConfigTransformer) Transform(g *Graph) error { + // If no module is given, we don't do anything + if t.Module == nil { + return nil + } + + // If the module isn't loaded, that is simply an error + if !t.Module.Loaded() { + return errors.New("module must be loaded for ProviderConfigTransformer") + } + + t.providers = make(map[string]GraphNodeProvider) + + // Start the transformation process + if err := t.transform(g, t.Module); err != nil { + return nil + } + + // finally attach the configs to the new nodes + return t.attachProviderConfigs(g) +} + +func (t *ProviderConfigTransformer) transform(g *Graph, m *module.Tree) error { + // If no config, do nothing + if m == nil { + return nil + } + + // Add our resources + if err := t.transformSingle(g, m); err != nil { + return err + } + + // Transform all the children. + for _, c := range m.Children() { + if err := t.transform(g, c); err != nil { + return err + } + } + return nil +} + +func (t *ProviderConfigTransformer) transformSingle(g *Graph, m *module.Tree) error { + log.Printf("[TRACE] ProviderConfigTransformer: Starting for path: %v", m.Path()) + + // Get the configuration for this module + conf := m.Config() + + // Build the path we're at + path := m.Path() + if len(path) > 0 { + path = append([]string{RootModuleName}, path...) + } + + // add all provider configs + for _, p := range conf.ProviderConfigs { + name := p.Name + if p.Alias != "" { + name += "." + p.Alias + } + + // if this is an empty config placeholder to accept a provier from a + // parent module, add a proxy and continue. + if t.addProxyProvider(g, m, p, name) { + continue + } + + v := t.Concrete(&NodeAbstractProvider{ + NameValue: name, + PathValue: path, + }) + + // Add it to the graph + g.Add(v) + t.providers[ResolveProviderName(name, path)] = v.(GraphNodeProvider) + } + + return nil +} + +// add a ProxyProviderConfig if this was inherited from a parent module. Return +// whether the proxy was added to the graph or not. +func (t *ProviderConfigTransformer) addProxyProvider(g *Graph, m *module.Tree, pc *config.ProviderConfig, name string) bool { + path := m.Path() + + // This isn't a proxy if there's a config, or we're at the root + if len(pc.RawConfig.RawMap()) > 0 || len(path) == 0 { + return false + } + + parentPath := path[:len(path)-1] + parent := t.Module.Child(parentPath) + if parent == nil { + return false + } + + var parentCfg *config.Module + for _, mod := range parent.Config().Modules { + if mod.Name == m.Name() { + parentCfg = mod + break + } + } + + if parentCfg == nil { + panic("immaculately conceived module " + m.Name()) + } + + parentProviderName, ok := parentCfg.Providers[name] + if !ok { + // this provider isn't listed in a parent module block, so we just have + // an empty config + return false + } + + // the parent module is passing in a provider + fullParentName := ResolveProviderName(parentProviderName, parentPath) + parentProvider := t.providers[fullParentName] + + if parentProvider == nil { + panic(fmt.Sprintf("missing provider %s in module %s", parentProviderName, m.Name())) + } + + v := &graphNodeProxyProvider{ + NameValue: name, + Path: path, + Target: parentProvider, + } + + // Add it to the graph + g.Add(v) + t.providers[v.NameValue] = v + return true +} + +func (t *ProviderConfigTransformer) attachProviderConfigs(g *Graph) error { + for _, v := range g.Vertices() { + // Only care about GraphNodeAttachProvider implementations + apn, ok := v.(GraphNodeAttachProvider) + if !ok { + continue + } + + // Determine what we're looking for + path := normalizeModulePath(apn.Path())[1:] + name := apn.ProviderName() + log.Printf("[TRACE] Attach provider request: %#v %s", path, name) + + // Get the configuration. + tree := t.Module.Child(path) + if tree == nil { + continue + } + + // Go through the provider configs to find the matching config + for _, p := range tree.Config().ProviderConfigs { + // Build the name, which is "name.alias" if an alias exists + current := p.Name + if p.Alias != "" { + current += "." + p.Alias + } + + // If the configs match then attach! + if current == name { + log.Printf("[TRACE] Attaching provider config: %#v", p) + apn.AttachProvider(p) + break + } + } + } + + return nil +} diff --git a/terraform/transform_provider_disable.go b/terraform/transform_provider_disable.go index d9919f3a7..656a6385c 100644 --- a/terraform/transform_provider_disable.go +++ b/terraform/transform_provider_disable.go @@ -7,9 +7,9 @@ import ( ) // DisableProviderTransformer "disables" any providers that are not actually -// used by anything. This avoids the provider being initialized and configured. -// This both saves resources but also avoids errors since configuration -// may imply initialization which may require auth. +// used by anything, and provider proxies. This avoids the provider being +// initialized and configured. This both saves resources but also avoids +// errors since configuration may imply initialization which may require auth. type DisableProviderTransformer struct{} func (t *DisableProviderTransformer) Transform(g *Graph) error { @@ -20,6 +20,12 @@ func (t *DisableProviderTransformer) Transform(g *Graph) error { continue } + // remove the proxy nodes now that we're done with them + if pn, ok := v.(*graphNodeProxyProvider); ok { + g.Remove(pn) + continue + } + // If we have dependencies, then don't disable if g.UpEdges(v).Len() > 0 { continue diff --git a/terraform/transform_provider_test.go b/terraform/transform_provider_test.go index 42995b8a3..36c9c5e07 100644 --- a/terraform/transform_provider_test.go +++ b/terraform/transform_provider_test.go @@ -445,6 +445,39 @@ func TestPruneProviderTransformer(t *testing.T) { } } +// the child module resource is attached to the configured parent provider +func TestProviderConfigTransformer_parentProviders(t *testing.T) { + mod := testModule(t, "transform-provider-inherit") + concrete := func(a *NodeAbstractProvider) dag.Vertex { return a } + + g := Graph{Path: RootModulePath} + { + tf := &ConfigTransformer{Module: mod} + if err := tf.Transform(&g); err != nil { + t.Fatalf("err: %s", err) + } + } + { + tf := &AttachResourceConfigTransformer{Module: mod} + if err := tf.Transform(&g); err != nil { + t.Fatalf("err: %s", err) + } + } + + { + tf := TransformProviders([]string{"aws"}, concrete, mod) + if err := tf.Transform(&g); err != nil { + t.Fatalf("err: %s", err) + } + } + + actual := strings.TrimSpace(g.String()) + expected := strings.TrimSpace(testTransformModuleProviderConfigStr) + if actual != expected { + t.Fatalf("expected:\n%s\n\ngot:\n%s", expected, actual) + } +} + const testTransformProviderBasicStr = ` aws_instance.web provider.aws @@ -545,3 +578,9 @@ provider.aws (close) provider.aws var.foo ` + +const testTransformModuleProviderConfigStr = ` +module.child.aws_instance.thing + provider.aws.foo +provider.aws.foo +`