From d4285dd27fc63aaacf1791f0672b9c2da82346bf Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 2 May 2018 20:16:22 -0700 Subject: [PATCH] core: Attach resource and provider config schemas during graph build This is a little awkward since we need to instantiate the providers much earlier than before. To avoid a lot of reshuffling here we just spin each one up and then immediately shut it down again, letting our existing init functionality during the graph walk still do the main initialization. --- terraform/context.go | 38 +++-- terraform/graph_builder_apply.go | 16 ++- terraform/graph_builder_plan.go | 25 ++-- terraform/graph_builder_refresh.go | 11 +- terraform/graph_walk_context.go | 14 +- terraform/node_data_refresh.go | 1 + terraform/node_provider_abstract.go | 24 ++-- terraform/node_resource_abstract.go | 17 ++- terraform/node_resource_plan.go | 1 + terraform/node_resource_refresh.go | 1 + terraform/node_resource_validate.go | 1 + terraform/transform_attach_config_provider.go | 3 - terraform/transform_attach_schema.go | 132 ++++++++++++++++++ terraform/transform_provisioner.go | 2 +- terraform/transform_resource_count.go | 3 + 15 files changed, 226 insertions(+), 63 deletions(-) create mode 100644 terraform/transform_attach_schema.go diff --git a/terraform/context.go b/terraform/context.go index d87c0d8cb..488588664 100644 --- a/terraform/context.go +++ b/terraform/context.go @@ -249,14 +249,13 @@ func (c *Context) Graph(typ GraphType, opts *ContextGraphOpts) (*Graph, tfdiags. switch typ { case GraphTypeApply: return (&ApplyGraphBuilder{ - Config: c.config, - Diff: c.diff, - State: c.state, - Providers: c.components.ResourceProviders(), - Provisioners: c.components.ResourceProvisioners(), - Targets: c.targets, - Destroy: c.destroy, - Validate: opts.Validate, + Config: c.config, + Diff: c.diff, + State: c.state, + Components: c.components, + Targets: c.targets, + Destroy: c.destroy, + Validate: opts.Validate, }).Build(addrs.RootModuleInstance) case GraphTypeInput: @@ -268,11 +267,11 @@ func (c *Context) Graph(typ GraphType, opts *ContextGraphOpts) (*Graph, tfdiags. case GraphTypePlan: // Create the plan graph builder p := &PlanGraphBuilder{ - Config: c.config, - State: c.state, - Providers: c.components.ResourceProviders(), - Targets: c.targets, - Validate: opts.Validate, + Config: c.config, + State: c.state, + Components: c.components, + Targets: c.targets, + Validate: opts.Validate, } // Some special cases for other graph types shared with plan currently @@ -281,9 +280,6 @@ func (c *Context) Graph(typ GraphType, opts *ContextGraphOpts) (*Graph, tfdiags. case GraphTypeInput: b = InputGraphBuilder(p) case GraphTypeValidate: - // We need to set the provisioners so those can be validated - p.Provisioners = c.components.ResourceProvisioners() - b = ValidateGraphBuilder(p) } @@ -299,11 +295,11 @@ func (c *Context) Graph(typ GraphType, opts *ContextGraphOpts) (*Graph, tfdiags. case GraphTypeRefresh: return (&RefreshGraphBuilder{ - Config: c.config, - State: c.state, - Providers: c.components.ResourceProviders(), - Targets: c.targets, - Validate: opts.Validate, + Config: c.config, + State: c.state, + Components: c.components, + Targets: c.targets, + Validate: opts.Validate, }).Build(addrs.RootModuleInstance) default: diff --git a/terraform/graph_builder_apply.go b/terraform/graph_builder_apply.go index b572f8a47..6eca326e7 100644 --- a/terraform/graph_builder_apply.go +++ b/terraform/graph_builder_apply.go @@ -24,11 +24,9 @@ type ApplyGraphBuilder struct { // State is the current state State *State - // Providers is the list of providers supported. - Providers []string - - // Provisioners is the list of provisioners supported. - Provisioners []string + // Components is a factory for the plug-in components (providers and + // provisioners) available for use. + Components contextComponentFactory // Targets are resources to target. This is only required to make sure // unnecessary outputs aren't included in the apply graph. The plan @@ -87,7 +85,7 @@ func (b *ApplyGraphBuilder) Steps() []GraphTransformer { &AttachStateTransformer{State: b.State}, // add providers - TransformProviders(b.Providers, concreteProvider, b.Config), + TransformProviders(b.Components.ResourceProviders(), concreteProvider, b.Config), // Destruction ordering &DestroyEdgeTransformer{Config: b.Config, State: b.State}, @@ -97,7 +95,7 @@ func (b *ApplyGraphBuilder) Steps() []GraphTransformer { ), // Provisioner-related transformations - &MissingProvisionerTransformer{Provisioners: b.Provisioners}, + &MissingProvisionerTransformer{Provisioners: b.Components.ResourceProvisioners()}, &ProvisionerTransformer{}, // Add root variables @@ -115,6 +113,10 @@ func (b *ApplyGraphBuilder) Steps() []GraphTransformer { // Remove modules no longer present in the config &RemovedModuleTransformer{Config: b.Config, State: b.State}, + // Must be before ReferenceTransformer, since schema is required to + // extract references from config. + &AttachSchemaTransformer{Components: b.Components}, + // Connect references so ordering is correct &ReferenceTransformer{}, diff --git a/terraform/graph_builder_plan.go b/terraform/graph_builder_plan.go index 19dee0e2e..d2bf79c32 100644 --- a/terraform/graph_builder_plan.go +++ b/terraform/graph_builder_plan.go @@ -29,11 +29,9 @@ type PlanGraphBuilder struct { // State is the current state State *State - // Providers is the list of providers supported. - Providers []string - - // Provisioners is the list of provisioners supported. - Provisioners []string + // Components is a factory for the plug-in components (providers and + // provisioners) available for use. + Components contextComponentFactory // Targets are resources to target Targets []addrs.Targetable @@ -103,16 +101,11 @@ func (b *PlanGraphBuilder) Steps() []GraphTransformer { // Add root variables &RootVariableTransformer{Config: b.Config}, - TransformProviders(b.Providers, b.ConcreteProvider, b.Config), + TransformProviders(b.Components.ResourceProviders(), b.ConcreteProvider, b.Config), - // Provisioner-related transformations. Only add these if requested. - GraphTransformIf( - func() bool { return b.Provisioners != nil }, - GraphTransformMulti( - &MissingProvisionerTransformer{Provisioners: b.Provisioners}, - &ProvisionerTransformer{}, - ), - ), + &MissingProvisionerTransformer{Provisioners: b.Components.ResourceProvisioners()}, + + &AttachSchemaTransformer{Components: b.Components}, // Add module variables &ModuleVariableTransformer{ @@ -122,6 +115,10 @@ func (b *PlanGraphBuilder) Steps() []GraphTransformer { // Remove modules no longer present in the config &RemovedModuleTransformer{Config: b.Config, State: b.State}, + // Must be before ReferenceTransformer, since schema is required to + // extract references from config. + &AttachSchemaTransformer{Components: b.Components}, + // Connect so that the references are ready for targeting. We'll // have to connect again later for providers and so on. &ReferenceTransformer{}, diff --git a/terraform/graph_builder_refresh.go b/terraform/graph_builder_refresh.go index 18613ecec..a6c5373d4 100644 --- a/terraform/graph_builder_refresh.go +++ b/terraform/graph_builder_refresh.go @@ -29,8 +29,9 @@ type RefreshGraphBuilder struct { // State is the current state State *State - // Providers is the list of providers supported. - Providers []string + // Components is a factory for the plug-in components (providers and + // provisioners) available for use. + Components contextComponentFactory // Targets are resources to target Targets []addrs.Targetable @@ -124,7 +125,7 @@ func (b *RefreshGraphBuilder) Steps() []GraphTransformer { // Add root variables &RootVariableTransformer{Config: b.Config}, - TransformProviders(b.Providers, concreteProvider, b.Config), + TransformProviders(b.Components.ResourceProviders(), concreteProvider, b.Config), // Add the local values &LocalTransformer{Config: b.Config}, @@ -135,6 +136,10 @@ func (b *RefreshGraphBuilder) Steps() []GraphTransformer { // Add module variables &ModuleVariableTransformer{Config: b.Config}, + // Must be before ReferenceTransformer, since schema is required to + // extract references from config. + &AttachSchemaTransformer{Components: b.Components}, + // Connect so that the references are ready for targeting. We'll // have to connect again later for providers and so on. &ReferenceTransformer{}, diff --git a/terraform/graph_walk_context.go b/terraform/graph_walk_context.go index 82857cf4b..68ad54961 100644 --- a/terraform/graph_walk_context.go +++ b/terraform/graph_walk_context.go @@ -73,7 +73,18 @@ func (w *ContextGraphWalker) EnterPath(path addrs.ModuleInstance) EvalContext { // so that we can safely run multiple evaluations at once across // different modules. evaluator := &Evaluator{ - StateLock: &w.Context.stateLock, + Meta: w.Context.meta, + Config: w.Context.config, + State: w.Context.state, + StateLock: &w.Context.stateLock, + ProviderSchemas: w.providerSchemas, + ProvidersLock: &w.providerLock, + + // FIXME: This was a design mistake on the evaluator, which should + // get replaced with something like the interpolatorVars thing above + // once we verify exactly how that was used in the old Interpolator + // codepath. + RootVariableValues: map[string]*InputValue{}, } ctx := &BuiltinEvalContext{ @@ -84,6 +95,7 @@ func (w *ContextGraphWalker) EnterPath(path addrs.ModuleInstance) EvalContext { Components: w.Context.components, ProviderCache: w.providerCache, ProviderInputConfig: w.Context.providerInputConfig, + ProviderSchemas: w.providerSchemas, ProviderLock: &w.providerLock, ProvisionerCache: w.provisionerCache, ProvisionerLock: &w.provisionerLock, diff --git a/terraform/node_data_refresh.go b/terraform/node_data_refresh.go index 610c2c13f..d6909e962 100644 --- a/terraform/node_data_refresh.go +++ b/terraform/node_data_refresh.go @@ -66,6 +66,7 @@ func (n *NodeRefreshableDataResource) DynamicExpand(ctx EvalContext) (*Graph, er // Expand the count. &ResourceCountTransformer{ Concrete: concreteResource, + Schema: n.Schema, Count: count, Addr: n.ResourceAddr(), }, diff --git a/terraform/node_provider_abstract.go b/terraform/node_provider_abstract.go index ac6e54898..ff82a2197 100644 --- a/terraform/node_provider_abstract.go +++ b/terraform/node_provider_abstract.go @@ -2,6 +2,7 @@ package terraform import ( "github.com/hashicorp/terraform/addrs" + "github.com/hashicorp/terraform/config/configschema" "github.com/hashicorp/terraform/configs" "github.com/hashicorp/terraform/dag" @@ -21,16 +22,17 @@ type NodeAbstractProvider struct { // set if you already have that information. Config *configs.Provider - Schema *ProviderSchema + Schema *configschema.Block } var ( - _ GraphNodeSubPath = (*NodeAbstractProvider)(nil) - _ RemovableIfNotTargeted = (*NodeAbstractProvider)(nil) - _ GraphNodeReferencer = (*NodeAbstractProvider)(nil) - _ GraphNodeProvider = (*NodeAbstractProvider)(nil) - _ GraphNodeAttachProvider = (*NodeAbstractProvider)(nil) - _ dag.GraphNodeDotter = (*NodeAbstractProvider)(nil) + _ GraphNodeSubPath = (*NodeAbstractProvider)(nil) + _ RemovableIfNotTargeted = (*NodeAbstractProvider)(nil) + _ GraphNodeReferencer = (*NodeAbstractProvider)(nil) + _ GraphNodeProvider = (*NodeAbstractProvider)(nil) + _ GraphNodeAttachProvider = (*NodeAbstractProvider)(nil) + _ GraphNodeAttachProviderConfigSchema = (*NodeAbstractProvider)(nil) + _ dag.GraphNodeDotter = (*NodeAbstractProvider)(nil) ) func (n *NodeAbstractProvider) Name() string { @@ -55,7 +57,7 @@ func (n *NodeAbstractProvider) References() []*addrs.Reference { return nil } - return ReferencesFromConfig(n.Config.Config, n.Schema.Provider) + return ReferencesFromConfig(n.Config.Config, n.Schema) } // GraphNodeProvider @@ -77,9 +79,9 @@ func (n *NodeAbstractProvider) AttachProvider(c *configs.Provider) { n.Config = c } -// GraphNodeAttachProvider -func (n *NodeAbstractProvider) AttachProviderSchema(s *ProviderSchema) { - n.Schema = s +// GraphNodeAttachProviderConfigSchema impl. +func (n *NodeAbstractProvider) AttachProviderConfigSchema(schema *configschema.Block) { + n.Schema = schema } // GraphNodeDotter impl. diff --git a/terraform/node_resource_abstract.go b/terraform/node_resource_abstract.go index f399be9f8..3fc45cc4d 100644 --- a/terraform/node_resource_abstract.go +++ b/terraform/node_resource_abstract.go @@ -68,6 +68,7 @@ var ( _ GraphNodeProvisionerConsumer = (*NodeAbstractResource)(nil) _ GraphNodeResource = (*NodeAbstractResource)(nil) _ GraphNodeAttachResourceConfig = (*NodeAbstractResource)(nil) + _ GraphNodeAttachResourceSchema = (*NodeAbstractResource)(nil) _ GraphNodeTargetable = (*NodeAbstractResource)(nil) _ dag.GraphNodeDotter = (*NodeAbstractResource)(nil) ) @@ -106,7 +107,8 @@ var ( _ GraphNodeResourceInstance = (*NodeAbstractResourceInstance)(nil) _ GraphNodeAttachResourceState = (*NodeAbstractResourceInstance)(nil) _ GraphNodeAttachResourceConfig = (*NodeAbstractResourceInstance)(nil) - _ GraphNodeTargetable = (*NodeAbstractResource)(nil) + _ GraphNodeAttachResourceSchema = (*NodeAbstractResourceInstance)(nil) + _ GraphNodeTargetable = (*NodeAbstractResourceInstance)(nil) _ dag.GraphNodeDotter = (*NodeAbstractResourceInstance)(nil) ) @@ -176,6 +178,12 @@ func (n *NodeAbstractResource) References() []*addrs.Reference { result = append(result, ref) } + if n.Schema == nil { + // Should never happens, but we'll log if it does so that we can + // see this easily when debugging. + log.Printf("[WARN] no schema is attached to %s, so references cannot be detected", n.Name()) + } + refs, _ := lang.ReferencesInExpr(c.Count) result = append(result, refs...) refs, _ = lang.ReferencesInBlock(c.Config, n.Schema) @@ -353,7 +361,7 @@ func (n *NodeAbstractResource) ProvisionedBy() []string { } // GraphNodeProvisionerConsumer -func (n *NodeAbstractResource) SetProvisionerSchema(name string, schema *configschema.Block) { +func (n *NodeAbstractResource) AttachProvisionerSchema(name string, schema *configschema.Block) { n.ProvisionerSchemas[name] = schema } @@ -387,6 +395,11 @@ func (n *NodeAbstractResource) AttachResourceConfig(c *configs.Resource) { n.Config = c } +// GraphNodeAttachResourceSchema impl +func (n *NodeAbstractResource) AttachResourceSchema(schema *configschema.Block) { + n.Schema = schema +} + // GraphNodeDotter impl. func (n *NodeAbstractResource) DotNode(name string, opts *dag.DotOpts) *dag.DotNode { return &dag.DotNode{ diff --git a/terraform/node_resource_plan.go b/terraform/node_resource_plan.go index 1cd68b2a0..54d5dd67f 100644 --- a/terraform/node_resource_plan.go +++ b/terraform/node_resource_plan.go @@ -66,6 +66,7 @@ func (n *NodePlannableResource) DynamicExpand(ctx EvalContext) (*Graph, error) { // Expand the count. &ResourceCountTransformer{ Concrete: concreteResource, + Schema: n.Schema, Count: count, Addr: n.ResourceAddr(), }, diff --git a/terraform/node_resource_refresh.go b/terraform/node_resource_refresh.go index 39983e742..3337867f8 100644 --- a/terraform/node_resource_refresh.go +++ b/terraform/node_resource_refresh.go @@ -58,6 +58,7 @@ func (n *NodeRefreshableManagedResource) DynamicExpand(ctx EvalContext) (*Graph, // Expand the count. &ResourceCountTransformer{ Concrete: concreteResource, + Schema: n.Schema, Count: count, Addr: n.ResourceAddr(), }, diff --git a/terraform/node_resource_validate.go b/terraform/node_resource_validate.go index f858f9d1e..d20087b38 100644 --- a/terraform/node_resource_validate.go +++ b/terraform/node_resource_validate.go @@ -65,6 +65,7 @@ func (n *NodeValidatableResource) DynamicExpand(ctx EvalContext) (*Graph, error) // Expand the count. &ResourceCountTransformer{ Concrete: concreteResource, + Schema: n.Schema, Count: count, Addr: n.ResourceAddr(), }, diff --git a/terraform/transform_attach_config_provider.go b/terraform/transform_attach_config_provider.go index 04eee1cb7..897a7e791 100644 --- a/terraform/transform_attach_config_provider.go +++ b/terraform/transform_attach_config_provider.go @@ -16,7 +16,4 @@ type GraphNodeAttachProvider interface { // Sets the configuration AttachProvider(*configs.Provider) - - // Sets the schema - AttachProviderSchema(*ProviderSchema) } diff --git a/terraform/transform_attach_schema.go b/terraform/transform_attach_schema.go new file mode 100644 index 000000000..034af35d6 --- /dev/null +++ b/terraform/transform_attach_schema.go @@ -0,0 +1,132 @@ +package terraform + +import ( + "fmt" + "log" + + "github.com/hashicorp/terraform/dag" + + "github.com/hashicorp/terraform/addrs" + + "github.com/hashicorp/terraform/config/configschema" +) + +// GraphNodeAttachResourceSchema is an interface implemented by node types +// that need a resource schema attached. +type GraphNodeAttachResourceSchema interface { + GraphNodeResource + GraphNodeProviderConsumer + + AttachResourceSchema(*configschema.Block) +} + +// GraphNodeAttachProviderConfigSchema is an interface implemented by node types +// that need a provider configuration schema attached. +type GraphNodeAttachProviderConfigSchema interface { + GraphNodeProvider + + AttachProviderConfigSchema(*configschema.Block) +} + +// AttachSchemaTransformer finds nodes that implement either +// GraphNodeAttachResourceSchema or GraphNodeAttachProviderConfigSchema, looks up +// the schema for each, and then passes it to a method implemented by the +// node. +type AttachSchemaTransformer struct { + Components contextComponentFactory +} + +func (t *AttachSchemaTransformer) Transform(g *Graph) error { + + // First we'll figure out which provider types we need to fetch schemas for. + needProviders := make(map[string]struct{}) + for _, v := range g.Vertices() { + switch tv := v.(type) { + case GraphNodeAttachResourceSchema: + providerAddr, _ := tv.ProvidedBy() + needProviders[providerAddr.ProviderConfig.Type] = struct{}{} + case GraphNodeAttachProviderConfigSchema: + providerAddr := tv.ProviderAddr() + needProviders[providerAddr.ProviderConfig.Type] = struct{}{} + } + } + + // Now we'll fetch each one. This requires us to temporarily instantiate + // them, though this is not a full bootstrap since we don't yet have + // configuration information; the providers will be re-instantiated and + // properly configured during the graph walk. + schemas := make(map[string]*ProviderSchema) + for typeName := range needProviders { + log.Printf("[TRACE] AttachSchemaTransformer: retrieving schema for provider type %q", typeName) + provider, err := t.Components.ResourceProvider(typeName, "early/"+typeName) + if err != nil { + return fmt.Errorf("failed to instantiate provider %q to obtain schema: %s", typeName, err) + } + + // FIXME: The provider interface is currently awkward in that it + // requires us to tell the provider which resources types and data + // sources we need. In future this will change to just return + // everything available, but for now we'll fake that by fetching all + // of the available names and then requesting them. + resourceTypes := provider.Resources() + dataSources := provider.DataSources() + resourceTypeNames := make([]string, len(resourceTypes)) + for i, o := range resourceTypes { + resourceTypeNames[i] = o.Name + } + dataSourceNames := make([]string, len(dataSources)) + for i, o := range dataSources { + dataSourceNames[i] = o.Name + } + + schema, err := provider.GetSchema(&ProviderSchemaRequest{ + ResourceTypes: resourceTypeNames, + DataSources: dataSourceNames, + }) + if err != nil { + return fmt.Errorf("failed to retrieve schema from provider %q: %s", typeName, err) + } + + schemas[typeName] = schema + + if closer, ok := provider.(ResourceProviderCloser); ok { + closer.Close() + } + } + + // Finally we'll once again visit all of the vertices and attach to + // them the schemas we found for them. + for _, v := range g.Vertices() { + switch tv := v.(type) { + case GraphNodeAttachResourceSchema: + addr := tv.ResourceAddr() + mode := addr.Resource.Mode + typeName := addr.Resource.Type + providerAddr, _ := tv.ProvidedBy() + var schema *configschema.Block + switch mode { + case addrs.ManagedResourceMode: + schema = schemas[providerAddr.ProviderConfig.Type].ResourceTypes[typeName] + case addrs.DataResourceMode: + schema = schemas[providerAddr.ProviderConfig.Type].DataSources[typeName] + } + if schema != nil { + log.Printf("[TRACE] AttachSchemaTransformer: attaching schema to %s", dag.VertexName(v)) + tv.AttachResourceSchema(schema) + } else { + log.Printf("[ERROR] AttachSchemaTransformer: No schema available for %s", addr) + } + case GraphNodeAttachProviderConfigSchema: + providerAddr := tv.ProviderAddr() + schema := schemas[providerAddr.ProviderConfig.Type].Provider + if schema != nil { + log.Printf("[TRACE] AttachSchemaTransformer: attaching schema to %s", dag.VertexName(v)) + tv.AttachProviderConfigSchema(schema) + } else { + log.Printf("[ERROR] AttachSchemaTransformer: No schema available for %s", providerAddr) + } + } + } + + return nil +} diff --git a/terraform/transform_provisioner.go b/terraform/transform_provisioner.go index c9d2e233d..a3081207c 100644 --- a/terraform/transform_provisioner.go +++ b/terraform/transform_provisioner.go @@ -34,7 +34,7 @@ type GraphNodeProvisionerConsumer interface { // type returned from ProvisionedBy, providing the configuration schema // for each provisioner in turn. The implementer should save these for // later use in evaluating provisioner configuration blocks. - SetProvisionerSchema(name string, schema *configschema.Block) + AttachProvisionerSchema(name string, schema *configschema.Block) } // ProvisionerTransformer is a GraphTransformer that maps resources to diff --git a/terraform/transform_resource_count.go b/terraform/transform_resource_count.go index baef38105..7f325df38 100644 --- a/terraform/transform_resource_count.go +++ b/terraform/transform_resource_count.go @@ -2,6 +2,7 @@ package terraform import ( "github.com/hashicorp/terraform/addrs" + "github.com/hashicorp/terraform/config/configschema" "github.com/hashicorp/terraform/dag" ) @@ -11,6 +12,7 @@ import ( // This assumes that the count is already interpolated. type ResourceCountTransformer struct { Concrete ConcreteResourceInstanceNodeFunc + Schema *configschema.Block // Count is either the number of indexed instances to create, or -1 to // indicate that count is not set at all and thus a no-key instance should @@ -25,6 +27,7 @@ func (t *ResourceCountTransformer) Transform(g *Graph) error { addr := t.Addr.Instance(addrs.NoKey) abstract := NewNodeAbstractResourceInstance(addr) + abstract.Schema = t.Schema var node dag.Vertex = abstract if f := t.Concrete; f != nil { node = f(abstract)