From 88b5607a7ad9f7491c151f74ca84612a2d7a3c1d Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 31 May 2018 12:39:45 -0700 Subject: [PATCH] core: Fetch schemas during context construction Previously we fetched schemas during the AttachSchemaTransformer, potentially multiple times as that was re-run for each graph built. Now we fetch the schemas just once during context construction, passing that result into each of the graph builders. This only addresses the schema accesses during graph construction. We're still separately loading schemas during the main walk for evaluation purposes. This will be addressed in a later commit. --- terraform/context.go | 52 ++-- terraform/context_import.go | 1 + terraform/context_validate_test.go | 1 + terraform/graph_builder_apply.go | 18 +- terraform/graph_builder_apply_test.go | 9 + terraform/graph_builder_destroy_plan.go | 12 +- terraform/graph_builder_eval.go | 11 +- terraform/graph_builder_import.go | 6 +- terraform/graph_builder_plan.go | 9 +- terraform/graph_builder_plan_test.go | 53 ++-- terraform/graph_builder_refresh.go | 6 +- terraform/graph_builder_refresh_test.go | 1 + terraform/node_resource_plan.go | 12 +- terraform/schemas.go | 256 +++++++++++++++++- terraform/schemas_test.go | 18 ++ terraform/transform_attach_schema.go | 183 ++----------- terraform/transform_destroy_cbd.go | 10 +- terraform/transform_destroy_cbd_test.go | 32 +-- terraform/transform_destroy_edge.go | 6 +- terraform/transform_destroy_edge_test.go | 24 +- .../transform_transitive_reduction_test.go | 31 ++- 21 files changed, 466 insertions(+), 285 deletions(-) create mode 100644 terraform/schemas_test.go diff --git a/terraform/context.go b/terraform/context.go index 0841223e9..e025e2a1e 100644 --- a/terraform/context.go +++ b/terraform/context.go @@ -96,6 +96,7 @@ type Context struct { // fail regardless but putting this note here as well. components contextComponentFactory + schemas *Schemas destroy bool diff *Diff diffLock sync.RWMutex @@ -203,26 +204,35 @@ func NewContext(opts *ContextOpts) (*Context, tfdiags.Diagnostics) { providers = make(map[string]ResourceProviderFactory) } + components := &basicComponentFactory{ + providers: providers, + provisioners: opts.Provisioners, + } + + schemas, err := LoadSchemas(opts.Config, opts.State, components) + if err != nil { + diags = diags.Append(err) + return nil, diags + } + diff := opts.Diff if diff == nil { diff = &Diff{} } return &Context{ - components: &basicComponentFactory{ - providers: providers, - provisioners: opts.Provisioners, - }, - destroy: opts.Destroy, - diff: diff, - hooks: hooks, - meta: opts.Meta, - config: opts.Config, - shadow: opts.Shadow, - state: state, - targets: opts.Targets, - uiInput: opts.UIInput, - variables: variables, + components: components, + schemas: schemas, + destroy: opts.Destroy, + diff: diff, + hooks: hooks, + meta: opts.Meta, + config: opts.Config, + shadow: opts.Shadow, + state: state, + targets: opts.Targets, + uiInput: opts.UIInput, + variables: variables, parallelSem: NewSemaphore(par), providerInputConfig: make(map[string]map[string]cty.Value), @@ -255,6 +265,7 @@ func (c *Context) Graph(typ GraphType, opts *ContextGraphOpts) (*Graph, tfdiags. Diff: c.diff, State: c.state, Components: c.components, + Schemas: c.schemas, Targets: c.targets, Destroy: c.destroy, Validate: opts.Validate, @@ -272,6 +283,7 @@ func (c *Context) Graph(typ GraphType, opts *ContextGraphOpts) (*Graph, tfdiags. Config: c.config, State: c.state, Components: c.components, + Schemas: c.schemas, Targets: c.targets, Validate: opts.Validate, } @@ -289,11 +301,11 @@ func (c *Context) Graph(typ GraphType, opts *ContextGraphOpts) (*Graph, tfdiags. case GraphTypePlanDestroy: return (&DestroyPlanGraphBuilder{ - Config: c.config, - State: c.state, - Components: c.components, - Targets: c.targets, - Validate: opts.Validate, + Config: c.config, + State: c.state, + Schemas: c.schemas, + Targets: c.targets, + Validate: opts.Validate, }).Build(addrs.RootModuleInstance) case GraphTypeRefresh: @@ -301,6 +313,7 @@ func (c *Context) Graph(typ GraphType, opts *ContextGraphOpts) (*Graph, tfdiags. Config: c.config, State: c.state, Components: c.components, + Schemas: c.schemas, Targets: c.targets, Validate: opts.Validate, }).Build(addrs.RootModuleInstance) @@ -310,6 +323,7 @@ func (c *Context) Graph(typ GraphType, opts *ContextGraphOpts) (*Graph, tfdiags. Config: c.config, State: c.state, Components: c.components, + Schemas: c.schemas, }).Build(addrs.RootModuleInstance) default: diff --git a/terraform/context_import.go b/terraform/context_import.go index 71edfe2b1..5b1aabc48 100644 --- a/terraform/context_import.go +++ b/terraform/context_import.go @@ -61,6 +61,7 @@ func (c *Context) Import(opts *ImportOpts) (*State, tfdiags.Diagnostics) { ImportTargets: opts.Targets, Config: config, Components: c.components, + Schemas: c.schemas, } // Build the graph! diff --git a/terraform/context_validate_test.go b/terraform/context_validate_test.go index ba1076644..ccffffe1f 100644 --- a/terraform/context_validate_test.go +++ b/terraform/context_validate_test.go @@ -1220,6 +1220,7 @@ func TestContext2Validate_PlanGraphBuilder(t *testing.T) { Config: c.config, State: NewState(), Components: c.components, + Schemas: c.schemas, Targets: c.targets, }).Build(addrs.RootModuleInstance) if diags.HasErrors() { diff --git a/terraform/graph_builder_apply.go b/terraform/graph_builder_apply.go index a2e37aedf..1a0c4a9cb 100644 --- a/terraform/graph_builder_apply.go +++ b/terraform/graph_builder_apply.go @@ -28,6 +28,10 @@ type ApplyGraphBuilder struct { // provisioners) available for use. Components contextComponentFactory + // Schemas is the repository of schemas we will draw from to analyse + // the configuration. + Schemas *Schemas + // Targets are resources to target. This is only required to make sure // unnecessary outputs aren't included in the apply graph. The plan // builder successfully handles targeting resources. In the future, @@ -86,16 +90,16 @@ func (b *ApplyGraphBuilder) Steps() []GraphTransformer { // Destruction ordering &DestroyEdgeTransformer{ - Config: b.Config, - State: b.State, - Components: b.Components, + Config: b.Config, + State: b.State, + Schemas: b.Schemas, }, GraphTransformIf( func() bool { return !b.Destroy }, &CBDEdgeTransformer{ - Config: b.Config, - State: b.State, - Components: b.Components, + Config: b.Config, + State: b.State, + Schemas: b.Schemas, }, ), @@ -117,7 +121,7 @@ func (b *ApplyGraphBuilder) Steps() []GraphTransformer { // Must be before TransformProviders and ReferenceTransformer, since // schema is required to extract references from config. - &AttachSchemaTransformer{Components: b.Components}, + &AttachSchemaTransformer{Schemas: b.Schemas}, // add providers TransformProviders(b.Components.ResourceProviders(), concreteProvider, b.Config), diff --git a/terraform/graph_builder_apply_test.go b/terraform/graph_builder_apply_test.go index f6ccafff4..b489de419 100644 --- a/terraform/graph_builder_apply_test.go +++ b/terraform/graph_builder_apply_test.go @@ -69,6 +69,7 @@ func TestApplyGraphBuilder(t *testing.T) { Config: testModule(t, "graph-builder-apply-basic"), Diff: diff, Components: simpleMockComponentFactory(), + Schemas: simpleTestSchemas(), DisableReduce: true, } @@ -122,6 +123,7 @@ func TestApplyGraphBuilder_depCbd(t *testing.T) { Config: testModule(t, "graph-builder-apply-dep-cbd"), Diff: diff, Components: simpleMockComponentFactory(), + Schemas: simpleTestSchemas(), DisableReduce: true, } @@ -187,6 +189,7 @@ func TestApplyGraphBuilder_doubleCBD(t *testing.T) { Config: testModule(t, "graph-builder-apply-double-cbd"), Diff: diff, Components: simpleMockComponentFactory(), + Schemas: simpleTestSchemas(), DisableReduce: true, } @@ -257,6 +260,7 @@ func TestApplyGraphBuilder_destroyStateOnly(t *testing.T) { Diff: diff, State: state, Components: simpleMockComponentFactory(), + Schemas: simpleTestSchemas(), DisableReduce: true, } @@ -304,6 +308,7 @@ func TestApplyGraphBuilder_destroyCount(t *testing.T) { Config: testModule(t, "graph-builder-apply-count"), Diff: diff, Components: simpleMockComponentFactory(), + Schemas: simpleTestSchemas(), DisableReduce: true, } @@ -350,6 +355,7 @@ func TestApplyGraphBuilder_moduleDestroy(t *testing.T) { Config: testModule(t, "graph-builder-apply-module-destroy"), Diff: diff, Components: simpleMockComponentFactory(), + Schemas: simpleTestSchemas(), } g, err := b.Build(addrs.RootModuleInstance) @@ -387,6 +393,7 @@ func TestApplyGraphBuilder_provisioner(t *testing.T) { Config: testModule(t, "graph-builder-apply-provisioner"), Diff: diff, Components: simpleMockComponentFactory(), + Schemas: simpleTestSchemas(), } g, err := b.Build(addrs.RootModuleInstance) @@ -421,6 +428,7 @@ func TestApplyGraphBuilder_provisionerDestroy(t *testing.T) { Config: testModule(t, "graph-builder-apply-provisioner"), Diff: diff, Components: simpleMockComponentFactory(), + Schemas: simpleTestSchemas(), } g, err := b.Build(addrs.RootModuleInstance) @@ -472,6 +480,7 @@ func TestApplyGraphBuilder_targetModule(t *testing.T) { Config: testModule(t, "graph-builder-apply-target-module"), Diff: diff, Components: simpleMockComponentFactory(), + Schemas: simpleTestSchemas(), Targets: []addrs.Targetable{ addrs.RootModuleInstance.Child("child2", addrs.NoKey), }, diff --git a/terraform/graph_builder_destroy_plan.go b/terraform/graph_builder_destroy_plan.go index dbf43e0e9..80568363d 100644 --- a/terraform/graph_builder_destroy_plan.go +++ b/terraform/graph_builder_destroy_plan.go @@ -22,9 +22,9 @@ type DestroyPlanGraphBuilder struct { // Targets are resources to target Targets []addrs.Targetable - // Components is a factory for the plug-in components (providers and - // provisioners) available for use. - Components contextComponentFactory + // Schemas is the repository of schemas we will draw from to analyse + // the configuration. + Schemas *Schemas // Validate will do structural validation of the graph. Validate bool @@ -60,9 +60,9 @@ func (b *DestroyPlanGraphBuilder) Steps() []GraphTransformer { // Destruction ordering. We require this only so that // targeting below will prune the correct things. &DestroyEdgeTransformer{ - Config: b.Config, - State: b.State, - Components: b.Components, + Config: b.Config, + State: b.State, + Schemas: b.Schemas, }, // Target. Note we don't set "Destroy: true" here since we already diff --git a/terraform/graph_builder_eval.go b/terraform/graph_builder_eval.go index df494f39f..0da15b3b9 100644 --- a/terraform/graph_builder_eval.go +++ b/terraform/graph_builder_eval.go @@ -1,11 +1,10 @@ package terraform import ( - "github.com/hashicorp/terraform/dag" - "github.com/hashicorp/terraform/tfdiags" - "github.com/hashicorp/terraform/addrs" "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/dag" + "github.com/hashicorp/terraform/tfdiags" ) // EvalGraphBuilder implements GraphBuilder and constructs a graph suitable @@ -33,6 +32,10 @@ type EvalGraphBuilder struct { // Components is a factory for the plug-in components (providers and // provisioners) available for use. Components contextComponentFactory + + // Schemas is the repository of schemas we will draw from to analyse + // the configuration. + Schemas *Schemas } // See GraphBuilder @@ -72,7 +75,7 @@ func (b *EvalGraphBuilder) Steps() []GraphTransformer { // Must be before TransformProviders and ReferenceTransformer, since // schema is required to extract references from config. - &AttachSchemaTransformer{Components: b.Components}, + &AttachSchemaTransformer{Schemas: b.Schemas}, TransformProviders(b.Components.ResourceProviders(), concreteProvider, b.Config), diff --git a/terraform/graph_builder_import.go b/terraform/graph_builder_import.go index e7d6d20c7..50f38e394 100644 --- a/terraform/graph_builder_import.go +++ b/terraform/graph_builder_import.go @@ -19,6 +19,10 @@ type ImportGraphBuilder struct { // Components is the factory for our available plugin components. Components contextComponentFactory + + // Schemas is the repository of schemas we will draw from to analyse + // the configuration. + Schemas *Schemas } // Build builds the graph according to the steps returned by Steps. @@ -59,7 +63,7 @@ func (b *ImportGraphBuilder) Steps() []GraphTransformer { // Must be before TransformProviders and ReferenceTransformer, since // schema is required to extract references from config. - &AttachSchemaTransformer{Components: b.Components}, + &AttachSchemaTransformer{Schemas: b.Schemas}, TransformProviders(b.Components.ResourceProviders(), concreteProvider, config), diff --git a/terraform/graph_builder_plan.go b/terraform/graph_builder_plan.go index fffa5554a..ed734d704 100644 --- a/terraform/graph_builder_plan.go +++ b/terraform/graph_builder_plan.go @@ -33,6 +33,10 @@ type PlanGraphBuilder struct { // provisioners) available for use. Components contextComponentFactory + // Schemas is the repository of schemas we will draw from to analyse + // the configuration. + Schemas *Schemas + // Targets are resources to target Targets []addrs.Targetable @@ -103,7 +107,7 @@ func (b *PlanGraphBuilder) Steps() []GraphTransformer { &MissingProvisionerTransformer{Provisioners: b.Components.ResourceProvisioners()}, - &AttachSchemaTransformer{Components: b.Components}, + &AttachSchemaTransformer{Schemas: b.Schemas}, // Add module variables &ModuleVariableTransformer{ @@ -117,7 +121,7 @@ func (b *PlanGraphBuilder) Steps() []GraphTransformer { // Must be before ReferenceTransformer, since schema is required to // extract references from config. - &AttachSchemaTransformer{Components: b.Components}, + &AttachSchemaTransformer{Schemas: b.Schemas}, // Connect so that the references are ready for targeting. We'll // have to connect again later for providers and so on. @@ -169,7 +173,6 @@ func (b *PlanGraphBuilder) init() { b.ConcreteResource = func(a *NodeAbstractResource) dag.Vertex { return &NodePlannableResource{ NodeAbstractResource: a, - Components: b.Components, } } diff --git a/terraform/graph_builder_plan_test.go b/terraform/graph_builder_plan_test.go index 84aaadc99..6fab643a7 100644 --- a/terraform/graph_builder_plan_test.go +++ b/terraform/graph_builder_plan_test.go @@ -13,28 +13,38 @@ func TestPlanGraphBuilder_impl(t *testing.T) { } func TestPlanGraphBuilder(t *testing.T) { + awsProvider := &MockResourceProvider{ + GetSchemaReturn: &ProviderSchema{ + Provider: simpleTestSchema(), + ResourceTypes: map[string]*configschema.Block{ + "aws_security_group": simpleTestSchema(), + "aws_instance": simpleTestSchema(), + "aws_load_balancer": simpleTestSchema(), + }, + }, + } + openstackProvider := &MockResourceProvider{ + GetSchemaReturn: &ProviderSchema{ + Provider: simpleTestSchema(), + ResourceTypes: map[string]*configschema.Block{ + "openstack_floating_ip": simpleTestSchema(), + }, + }, + } + components := &basicComponentFactory{ + providers: map[string]ResourceProviderFactory{ + "aws": ResourceProviderFactoryFixed(awsProvider), + "openstack": ResourceProviderFactoryFixed(openstackProvider), + }, + } + b := &PlanGraphBuilder{ - Config: testModule(t, "graph-builder-plan-basic"), - Components: &basicComponentFactory{ - providers: map[string]ResourceProviderFactory{ - "aws": ResourceProviderFactoryFixed(&MockResourceProvider{ - GetSchemaReturn: &ProviderSchema{ - Provider: simpleTestSchema(), - ResourceTypes: map[string]*configschema.Block{ - "aws_security_group": simpleTestSchema(), - "aws_instance": simpleTestSchema(), - "aws_load_balancer": simpleTestSchema(), - }, - }, - }), - "openstack": ResourceProviderFactoryFixed(&MockResourceProvider{ - GetSchemaReturn: &ProviderSchema{ - Provider: simpleTestSchema(), - ResourceTypes: map[string]*configschema.Block{ - "openstack_floating_ip": simpleTestSchema(), - }, - }, - }), + Config: testModule(t, "graph-builder-plan-basic"), + Components: components, + Schemas: &Schemas{ + providers: map[string]*ProviderSchema{ + "aws": awsProvider.GetSchemaReturn, + "openstack": openstackProvider.GetSchemaReturn, }, }, DisableReduce: true, @@ -60,6 +70,7 @@ func TestPlanGraphBuilder_targetModule(t *testing.T) { b := &PlanGraphBuilder{ Config: testModule(t, "graph-builder-plan-target-module-provider"), Components: simpleMockComponentFactory(), + Schemas: simpleTestSchemas(), Targets: []addrs.Targetable{ addrs.RootModuleInstance.Child("child2", addrs.NoKey), }, diff --git a/terraform/graph_builder_refresh.go b/terraform/graph_builder_refresh.go index f181a8e44..87892191c 100644 --- a/terraform/graph_builder_refresh.go +++ b/terraform/graph_builder_refresh.go @@ -33,6 +33,10 @@ type RefreshGraphBuilder struct { // provisioners) available for use. Components contextComponentFactory + // Schemas is the repository of schemas we will draw from to analyse + // the configuration. + Schemas *Schemas + // Targets are resources to target Targets []addrs.Targetable @@ -136,7 +140,7 @@ func (b *RefreshGraphBuilder) Steps() []GraphTransformer { // Must be before TransformProviders and ReferenceTransformer, since // schema is required to extract references from config. - &AttachSchemaTransformer{Components: b.Components}, + &AttachSchemaTransformer{Schemas: b.Schemas}, TransformProviders(b.Components.ResourceProviders(), concreteProvider, b.Config), diff --git a/terraform/graph_builder_refresh_test.go b/terraform/graph_builder_refresh_test.go index a22989455..6be874529 100644 --- a/terraform/graph_builder_refresh_test.go +++ b/terraform/graph_builder_refresh_test.go @@ -73,6 +73,7 @@ func TestRefreshGraphBuilder_configOrphans(t *testing.T) { Config: m, State: state, Components: simpleMockComponentFactory(), + Schemas: simpleTestSchemas(), } g, err := b.Build(addrs.RootModuleInstance) if err != nil { diff --git a/terraform/node_resource_plan.go b/terraform/node_resource_plan.go index dfa14a342..8884a918e 100644 --- a/terraform/node_resource_plan.go +++ b/terraform/node_resource_plan.go @@ -9,10 +9,6 @@ import ( // it is ready to be planned in order to create a diff. type NodePlannableResource struct { *NodeAbstractResource - - // Components is the component factory to use when performing - // DynamicExpand, to access plugins necessary to build the subgraph. - Components contextComponentFactory } var ( @@ -48,6 +44,7 @@ func (n *NodePlannableResource) DynamicExpand(ctx EvalContext) (*Graph, error) { // Add the config and state since we don't do that via transforms a.Config = n.Config a.ResolvedProvider = n.ResolvedProvider + a.Schema = n.Schema return &NodePlannableResourceInstance{ NodeAbstractResourceInstance: a, @@ -59,6 +56,7 @@ func (n *NodePlannableResource) DynamicExpand(ctx EvalContext) (*Graph, error) { // Add the config and state since we don't do that via transforms a.Config = n.Config a.ResolvedProvider = n.ResolvedProvider + a.Schema = n.Schema return &NodePlannableResourceInstanceOrphan{ NodeAbstractResourceInstance: a, @@ -89,12 +87,6 @@ func (n *NodePlannableResource) DynamicExpand(ctx EvalContext) (*Graph, error) { // Targeting &TargetsTransformer{Targets: n.Targets}, - // Schemas must be attached before ReferenceTransformer, so we can - // properly analyze the configuration. - &AttachSchemaTransformer{ - Components: n.Components, - }, - // Connect references so ordering is correct &ReferenceTransformer{}, diff --git a/terraform/schemas.go b/terraform/schemas.go index ec46efcf7..998c0d5cd 100644 --- a/terraform/schemas.go +++ b/terraform/schemas.go @@ -1,18 +1,264 @@ package terraform import ( + "fmt" + "log" + + "github.com/hashicorp/terraform/addrs" "github.com/hashicorp/terraform/config/configschema" + "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/tfdiags" ) +// Schemas is a container for various kinds of schema that Terraform needs +// during processing. type Schemas struct { - Providers ProviderSchemas + providers map[string]*ProviderSchema + provisioners map[string]*configschema.Block } -// ProviderSchemas is a map from provider names to provider schemas. +// ProviderConfig returns the schema for the provider configuration of the +// given provider type, or nil if no such schema is available. +func (ss *Schemas) ProviderConfig(typeName string) *configschema.Block { + ps, exists := ss.providers[typeName] + if !exists { + return nil + } + return ps.Provider +} + +// ResourceTypeConfig returns the schema for the configuration of a given +// resource type belonging to a given provider type, or nil of no such +// schema is available. // -// The names in this map are the direct plugin name (e.g. "aws") rather than -// any alias name (e.g. "aws.foo"), since. -type ProviderSchemas map[string]*ProviderSchema +// In many cases the provider type is inferrable from the resource type name, +// but this is not always true because users can override the provider for +// a resource using the "provider" meta-argument. Therefore it's important to +// always pass the correct provider name, even though it many cases it feels +// redundant. +func (ss *Schemas) ResourceTypeConfig(providerType string, resourceType string) *configschema.Block { + ps, exists := ss.providers[providerType] + if !exists { + return nil + } + + if ps.ResourceTypes == nil { + return nil + } + + return ps.ResourceTypes[resourceType] +} + +// DataSourceConfig returns the schema for the configuration of a given +// data source belonging to a given provider type, or nil of no such +// schema is available. +// +// In many cases the provider type is inferrable from the data source name, +// but this is not always true because users can override the provider for +// a resource using the "provider" meta-argument. Therefore it's important to +// always pass the correct provider name, even though it many cases it feels +// redundant. +func (ss *Schemas) DataSourceConfig(providerType string, dataSource string) *configschema.Block { + ps, exists := ss.providers[providerType] + if !exists { + return nil + } + + if ps.DataSources == nil { + return nil + } + + return ps.DataSources[dataSource] +} + +// ProvisionerConfig returns the schema for the configuration of a given +// provisioner, or nil of no such schema is available. +func (ss *Schemas) ProvisionerConfig(name string) *configschema.Block { + return ss.provisioners[name] +} + +// LoadSchemas searches the given configuration and state (either of which may +// be nil) for constructs that have an associated schema, requests the +// necessary schemas from the given component factory (which may _not_ be nil), +// and returns a single object representing all of the necessary schemas. +// +// If an error is returned, it may be a wrapped tfdiags.Diagnostics describing +// errors across multiple separate objects. Errors here will usually indicate +// either misbehavior on the part of one of the providers or of the provider +// protocol itself. When returned with errors, the returned schemas object is +// still valid but may be incomplete. +func LoadSchemas(config *configs.Config, state *State, components contextComponentFactory) (*Schemas, error) { + schemas := &Schemas{ + providers: map[string]*ProviderSchema{}, + provisioners: map[string]*configschema.Block{}, + } + var diags tfdiags.Diagnostics + + newDiags := loadProviderSchemas(schemas.providers, config, state, components) + diags = diags.Append(newDiags) + newDiags = loadProvisionerSchemas(schemas.provisioners, config, components) + diags = diags.Append(newDiags) + + return schemas, diags.Err() +} + +func loadProviderSchemas(schemas map[string]*ProviderSchema, config *configs.Config, state *State, components contextComponentFactory) tfdiags.Diagnostics { + var diags tfdiags.Diagnostics + + ensure := func(typeName string) { + if _, exists := schemas[typeName]; exists { + return + } + + log.Printf("[TRACE] LoadSchemas: retrieving schema for provider type %q", typeName) + provider, err := components.ResourceProvider(typeName, "early/"+typeName) + if err != nil { + // We'll put a stub in the map so we won't re-attempt this on + // future calls. + schemas[typeName] = &ProviderSchema{} + diags = diags.Append( + fmt.Errorf("Failed to instantiate provider %q to obtain schema: %s", typeName, err), + ) + return + } + defer func() { + if closer, ok := provider.(ResourceProviderCloser); ok { + closer.Close() + } + }() + + // FIXME: The provider interface is currently awkward in that it + // requires us to tell the provider which resources types and data + // sources we need. In future this will change to just return + // everything available, but for now we'll fake that by fetching all + // of the available names and then requesting them. + resourceTypes := provider.Resources() + dataSources := provider.DataSources() + resourceTypeNames := make([]string, len(resourceTypes)) + for i, o := range resourceTypes { + resourceTypeNames[i] = o.Name + } + dataSourceNames := make([]string, len(dataSources)) + for i, o := range dataSources { + dataSourceNames[i] = o.Name + } + + schema, err := provider.GetSchema(&ProviderSchemaRequest{ + ResourceTypes: resourceTypeNames, + DataSources: dataSourceNames, + }) + if err != nil { + // We'll put a stub in the map so we won't re-attempt this on + // future calls. + schemas[typeName] = &ProviderSchema{} + diags = diags.Append( + fmt.Errorf("Failed to retrieve schema from provider %q: %s", typeName, err), + ) + return + } + + schemas[typeName] = schema + } + + if config != nil { + for _, pc := range config.Module.ProviderConfigs { + ensure(pc.Name) + } + for _, rc := range config.Module.ManagedResources { + providerAddr := rc.ProviderConfigAddr() + ensure(providerAddr.Type) + } + for _, rc := range config.Module.DataResources { + providerAddr := rc.ProviderConfigAddr() + ensure(providerAddr.Type) + } + + // Must also visit our child modules, recursively. + for _, cc := range config.Children { + childDiags := loadProviderSchemas(schemas, cc, nil, components) + diags = diags.Append(childDiags) + } + } + + if state != nil { + for _, ms := range state.Modules { + for rsKey, rs := range ms.Resources { + providerAddrStr := rs.Provider + providerAddr, addrDiags := addrs.ParseAbsProviderConfigStr(providerAddrStr) + if addrDiags.HasErrors() { + // Should happen only if someone has tampered manually with + // the state, since we always write valid provider addrs. + moduleAddrStr := normalizeModulePath(ms.Path).String() + if moduleAddrStr == "" { + moduleAddrStr = "the root module" + } + // For now this is a warning, since there are many existing + // test fixtures that have invalid provider configurations. + // There's a check deeper in Terraform that makes this a + // failure when an empty/invalid provider string is present + // in practice. + diags = diags.Append( + tfdiags.SimpleWarning(fmt.Sprintf("Resource %s in %s has invalid provider address %q in its state", rsKey, moduleAddrStr, providerAddrStr)), + ) + continue + } + ensure(providerAddr.ProviderConfig.Type) + } + } + } + + return diags +} + +func loadProvisionerSchemas(schemas map[string]*configschema.Block, config *configs.Config, components contextComponentFactory) tfdiags.Diagnostics { + var diags tfdiags.Diagnostics + + ensure := func(name string) { + if _, exists := schemas[name]; exists { + return + } + + log.Printf("[TRACE] AttachSchemaTransformer: retrieving schema for provisioner %q", name) + provisioner, err := components.ResourceProvisioner(name, "early/"+name) + if err != nil { + // We'll put a stub in the map so we won't re-attempt this on + // future calls. + schemas[name] = &configschema.Block{} + diags = diags.Append( + fmt.Errorf("Failed to instantiate provisioner %q to obtain schema: %s", name, err), + ) + return + } + defer func() { + if closer, ok := provisioner.(ResourceProvisionerCloser); ok { + closer.Close() + } + }() + + schema, err := provisioner.GetConfigSchema() + if err != nil { + // We'll put a stub in the map so we won't re-attempt this on + // future calls. + schemas[name] = &configschema.Block{} + diags = diags.Append( + fmt.Errorf("Failed to retrieve schema from provisioner %q: %s", name, err), + ) + return + } + + schemas[name] = schema + } + + if config != nil { + for _, rc := range config.Module.ManagedResources { + for _, pc := range rc.Managed.Provisioners { + ensure(pc.Type) + } + } + } + + return diags +} // ProviderSchema represents the schema for a provider's own configuration // and the configuration for some or all of its resources and data sources. diff --git a/terraform/schemas_test.go b/terraform/schemas_test.go new file mode 100644 index 000000000..613de44ed --- /dev/null +++ b/terraform/schemas_test.go @@ -0,0 +1,18 @@ +package terraform + +import ( + "github.com/hashicorp/terraform/config/configschema" +) + +func simpleTestSchemas() *Schemas { + provider := simpleMockProvider() + provisioner := simpleMockProvisioner() + return &Schemas{ + providers: map[string]*ProviderSchema{ + "test": provider.GetSchemaReturn, + }, + provisioners: map[string]*configschema.Block{ + "test": provisioner.GetConfigSchemaReturnSchema, + }, + } +} diff --git a/terraform/transform_attach_schema.go b/terraform/transform_attach_schema.go index a5bc07040..3a124e4f9 100644 --- a/terraform/transform_attach_schema.go +++ b/terraform/transform_attach_schema.go @@ -4,11 +4,9 @@ import ( "fmt" "log" - "github.com/hashicorp/terraform/dag" - "github.com/hashicorp/terraform/addrs" - "github.com/hashicorp/terraform/config/configschema" + "github.com/hashicorp/terraform/dag" ) // GraphNodeAttachResourceSchema is an interface implemented by node types @@ -45,91 +43,16 @@ type GraphNodeAttachProvisionerSchema interface { // GraphNodeAttachProvisionerSchema, looks up the needed schemas for each // and then passes them to a method implemented by the node. type AttachSchemaTransformer struct { - GraphNodeProvisionerConsumer - Components contextComponentFactory + Schemas *Schemas } func (t *AttachSchemaTransformer) Transform(g *Graph) error { - if t.Components == nil { + if t.Schemas == nil { // Should never happen with a reasonable caller, but we'll return a // proper error here anyway so that we'll fail gracefully. - return fmt.Errorf("AttachSchemaTransformer used with nil Components") + return fmt.Errorf("AttachSchemaTransformer used with nil Schemas") } - err := t.attachProviderSchemas(g) - if err != nil { - return err - } - err = t.attachProvisionerSchemas(g) - if err != nil { - return err - } - - return nil -} - -func (t *AttachSchemaTransformer) attachProviderSchemas(g *Graph) error { - - // First we'll figure out which provider types we need to fetch schemas for. - needProviders := make(map[string]struct{}) - for _, v := range g.Vertices() { - switch tv := v.(type) { - case GraphNodeAttachResourceSchema: - providerAddr, _ := tv.ProvidedBy() - log.Printf("[TRACE] AttachSchemaTransformer: %q (%T) needs %s", dag.VertexName(v), v, providerAddr) - needProviders[providerAddr.ProviderConfig.Type] = struct{}{} - case GraphNodeAttachProviderConfigSchema: - providerAddr := tv.ProviderAddr() - log.Printf("[TRACE] AttachSchemaTransformer: %q (%T) needs %s", dag.VertexName(v), v, providerAddr) - needProviders[providerAddr.ProviderConfig.Type] = struct{}{} - } - } - - // Now we'll fetch each one. This requires us to temporarily instantiate - // them, though this is not a full bootstrap since we don't yet have - // configuration information; the providers will be re-instantiated and - // properly configured during the graph walk. - schemas := make(map[string]*ProviderSchema) - for typeName := range needProviders { - log.Printf("[TRACE] AttachSchemaTransformer: retrieving schema for provider type %q", typeName) - provider, err := t.Components.ResourceProvider(typeName, "early/"+typeName) - if err != nil { - return fmt.Errorf("failed to instantiate provider %q to obtain schema: %s", typeName, err) - } - - // FIXME: The provider interface is currently awkward in that it - // requires us to tell the provider which resources types and data - // sources we need. In future this will change to just return - // everything available, but for now we'll fake that by fetching all - // of the available names and then requesting them. - resourceTypes := provider.Resources() - dataSources := provider.DataSources() - resourceTypeNames := make([]string, len(resourceTypes)) - for i, o := range resourceTypes { - resourceTypeNames[i] = o.Name - } - dataSourceNames := make([]string, len(dataSources)) - for i, o := range dataSources { - dataSourceNames[i] = o.Name - } - - schema, err := provider.GetSchema(&ProviderSchemaRequest{ - ResourceTypes: resourceTypeNames, - DataSources: dataSourceNames, - }) - if err != nil { - return fmt.Errorf("failed to retrieve schema from provider %q: %s", typeName, err) - } - - schemas[typeName] = schema - - if closer, ok := provider.(ResourceProviderCloser); ok { - closer.Close() - } - } - - // Finally we'll once again visit all of the vertices and attach to - // them the schemas we found for them. for _, v := range g.Vertices() { switch tv := v.(type) { case GraphNodeAttachResourceSchema: @@ -137,98 +60,40 @@ func (t *AttachSchemaTransformer) attachProviderSchemas(g *Graph) error { mode := addr.Resource.Mode typeName := addr.Resource.Type providerAddr, _ := tv.ProvidedBy() + providerType := providerAddr.ProviderConfig.Type + var schema *configschema.Block - providerSchema := schemas[providerAddr.ProviderConfig.Type] - if providerSchema == nil { - log.Printf("[ERROR] AttachSchemaTransformer: No schema available for %s because provider schema for %q is missing", addr, providerAddr.ProviderConfig.Type) - continue - } switch mode { case addrs.ManagedResourceMode: - schema = providerSchema.ResourceTypes[typeName] + schema = t.Schemas.ResourceTypeConfig(providerType, typeName) case addrs.DataResourceMode: - schema = providerSchema.DataSources[typeName] + schema = t.Schemas.DataSourceConfig(providerType, typeName) } - if schema != nil { - log.Printf("[TRACE] AttachSchemaTransformer: attaching schema to %s", dag.VertexName(v)) - tv.AttachResourceSchema(schema) - } else { - log.Printf("[ERROR] AttachSchemaTransformer: No schema available for %s", addr) - } - case GraphNodeAttachProviderConfigSchema: - providerAddr := tv.ProviderAddr() - providerSchema := schemas[providerAddr.ProviderConfig.Type] - if providerSchema == nil { - log.Printf("[ERROR] AttachSchemaTransformer: No schema available for %s because the whole provider schema is missing", providerAddr) + if schema == nil { + log.Printf("[ERROR] AttachSchemaTransformer: No resource schema available for %s", addr) continue } - - schema := schemas[providerAddr.ProviderConfig.Type].Provider - - if schema != nil { - log.Printf("[TRACE] AttachSchemaTransformer: attaching schema to %s", dag.VertexName(v)) - tv.AttachProviderConfigSchema(schema) - } else { + log.Printf("[TRACE] AttachSchemaTransformer: attaching resource schema to %s", dag.VertexName(v)) + tv.AttachResourceSchema(schema) + case GraphNodeAttachProviderConfigSchema: + providerAddr := tv.ProviderAddr() + schema := t.Schemas.ProviderConfig(providerAddr.ProviderConfig.Type) + if schema == nil { log.Printf("[ERROR] AttachSchemaTransformer: No schema available for %s", providerAddr) + continue } - } - } - - return nil -} - -func (t *AttachSchemaTransformer) attachProvisionerSchemas(g *Graph) error { - - // First we'll figure out which provisioners we need to fetch schemas for. - needProvisioners := make(map[string]struct{}) - for _, v := range g.Vertices() { - switch tv := v.(type) { - case GraphNodeAttachProvisionerSchema: - names := tv.ProvisionedBy() - log.Printf("[TRACE] AttachSchemaTransformer: %q (%T) provisioned by %s", dag.VertexName(v), v, names) - for _, name := range names { - needProvisioners[name] = struct{}{} - } - } - } - - // Now we'll fetch each one. This requires us to temporarily instantiate - // them, though this is not a full bootstrap since we don't yet have - // configuration information; the provisioners will be re-instantiated and - // properly configured during the graph walk. - schemas := make(map[string]*configschema.Block) - for name := range needProvisioners { - log.Printf("[TRACE] AttachSchemaTransformer: retrieving schema for provisioner %q", name) - provisioner, err := t.Components.ResourceProvisioner(name, "early/"+name) - if err != nil { - return fmt.Errorf("failed to instantiate provisioner %q to obtain schema: %s", name, err) - } - - schema, err := provisioner.GetConfigSchema() - if err != nil { - return fmt.Errorf("failed to retrieve schema from provisioner %q: %s", name, err) - } - schemas[name] = schema - - if closer, ok := provisioner.(ResourceProvisionerCloser); ok { - closer.Close() - } - } - - // Finally we'll once again visit all of the vertices and attach to - // them the schemas we found for them. - for _, v := range g.Vertices() { - switch tv := v.(type) { + log.Printf("[TRACE] AttachSchemaTransformer: attaching schema to %s", dag.VertexName(v)) + tv.AttachProviderConfigSchema(schema) case GraphNodeAttachProvisionerSchema: names := tv.ProvisionedBy() for _, name := range names { - schema := schemas[name] - if schema != nil { - log.Printf("[TRACE] AttachSchemaTransformer: attaching provisioner %q schema to %s", name, dag.VertexName(v)) - tv.AttachProvisionerSchema(name, schema) - } else { + schema := t.Schemas.ProvisionerConfig(name) + if schema == nil { log.Printf("[ERROR] AttachSchemaTransformer: No schema available for provisioner %q on %q", name, dag.VertexName(v)) + continue } + log.Printf("[TRACE] AttachSchemaTransformer: attaching provisioner %q schema to %s", name, dag.VertexName(v)) + tv.AttachProvisionerSchema(name, schema) } } } diff --git a/terraform/transform_destroy_cbd.go b/terraform/transform_destroy_cbd.go index 6134e5a55..3470e1804 100644 --- a/terraform/transform_destroy_cbd.go +++ b/terraform/transform_destroy_cbd.go @@ -41,10 +41,10 @@ type CBDEdgeTransformer struct { Config *configs.Config State *State - // If configuration is present then Components is required in order to - // obtain schema information from providers and provisioners in order - // to properly resolve implicit dependencies. - Components contextComponentFactory + // If configuration is present then Schemas is required in order to + // obtain schema information from providers and provisioners so we can + // properly resolve implicit dependencies. + Schemas *Schemas } func (t *CBDEdgeTransformer) Transform(g *Graph) error { @@ -178,7 +178,7 @@ func (t *CBDEdgeTransformer) depMap(destroyMap map[string][]dag.Vertex) (map[str &FlatConfigTransformer{Config: t.Config}, &AttachResourceConfigTransformer{Config: t.Config}, &AttachStateTransformer{State: t.State}, - &AttachSchemaTransformer{Components: t.Components}, + &AttachSchemaTransformer{Schemas: t.Schemas}, &ReferenceTransformer{}, }, Name: "CBDEdgeTransformer", diff --git a/terraform/transform_destroy_cbd_test.go b/terraform/transform_destroy_cbd_test.go index 6def3fff4..5a830fe1f 100644 --- a/terraform/transform_destroy_cbd_test.go +++ b/terraform/transform_destroy_cbd_test.go @@ -17,8 +17,8 @@ func TestCBDEdgeTransformer(t *testing.T) { { tf := &DestroyEdgeTransformer{ - Config: module, - Components: simpleMockComponentFactory(), + Config: module, + Schemas: simpleTestSchemas(), } if err := tf.Transform(&g); err != nil { t.Fatalf("err: %s", err) @@ -27,8 +27,8 @@ func TestCBDEdgeTransformer(t *testing.T) { { tf := &CBDEdgeTransformer{ - Config: module, - Components: simpleMockComponentFactory(), + Config: module, + Schemas: simpleTestSchemas(), } if err := tf.Transform(&g); err != nil { t.Fatalf("err: %s", err) @@ -53,8 +53,8 @@ func TestCBDEdgeTransformer_depNonCBD(t *testing.T) { { tf := &DestroyEdgeTransformer{ - Config: module, - Components: simpleMockComponentFactory(), + Config: module, + Schemas: simpleTestSchemas(), } if err := tf.Transform(&g); err != nil { t.Fatalf("err: %s", err) @@ -63,8 +63,8 @@ func TestCBDEdgeTransformer_depNonCBD(t *testing.T) { { tf := &CBDEdgeTransformer{ - Config: module, - Components: simpleMockComponentFactory(), + Config: module, + Schemas: simpleTestSchemas(), } if err := tf.Transform(&g); err != nil { t.Fatalf("err: %s", err) @@ -89,8 +89,8 @@ func TestCBDEdgeTransformer_depNonCBDCount(t *testing.T) { { tf := &DestroyEdgeTransformer{ - Config: module, - Components: simpleMockComponentFactory(), + Config: module, + Schemas: simpleTestSchemas(), } if err := tf.Transform(&g); err != nil { t.Fatalf("err: %s", err) @@ -99,8 +99,8 @@ func TestCBDEdgeTransformer_depNonCBDCount(t *testing.T) { { tf := &CBDEdgeTransformer{ - Config: module, - Components: simpleMockComponentFactory(), + Config: module, + Schemas: simpleTestSchemas(), } if err := tf.Transform(&g); err != nil { t.Fatalf("err: %s", err) @@ -135,8 +135,8 @@ func TestCBDEdgeTransformer_depNonCBDCountBoth(t *testing.T) { { tf := &DestroyEdgeTransformer{ - Config: module, - Components: simpleMockComponentFactory(), + Config: module, + Schemas: simpleTestSchemas(), } if err := tf.Transform(&g); err != nil { t.Fatalf("err: %s", err) @@ -145,8 +145,8 @@ func TestCBDEdgeTransformer_depNonCBDCountBoth(t *testing.T) { { tf := &CBDEdgeTransformer{ - Config: module, - Components: simpleMockComponentFactory(), + Config: module, + Schemas: simpleTestSchemas(), } if err := tf.Transform(&g); err != nil { t.Fatalf("err: %s", err) diff --git a/terraform/transform_destroy_edge.go b/terraform/transform_destroy_edge.go index 29f3f98a6..fdbf5081a 100644 --- a/terraform/transform_destroy_edge.go +++ b/terraform/transform_destroy_edge.go @@ -45,10 +45,10 @@ type DestroyEdgeTransformer struct { Config *configs.Config State *State - // If configuration is present then Components is required in order to + // If configuration is present then Schemas is required in order to // obtain schema information from providers and provisioners in order // to properly resolve implicit dependencies. - Components contextComponentFactory + Schemas *Schemas } func (t *DestroyEdgeTransformer) Transform(g *Graph) error { @@ -141,7 +141,7 @@ func (t *DestroyEdgeTransformer) Transform(g *Graph) error { // Must be before ReferenceTransformer, since schema is required to // extract references from config. - &AttachSchemaTransformer{Components: t.Components}, + &AttachSchemaTransformer{Schemas: t.Schemas}, TransformProviders(nil, providerFn, t.Config), diff --git a/terraform/transform_destroy_edge_test.go b/terraform/transform_destroy_edge_test.go index b3debd366..01059db02 100644 --- a/terraform/transform_destroy_edge_test.go +++ b/terraform/transform_destroy_edge_test.go @@ -12,8 +12,8 @@ func TestDestroyEdgeTransformer_basic(t *testing.T) { g.Add(&graphNodeDestroyerTest{AddrString: "test_object.A"}) g.Add(&graphNodeDestroyerTest{AddrString: "test_object.B"}) tf := &DestroyEdgeTransformer{ - Config: testModule(t, "transform-destroy-edge-basic"), - Components: simpleMockComponentFactory(), + Config: testModule(t, "transform-destroy-edge-basic"), + Schemas: simpleTestSchemas(), } if err := tf.Transform(&g); err != nil { t.Fatalf("err: %s", err) @@ -32,8 +32,8 @@ func TestDestroyEdgeTransformer_create(t *testing.T) { g.Add(&graphNodeDestroyerTest{AddrString: "test_object.B"}) g.Add(&graphNodeCreatorTest{AddrString: "test_object.A"}) tf := &DestroyEdgeTransformer{ - Config: testModule(t, "transform-destroy-edge-basic"), - Components: simpleMockComponentFactory(), + Config: testModule(t, "transform-destroy-edge-basic"), + Schemas: simpleTestSchemas(), } if err := tf.Transform(&g); err != nil { t.Fatalf("err: %s", err) @@ -52,8 +52,8 @@ func TestDestroyEdgeTransformer_multi(t *testing.T) { g.Add(&graphNodeDestroyerTest{AddrString: "test_object.B"}) g.Add(&graphNodeDestroyerTest{AddrString: "test_object.C"}) tf := &DestroyEdgeTransformer{ - Config: testModule(t, "transform-destroy-edge-multi"), - Components: simpleMockComponentFactory(), + Config: testModule(t, "transform-destroy-edge-multi"), + Schemas: simpleTestSchemas(), } if err := tf.Transform(&g); err != nil { t.Fatalf("err: %s", err) @@ -70,8 +70,8 @@ func TestDestroyEdgeTransformer_selfRef(t *testing.T) { g := Graph{Path: addrs.RootModuleInstance} g.Add(&graphNodeDestroyerTest{AddrString: "test_object.A"}) tf := &DestroyEdgeTransformer{ - Config: testModule(t, "transform-destroy-edge-self-ref"), - Components: simpleMockComponentFactory(), + Config: testModule(t, "transform-destroy-edge-self-ref"), + Schemas: simpleTestSchemas(), } if err := tf.Transform(&g); err != nil { t.Fatalf("err: %s", err) @@ -89,8 +89,8 @@ func TestDestroyEdgeTransformer_module(t *testing.T) { g.Add(&graphNodeDestroyerTest{AddrString: "module.child.test_object.b"}) g.Add(&graphNodeDestroyerTest{AddrString: "test_object.a"}) tf := &DestroyEdgeTransformer{ - Config: testModule(t, "transform-destroy-edge-module"), - Components: simpleMockComponentFactory(), + Config: testModule(t, "transform-destroy-edge-module"), + Schemas: simpleTestSchemas(), } if err := tf.Transform(&g); err != nil { t.Fatalf("err: %s", err) @@ -109,8 +109,8 @@ func TestDestroyEdgeTransformer_moduleOnly(t *testing.T) { g.Add(&graphNodeDestroyerTest{AddrString: "module.child.test_object.b"}) g.Add(&graphNodeDestroyerTest{AddrString: "module.child.test_object.c"}) tf := &DestroyEdgeTransformer{ - Config: testModule(t, "transform-destroy-edge-module-only"), - Components: simpleMockComponentFactory(), + Config: testModule(t, "transform-destroy-edge-module-only"), + Schemas: simpleTestSchemas(), } if err := tf.Transform(&g); err != nil { t.Fatalf("err: %s", err) diff --git a/terraform/transform_transitive_reduction_test.go b/terraform/transform_transitive_reduction_test.go index 3d136e96e..147101657 100644 --- a/terraform/transform_transitive_reduction_test.go +++ b/terraform/transform_transitive_reduction_test.go @@ -30,21 +30,26 @@ func TestTransitiveReductionTransformer(t *testing.T) { { transform := &AttachSchemaTransformer{ - Components: testProviderComponentFactory( - "aws", - mockProviderWithResourceTypeSchema("aws_instance", &configschema.Block{ - Attributes: map[string]*configschema.Attribute{ - "A": { - Type: cty.String, - Optional: true, - }, - "B": { - Type: cty.String, - Optional: true, + Schemas: &Schemas{ + providers: map[string]*ProviderSchema{ + "aws": { + ResourceTypes: map[string]*configschema.Block{ + "aws_instance": &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "A": { + Type: cty.String, + Optional: true, + }, + "B": { + Type: cty.String, + Optional: true, + }, + }, + }, }, }, - }), - ), + }, + }, } if err := transform.Transform(&g); err != nil { t.Fatalf("err: %s", err)