From 73053eb5ef50f2e3f31d86a1c4c9555c2f1e7efd Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 3 May 2018 20:41:46 -0700 Subject: [PATCH] core: Context.Eval method Some of the objects that are referencable from expressions are transient values computed only during a graph walk, and not persisted in state. In order to support arbitrary evaluation of expressions, such as in the "terraform console" CLI command, it's necessary to be able to evaluate these values before we start evaluating. This new Eval method achieves this by performing a special graph walk that ignores resources (except for dependency resolution) and just focuses on evaluating all of these transient values, before returning an evaluation scope that can then resolve expressions in terms of that result. This replaces the Context.Interpolator method, which was fraught with various issues due to it not properly priming the state before evaluating. --- terraform/context.go | 95 +++++++++++++++++++------ terraform/context_graph_type.go | 2 + terraform/eval_context.go | 5 ++ terraform/eval_context_builtin.go | 12 ++-- terraform/eval_context_mock.go | 13 ++++ terraform/graph_builder_eval.go | 104 ++++++++++++++++++++++++++++ terraform/graph_walk_operation.go | 1 + terraform/graphtype_string.go | 4 +- terraform/node_provider_eval.go | 20 ++++++ terraform/transform_provider.go | 10 ++- terraform/valuesourcetype_string.go | 41 +++++++++++ terraform/walkoperation_string.go | 4 +- 12 files changed, 279 insertions(+), 32 deletions(-) create mode 100644 terraform/graph_builder_eval.go create mode 100644 terraform/node_provider_eval.go create mode 100644 terraform/valuesourcetype_string.go diff --git a/terraform/context.go b/terraform/context.go index 91bc7a704..afe1c2e11 100644 --- a/terraform/context.go +++ b/terraform/context.go @@ -8,6 +8,8 @@ import ( "strings" "sync" + "github.com/hashicorp/terraform/lang" + "github.com/hashicorp/terraform/addrs" "github.com/hashicorp/terraform/configs" @@ -302,6 +304,13 @@ func (c *Context) Graph(typ GraphType, opts *ContextGraphOpts) (*Graph, tfdiags. Validate: opts.Validate, }).Build(addrs.RootModuleInstance) + case GraphTypeEval: + return (&EvalGraphBuilder{ + Config: c.config, + State: c.state, + Components: c.components, + }).Build(addrs.RootModuleInstance) + default: // Should never happen, because the above is exhaustive for all graph types. panic(fmt.Errorf("unsupported graph type %s", typ)) @@ -342,17 +351,63 @@ func (c *Context) State() *State { return c.state.DeepCopy() } -// Evaluator returns an Evaluator that references this context's state, and -// that can be used to obtain data for expression evaluation within the -// receiving context. -func (c *Context) Evaluator() *Evaluator { - return &Evaluator{ - Operation: walkApply, - Meta: c.meta, - Config: c.config, - State: c.state, - StateLock: &c.stateLock, +// Eval produces a scope in which expressions can be evaluated for +// the given module path. +// +// This method must first evaluate any ephemeral values (input variables, local +// values, and output values) in the configuration. These ephemeral values are +// not included in the persisted state, so they must be re-computed using other +// values in the state before they can be properly evaluated. The updated +// values are retained in the main state associated with the receiving context. +// +// This function takes no action against remote APIs but it does need access +// to all provider and provisioner instances in order to obtain their schemas +// for type checking. +// +// The result is an evaluation scope that can be used to resolve references +// against the root module. If the returned diagnostics contains errors then +// the returned scope may be nil. If it is not nil then it may still be used +// to attempt expression evaluation or other analysis, but some expressions +// may not behave as expected. +func (c *Context) Eval(path addrs.ModuleInstance) (*lang.Scope, tfdiags.Diagnostics) { + // This is intended for external callers such as the "terraform console" + // command. Internally, we create an evaluator in c.walk before walking + // the graph, and create scopes in ContextGraphWalker. + + var diags tfdiags.Diagnostics + defer c.acquireRun("eval")() + + // Start with a copy of state so that we don't affect any instances + // that other methods may have already returned. + c.state = c.state.DeepCopy() + var walker *ContextGraphWalker + + graph, graphDiags := c.Graph(GraphTypeEval, nil) + diags = diags.Append(graphDiags) + if !diags.HasErrors() { + var walkDiags tfdiags.Diagnostics + walker, walkDiags = c.walk(graph, walkEval) + diags = diags.Append(walker.NonFatalDiagnostics) + diags = diags.Append(walkDiags) + + // Clean out any unused things + c.state.prune() } + + if walker == nil { + // If we skipped walking the graph (due to errors) then we'll just + // use a placeholder graph walker here, which'll refer to the + // unmodified state. + walker = c.graphWalker(walkEval) + } + + // This is a bit weird since we don't normally evaluate outside of + // the context of a walk, but we'll "re-enter" our desired path here + // just to get hold of an EvalContext for it. GraphContextBuiltin + // caches its contexts, so we should get hold of the context that was + // previously used for evaluation here, unless we skipped walking. + evalCtx := walker.EnterPath(path) + return evalCtx.EvaluationScope(nil, addrs.NoKey), diags } // Interpolater is no longer used. Use Evaluator instead. @@ -778,18 +833,9 @@ func (c *Context) releaseRun() { } func (c *Context) walk(graph *Graph, operation walkOperation) (*ContextGraphWalker, tfdiags.Diagnostics) { - // Keep track of the "real" context which is the context that does - // the real work: talking to real providers, modifying real state, etc. - realCtx := c - log.Printf("[DEBUG] Starting graph walk: %s", operation.String()) - walker := &ContextGraphWalker{ - Context: realCtx, - Operation: operation, - StopContext: c.runContext, - RootVariableValues: c.variables, - } + walker := c.graphWalker(operation) // Watch for a stop so we can call the provider Stop() API. watchStop, watchWait := c.watchStop(walker) @@ -804,6 +850,15 @@ func (c *Context) walk(graph *Graph, operation walkOperation) (*ContextGraphWalk return walker, diags } +func (c *Context) graphWalker(operation walkOperation) *ContextGraphWalker { + return &ContextGraphWalker{ + Context: c, + Operation: operation, + StopContext: c.runContext, + RootVariableValues: c.variables, + } +} + // watchStop immediately returns a `stop` and a `wait` chan after dispatching // the watchStop goroutine. This will watch the runContext for cancellation and // stop the providers accordingly. When the watch is no longer needed, the diff --git a/terraform/context_graph_type.go b/terraform/context_graph_type.go index 084f0105d..97eefca35 100644 --- a/terraform/context_graph_type.go +++ b/terraform/context_graph_type.go @@ -16,6 +16,7 @@ const ( GraphTypeApply GraphTypeInput GraphTypeValidate + GraphTypeEval // only visits in-memory elements such as variables, locals, and outputs. ) // GraphTypeMap is a mapping of human-readable string to GraphType. This @@ -29,4 +30,5 @@ var GraphTypeMap = map[string]GraphType{ "refresh": GraphTypeRefresh, "legacy": GraphTypeLegacy, "validate": GraphTypeValidate, + "eval": GraphTypeEval, } diff --git a/terraform/eval_context.go b/terraform/eval_context.go index 48e178dcd..8f0a10e2f 100644 --- a/terraform/eval_context.go +++ b/terraform/eval_context.go @@ -6,6 +6,7 @@ import ( "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/terraform/addrs" "github.com/hashicorp/terraform/config/configschema" + "github.com/hashicorp/terraform/lang" "github.com/hashicorp/terraform/tfdiags" "github.com/zclconf/go-cty/cty" ) @@ -109,6 +110,10 @@ type EvalContext interface { // evaluating. Set this to nil if the "self" object should not be available. EvaluateExpr(expr hcl.Expression, wantType cty.Type, self addrs.Referenceable) (cty.Value, tfdiags.Diagnostics) + // EvaluationScope returns a scope that can be used to evaluate reference + // addresses in this context. + EvaluationScope(self addrs.Referenceable, key addrs.InstanceKey) *lang.Scope + // SetModuleCallArguments defines values for the variables of a particular // child module call. // diff --git a/terraform/eval_context_builtin.go b/terraform/eval_context_builtin.go index 1bffc0d1c..f7a6829af 100644 --- a/terraform/eval_context_builtin.go +++ b/terraform/eval_context_builtin.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/terraform/config/configschema" + "github.com/hashicorp/terraform/lang" "github.com/hashicorp/terraform/tfdiags" "github.com/hashicorp/terraform/addrs" @@ -300,8 +301,7 @@ func (ctx *BuiltinEvalContext) CloseProvisioner(n string) error { func (ctx *BuiltinEvalContext) EvaluateBlock(body hcl.Body, schema *configschema.Block, self addrs.Referenceable, key addrs.InstanceKey) (cty.Value, hcl.Body, tfdiags.Diagnostics) { var diags tfdiags.Diagnostics - evalData := ctx.evaluationStateData(key) - scope := ctx.Evaluator.Scope(evalData, self) + scope := ctx.EvaluationScope(self, key) body, evalDiags := scope.ExpandBlock(body, schema) diags = diags.Append(evalDiags) val, evalDiags := scope.EvalBlock(body, schema) @@ -310,17 +310,17 @@ func (ctx *BuiltinEvalContext) EvaluateBlock(body hcl.Body, schema *configschema } func (ctx *BuiltinEvalContext) EvaluateExpr(expr hcl.Expression, wantType cty.Type, self addrs.Referenceable) (cty.Value, tfdiags.Diagnostics) { - evalData := ctx.evaluationStateData(addrs.NoKey) - scope := ctx.Evaluator.Scope(evalData, self) + scope := ctx.EvaluationScope(self, addrs.NoKey) return scope.EvalExpr(expr, wantType) } -func (ctx *BuiltinEvalContext) evaluationStateData(key addrs.InstanceKey) *evaluationStateData { - return &evaluationStateData{ +func (ctx *BuiltinEvalContext) EvaluationScope(self addrs.Referenceable, key addrs.InstanceKey) *lang.Scope { + data := &evaluationStateData{ Evaluator: ctx.Evaluator, ModulePath: ctx.PathValue, InstanceKey: key, } + return ctx.Evaluator.Scope(data, self) } func (ctx *BuiltinEvalContext) Path() addrs.ModuleInstance { diff --git a/terraform/eval_context_mock.go b/terraform/eval_context_mock.go index 4df1801b8..2d52243d7 100644 --- a/terraform/eval_context_mock.go +++ b/terraform/eval_context_mock.go @@ -4,6 +4,7 @@ import ( "sync" "github.com/hashicorp/terraform/addrs" + "github.com/hashicorp/terraform/lang" "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/terraform/config/configschema" @@ -90,6 +91,11 @@ type MockEvalContext struct { EvaluateExprResult cty.Value EvaluateExprDiags tfdiags.Diagnostics + EvaluationScopeCalled bool + EvaluationScopeSelf addrs.Referenceable + EvaluationScopeKey addrs.InstanceKey + EvaluationScopeScope *lang.Scope + InterpolateCalled bool InterpolateConfig *config.RawConfig InterpolateResource *Resource @@ -227,6 +233,13 @@ func (c *MockEvalContext) EvaluateExpr(expr hcl.Expression, wantType cty.Type, s return c.EvaluateExprResult, c.EvaluateExprDiags } +func (c *MockEvalContext) EvaluationScope(self addrs.Referenceable, key addrs.InstanceKey) *lang.Scope { + c.EvaluationScopeCalled = true + c.EvaluationScopeSelf = self + c.EvaluationScopeKey = key + return c.EvaluationScopeScope +} + func (c *MockEvalContext) Interpolate( config *config.RawConfig, resource *Resource) (*ResourceConfig, error) { c.InterpolateCalled = true diff --git a/terraform/graph_builder_eval.go b/terraform/graph_builder_eval.go new file mode 100644 index 000000000..de25ff237 --- /dev/null +++ b/terraform/graph_builder_eval.go @@ -0,0 +1,104 @@ +package terraform + +import ( + "github.com/hashicorp/terraform/dag" + "github.com/hashicorp/terraform/tfdiags" + + "github.com/hashicorp/terraform/addrs" + "github.com/hashicorp/terraform/configs" +) + +// EvalGraphBuilder implements GraphBuilder and constructs a graph suitable +// for evaluating in-memory values (input variables, local values, output +// values) in the state without any other side-effects. +// +// This graph is used only in weird cases, such as the "terraform console" +// CLI command, where we need to evaluate expressions against the state +// without taking any other actions. +// +// The generated graph will include nodes for providers, resources, etc +// just to allow indirect dependencies to be resolved, but these nodes will +// not take any actions themselves since we assume that their parts of the +// state, if any, are already complete. +// +// Although the providers are never configured, they must still be available +// in order to obtain schema information used for type checking, etc. +type EvalGraphBuilder struct { + // Config is the configuration tree. + Config *configs.Config + + // State is the current state + State *State + + // Components is a factory for the plug-in components (providers and + // provisioners) available for use. + Components contextComponentFactory +} + +// See GraphBuilder +func (b *EvalGraphBuilder) Build(path addrs.ModuleInstance) (*Graph, tfdiags.Diagnostics) { + return (&BasicGraphBuilder{ + Steps: b.Steps(), + Validate: true, + Name: "EvalGraphBuilder", + }).Build(path) +} + +// See GraphBuilder +func (b *EvalGraphBuilder) Steps() []GraphTransformer { + concreteProvider := func(a *NodeAbstractProvider) dag.Vertex { + return &NodeEvalableProvider{ + NodeAbstractProvider: a, + } + } + + steps := []GraphTransformer{ + // Creates all the data resources that aren't in the state. This will also + // add any orphans from scaling in as destroy nodes. + &ConfigTransformer{ + Concrete: nil, // just use the abstract type + Config: b.Config, + Unique: true, + }, + + // Attach the state + &AttachStateTransformer{State: b.State}, + + // Attach the configuration to any resources + &AttachResourceConfigTransformer{Config: b.Config}, + + // Add root variables + &RootVariableTransformer{Config: b.Config}, + + TransformProviders(b.Components.ResourceProviders(), concreteProvider, b.Config), + + // Add the local values + &LocalTransformer{Config: b.Config}, + + // Add the outputs + &OutputTransformer{Config: b.Config}, + + // Add module variables + &ModuleVariableTransformer{Config: b.Config}, + + // Must be before ReferenceTransformer, since schema is required to + // extract references from config. + &AttachSchemaTransformer{Components: b.Components}, + + // Connect so that the references are ready for targeting. We'll + // have to connect again later for providers and so on. + &ReferenceTransformer{}, + + // Although we don't configure providers, we do still start them up + // to get their schemas, and so we must shut them down again here. + &CloseProviderTransformer{}, + + // Single root + &RootTransformer{}, + + // Remove redundant edges to simplify the graph. + &TransitiveReductionTransformer{}, + } + + return steps +} diff --git a/terraform/graph_walk_operation.go b/terraform/graph_walk_operation.go index 3fb374819..d2605d8c7 100644 --- a/terraform/graph_walk_operation.go +++ b/terraform/graph_walk_operation.go @@ -15,4 +15,5 @@ const ( walkValidate walkDestroy walkImport + walkEval // used just to prepare EvalContext for expression evaluation, with no other actions ) diff --git a/terraform/graphtype_string.go b/terraform/graphtype_string.go index 95ef4e94d..08009b9ad 100644 --- a/terraform/graphtype_string.go +++ b/terraform/graphtype_string.go @@ -4,9 +4,9 @@ package terraform import "strconv" -const _GraphType_name = "GraphTypeInvalidGraphTypeLegacyGraphTypeRefreshGraphTypePlanGraphTypePlanDestroyGraphTypeApplyGraphTypeInputGraphTypeValidate" +const _GraphType_name = "GraphTypeInvalidGraphTypeLegacyGraphTypeRefreshGraphTypePlanGraphTypePlanDestroyGraphTypeApplyGraphTypeInputGraphTypeValidateGraphTypeEval" -var _GraphType_index = [...]uint8{0, 16, 31, 47, 60, 80, 94, 108, 125} +var _GraphType_index = [...]uint8{0, 16, 31, 47, 60, 80, 94, 108, 125, 138} func (i GraphType) String() string { if i >= GraphType(len(_GraphType_index)-1) { diff --git a/terraform/node_provider_eval.go b/terraform/node_provider_eval.go new file mode 100644 index 000000000..580e60cb7 --- /dev/null +++ b/terraform/node_provider_eval.go @@ -0,0 +1,20 @@ +package terraform + +// NodeEvalableProvider represents a provider during an "eval" walk. +// This special provider node type just initializes a provider and +// fetches its schema, without configuring it or otherwise interacting +// with it. +type NodeEvalableProvider struct { + *NodeAbstractProvider +} + +// GraphNodeEvalable +func (n *NodeEvalableProvider) EvalTree() EvalNode { + addr := n.Addr + relAddr := addr.ProviderConfig + + return &EvalInitProvider{ + TypeName: relAddr.Type, + Addr: addr.ProviderConfig, + } +} diff --git a/terraform/transform_provider.go b/terraform/transform_provider.go index 97fd38ac5..6816d4ab1 100644 --- a/terraform/transform_provider.go +++ b/terraform/transform_provider.go @@ -478,9 +478,15 @@ func (t *ProviderConfigTransformer) transformSingle(g *Graph, c *configs.Config) relAddr := p.Addr() addr := relAddr.Absolute(path) - v := t.Concrete(&NodeAbstractProvider{ + abstract := &NodeAbstractProvider{ Addr: addr, - }) + } + var v dag.Vertex + if t.Concrete != nil { + v = t.Concrete(abstract) + } else { + v = abstract + } // Add it to the graph g.Add(v) diff --git a/terraform/valuesourcetype_string.go b/terraform/valuesourcetype_string.go new file mode 100644 index 000000000..e3218b414 --- /dev/null +++ b/terraform/valuesourcetype_string.go @@ -0,0 +1,41 @@ +// Code generated by "stringer -type ValueSourceType"; DO NOT EDIT. + +package terraform + +import "strconv" + +const ( + _ValueSourceType_name_0 = "ValueFromUnknown" + _ValueSourceType_name_1 = "ValueFromCLIArg" + _ValueSourceType_name_2 = "ValueFromConfig" + _ValueSourceType_name_3 = "ValueFromEnvVarValueFromFile" + _ValueSourceType_name_4 = "ValueFromInput" + _ValueSourceType_name_5 = "ValueFromPlan" + _ValueSourceType_name_6 = "ValueFromCaller" +) + +var ( + _ValueSourceType_index_3 = [...]uint8{0, 15, 28} +) + +func (i ValueSourceType) String() string { + switch { + case i == 0: + return _ValueSourceType_name_0 + case i == 65: + return _ValueSourceType_name_1 + case i == 67: + return _ValueSourceType_name_2 + case 69 <= i && i <= 70: + i -= 69 + return _ValueSourceType_name_3[_ValueSourceType_index_3[i]:_ValueSourceType_index_3[i+1]] + case i == 73: + return _ValueSourceType_name_4 + case i == 80: + return _ValueSourceType_name_5 + case i == 83: + return _ValueSourceType_name_6 + default: + return "ValueSourceType(" + strconv.FormatInt(int64(i), 10) + ")" + } +} diff --git a/terraform/walkoperation_string.go b/terraform/walkoperation_string.go index 4cfc528ef..37084c8ae 100644 --- a/terraform/walkoperation_string.go +++ b/terraform/walkoperation_string.go @@ -4,9 +4,9 @@ package terraform import "strconv" -const _walkOperation_name = "walkInvalidwalkInputwalkApplywalkPlanwalkPlanDestroywalkRefreshwalkValidatewalkDestroywalkImport" +const _walkOperation_name = "walkInvalidwalkInputwalkApplywalkPlanwalkPlanDestroywalkRefreshwalkValidatewalkDestroywalkImportwalkEval" -var _walkOperation_index = [...]uint8{0, 11, 20, 29, 37, 52, 63, 75, 86, 96} +var _walkOperation_index = [...]uint8{0, 11, 20, 29, 37, 52, 63, 75, 86, 96, 104} func (i walkOperation) String() string { if i >= walkOperation(len(_walkOperation_index)-1) {