core: Context.Eval method

Some of the objects that are referencable from expressions are transient
values computed only during a graph walk, and not persisted in state. In
order to support arbitrary evaluation of expressions, such as in the
"terraform console" CLI command, it's necessary to be able to evaluate
these values before we start evaluating.

This new Eval method achieves this by performing a special graph walk that
ignores resources (except for dependency resolution) and just focuses on
evaluating all of these transient values, before returning an evaluation
scope that can then resolve expressions in terms of that result.

This replaces the Context.Interpolator method, which was fraught with
various issues due to it not properly priming the state before evaluating.

terraform/context.go

@@ -8,6 +8,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/hashicorp/terraform/lang"
+
 	"github.com/hashicorp/terraform/addrs"
 
 	"github.com/hashicorp/terraform/configs"
@@ -302,6 +304,13 @@ func (c *Context) Graph(typ GraphType, opts *ContextGraphOpts) (*Graph, tfdiags.
 			Validate:   opts.Validate,
 		}).Build(addrs.RootModuleInstance)
 
+	case GraphTypeEval:
+		return (&EvalGraphBuilder{
+			Config:     c.config,
+			State:      c.state,
+			Components: c.components,
+		}).Build(addrs.RootModuleInstance)
+
 	default:
 		// Should never happen, because the above is exhaustive for all graph types.
 		panic(fmt.Errorf("unsupported graph type %s", typ))
@@ -342,17 +351,63 @@ func (c *Context) State() *State {
 	return c.state.DeepCopy()
 }
 
-// Evaluator returns an Evaluator that references this context's state, and
-// that can be used to obtain data for expression evaluation within the
-// receiving context.
-func (c *Context) Evaluator() *Evaluator {
-	return &Evaluator{
-		Operation: walkApply,
-		Meta:      c.meta,
-		Config:    c.config,
-		State:     c.state,
-		StateLock: &c.stateLock,
+// Eval produces a scope in which expressions can be evaluated for
+// the given module path.
+//
+// This method must first evaluate any ephemeral values (input variables, local
+// values, and output values) in the configuration. These ephemeral values are
+// not included in the persisted state, so they must be re-computed using other
+// values in the state before they can be properly evaluated. The updated
+// values are retained in the main state associated with the receiving context.
+//
+// This function takes no action against remote APIs but it does need access
+// to all provider and provisioner instances in order to obtain their schemas
+// for type checking.
+//
+// The result is an evaluation scope that can be used to resolve references
+// against the root module. If the returned diagnostics contains errors then
+// the returned scope may be nil. If it is not nil then it may still be used
+// to attempt expression evaluation or other analysis, but some expressions
+// may not behave as expected.
+func (c *Context) Eval(path addrs.ModuleInstance) (*lang.Scope, tfdiags.Diagnostics) {
+	// This is intended for external callers such as the "terraform console"
+	// command. Internally, we create an evaluator in c.walk before walking
+	// the graph, and create scopes in ContextGraphWalker.
+
+	var diags tfdiags.Diagnostics
+	defer c.acquireRun("eval")()
+
+	// Start with a copy of state so that we don't affect any instances
+	// that other methods may have already returned.
+	c.state = c.state.DeepCopy()
+	var walker *ContextGraphWalker
+
+	graph, graphDiags := c.Graph(GraphTypeEval, nil)
+	diags = diags.Append(graphDiags)
+	if !diags.HasErrors() {
+		var walkDiags tfdiags.Diagnostics
+		walker, walkDiags = c.walk(graph, walkEval)
+		diags = diags.Append(walker.NonFatalDiagnostics)
+		diags = diags.Append(walkDiags)
+
+		// Clean out any unused things
+		c.state.prune()
 	}
+
+	if walker == nil {
+		// If we skipped walking the graph (due to errors) then we'll just
+		// use a placeholder graph walker here, which'll refer to the
+		// unmodified state.
+		walker = c.graphWalker(walkEval)
+	}
+
+	// This is a bit weird since we don't normally evaluate outside of
+	// the context of a walk, but we'll "re-enter" our desired path here
+	// just to get hold of an EvalContext for it. GraphContextBuiltin
+	// caches its contexts, so we should get hold of the context that was
+	// previously used for evaluation here, unless we skipped walking.
+	evalCtx := walker.EnterPath(path)
+	return evalCtx.EvaluationScope(nil, addrs.NoKey), diags
 }
 
 // Interpolater is no longer used. Use Evaluator instead.
@@ -778,18 +833,9 @@ func (c *Context) releaseRun() {
 }
 
 func (c *Context) walk(graph *Graph, operation walkOperation) (*ContextGraphWalker, tfdiags.Diagnostics) {
-	// Keep track of the "real" context which is the context that does
-	// the real work: talking to real providers, modifying real state, etc.
-	realCtx := c
-
 	log.Printf("[DEBUG] Starting graph walk: %s", operation.String())
 
-	walker := &ContextGraphWalker{
-		Context:            realCtx,
-		Operation:          operation,
-		StopContext:        c.runContext,
-		RootVariableValues: c.variables,
-	}
+	walker := c.graphWalker(operation)
 
 	// Watch for a stop so we can call the provider Stop() API.
 	watchStop, watchWait := c.watchStop(walker)
@@ -804,6 +850,15 @@ func (c *Context) walk(graph *Graph, operation walkOperation) (*ContextGraphWalk
 	return walker, diags
 }
 
+func (c *Context) graphWalker(operation walkOperation) *ContextGraphWalker {
+	return &ContextGraphWalker{
+		Context:            c,
+		Operation:          operation,
+		StopContext:        c.runContext,
+		RootVariableValues: c.variables,
+	}
+}
+
 // watchStop immediately returns a `stop` and a `wait` chan after dispatching
 // the watchStop goroutine. This will watch the runContext for cancellation and
 // stop the providers accordingly.  When the watch is no longer needed, the

terraform/context_graph_type.go

@@ -16,6 +16,7 @@ const (
 	GraphTypeApply
 	GraphTypeInput
 	GraphTypeValidate
+	GraphTypeEval // only visits in-memory elements such as variables, locals, and outputs.
 )
 
 // GraphTypeMap is a mapping of human-readable string to GraphType. This
@@ -29,4 +30,5 @@ var GraphTypeMap = map[string]GraphType{
 	"refresh":      GraphTypeRefresh,
 	"legacy":       GraphTypeLegacy,
 	"validate":     GraphTypeValidate,
+	"eval":         GraphTypeEval,
 }

terraform/eval_context.go

@@ -6,6 +6,7 @@ import (
 	"github.com/hashicorp/hcl2/hcl"
 	"github.com/hashicorp/terraform/addrs"
 	"github.com/hashicorp/terraform/config/configschema"
+	"github.com/hashicorp/terraform/lang"
 	"github.com/hashicorp/terraform/tfdiags"
 	"github.com/zclconf/go-cty/cty"
 )
@@ -109,6 +110,10 @@ type EvalContext interface {
 	// evaluating. Set this to nil if the "self" object should not be available.
 	EvaluateExpr(expr hcl.Expression, wantType cty.Type, self addrs.Referenceable) (cty.Value, tfdiags.Diagnostics)
 
+	// EvaluationScope returns a scope that can be used to evaluate reference
+	// addresses in this context.
+	EvaluationScope(self addrs.Referenceable, key addrs.InstanceKey) *lang.Scope
+
 	// SetModuleCallArguments defines values for the variables of a particular
 	// child module call.
 	//

terraform/eval_context_builtin.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/hashicorp/hcl2/hcl"
 	"github.com/hashicorp/terraform/config/configschema"
+	"github.com/hashicorp/terraform/lang"
 	"github.com/hashicorp/terraform/tfdiags"
 
 	"github.com/hashicorp/terraform/addrs"
@@ -300,8 +301,7 @@ func (ctx *BuiltinEvalContext) CloseProvisioner(n string) error {
 
 func (ctx *BuiltinEvalContext) EvaluateBlock(body hcl.Body, schema *configschema.Block, self addrs.Referenceable, key addrs.InstanceKey) (cty.Value, hcl.Body, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
-	evalData := ctx.evaluationStateData(key)
-	scope := ctx.Evaluator.Scope(evalData, self)
+	scope := ctx.EvaluationScope(self, key)
 	body, evalDiags := scope.ExpandBlock(body, schema)
 	diags = diags.Append(evalDiags)
 	val, evalDiags := scope.EvalBlock(body, schema)
@@ -310,17 +310,17 @@ func (ctx *BuiltinEvalContext) EvaluateBlock(body hcl.Body, schema *configschema
 }
 
 func (ctx *BuiltinEvalContext) EvaluateExpr(expr hcl.Expression, wantType cty.Type, self addrs.Referenceable) (cty.Value, tfdiags.Diagnostics) {
-	evalData := ctx.evaluationStateData(addrs.NoKey)
-	scope := ctx.Evaluator.Scope(evalData, self)
+	scope := ctx.EvaluationScope(self, addrs.NoKey)
 	return scope.EvalExpr(expr, wantType)
 }
 
-func (ctx *BuiltinEvalContext) evaluationStateData(key addrs.InstanceKey) *evaluationStateData {
-	return &evaluationStateData{
+func (ctx *BuiltinEvalContext) EvaluationScope(self addrs.Referenceable, key addrs.InstanceKey) *lang.Scope {
+	data := &evaluationStateData{
 		Evaluator:   ctx.Evaluator,
 		ModulePath:  ctx.PathValue,
 		InstanceKey: key,
 	}
+	return ctx.Evaluator.Scope(data, self)
 }
 
 func (ctx *BuiltinEvalContext) Path() addrs.ModuleInstance {

terraform/eval_context_mock.go

@@ -4,6 +4,7 @@ import (
 	"sync"
 
 	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/lang"
 
 	"github.com/hashicorp/hcl2/hcl"
 	"github.com/hashicorp/terraform/config/configschema"
@@ -90,6 +91,11 @@ type MockEvalContext struct {
 	EvaluateExprResult   cty.Value
 	EvaluateExprDiags    tfdiags.Diagnostics
 
+	EvaluationScopeCalled bool
+	EvaluationScopeSelf   addrs.Referenceable
+	EvaluationScopeKey    addrs.InstanceKey
+	EvaluationScopeScope  *lang.Scope
+
 	InterpolateCalled       bool
 	InterpolateConfig       *config.RawConfig
 	InterpolateResource     *Resource
@@ -227,6 +233,13 @@ func (c *MockEvalContext) EvaluateExpr(expr hcl.Expression, wantType cty.Type, s
 	return c.EvaluateExprResult, c.EvaluateExprDiags
 }
 
+func (c *MockEvalContext) EvaluationScope(self addrs.Referenceable, key addrs.InstanceKey) *lang.Scope {
+	c.EvaluationScopeCalled = true
+	c.EvaluationScopeSelf = self
+	c.EvaluationScopeKey = key
+	return c.EvaluationScopeScope
+}
+
 func (c *MockEvalContext) Interpolate(
 	config *config.RawConfig, resource *Resource) (*ResourceConfig, error) {
 	c.InterpolateCalled = true

terraform/graph_builder_eval.go

@@ -0,0 +1,104 @@
+package terraform
+
+import (
+	"github.com/hashicorp/terraform/dag"
+	"github.com/hashicorp/terraform/tfdiags"
+
+	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/configs"
+)
+
+// EvalGraphBuilder implements GraphBuilder and constructs a graph suitable
+// for evaluating in-memory values (input variables, local values, output
+// values) in the state without any other side-effects.
+//
+// This graph is used only in weird cases, such as the "terraform console"
+// CLI command, where we need to evaluate expressions against the state
+// without taking any other actions.
+//
+// The generated graph will include nodes for providers, resources, etc
+// just to allow indirect dependencies to be resolved, but these nodes will
+// not take any actions themselves since we assume that their parts of the
+// state, if any, are already complete.
+//
+// Although the providers are never configured, they must still be available
+// in order to obtain schema information used for type checking, etc.
+type EvalGraphBuilder struct {
+	// Config is the configuration tree.
+	Config *configs.Config
+
+	// State is the current state
+	State *State
+
+	// Components is a factory for the plug-in components (providers and
+	// provisioners) available for use.
+	Components contextComponentFactory
+}
+
+// See GraphBuilder
+func (b *EvalGraphBuilder) Build(path addrs.ModuleInstance) (*Graph, tfdiags.Diagnostics) {
+	return (&BasicGraphBuilder{
+		Steps:    b.Steps(),
+		Validate: true,
+		Name:     "EvalGraphBuilder",
+	}).Build(path)
+}
+
+// See GraphBuilder
+func (b *EvalGraphBuilder) Steps() []GraphTransformer {
+	concreteProvider := func(a *NodeAbstractProvider) dag.Vertex {
+		return &NodeEvalableProvider{
+			NodeAbstractProvider: a,
+		}
+	}
+
+	steps := []GraphTransformer{
+		// Creates all the data resources that aren't in the state. This will also
+		// add any orphans from scaling in as destroy nodes.
+		&ConfigTransformer{
+			Concrete: nil, // just use the abstract type
+			Config:   b.Config,
+			Unique:   true,
+		},
+
+		// Attach the state
+		&AttachStateTransformer{State: b.State},
+
+		// Attach the configuration to any resources
+		&AttachResourceConfigTransformer{Config: b.Config},
+
+		// Add root variables
+		&RootVariableTransformer{Config: b.Config},
+
+		TransformProviders(b.Components.ResourceProviders(), concreteProvider, b.Config),
+
+		// Add the local values
+		&LocalTransformer{Config: b.Config},
+
+		// Add the outputs
+		&OutputTransformer{Config: b.Config},
+
+		// Add module variables
+		&ModuleVariableTransformer{Config: b.Config},
+
+		// Must be before ReferenceTransformer, since schema is required to
+		// extract references from config.
+		&AttachSchemaTransformer{Components: b.Components},
+
+		// Connect so that the references are ready for targeting. We'll
+		// have to connect again later for providers and so on.
+		&ReferenceTransformer{},
+
+		// Although we don't configure providers, we do still start them up
+		// to get their schemas, and so we must shut them down again here.
+		&CloseProviderTransformer{},
+
+		// Single root
+		&RootTransformer{},
+
+		// Remove redundant edges to simplify the graph.
+		&TransitiveReductionTransformer{},
+	}
+
+	return steps
+}

terraform/graph_walk_operation.go

@@ -15,4 +15,5 @@ const (
 	walkValidate
 	walkDestroy
 	walkImport
+	walkEval // used just to prepare EvalContext for expression evaluation, with no other actions
 )

terraform/graphtype_string.go

@@ -4,9 +4,9 @@ package terraform
 
 import "strconv"
 
-const _GraphType_name = "GraphTypeInvalidGraphTypeLegacyGraphTypeRefreshGraphTypePlanGraphTypePlanDestroyGraphTypeApplyGraphTypeInputGraphTypeValidate"
+const _GraphType_name = "GraphTypeInvalidGraphTypeLegacyGraphTypeRefreshGraphTypePlanGraphTypePlanDestroyGraphTypeApplyGraphTypeInputGraphTypeValidateGraphTypeEval"
 
-var _GraphType_index = [...]uint8{0, 16, 31, 47, 60, 80, 94, 108, 125}
+var _GraphType_index = [...]uint8{0, 16, 31, 47, 60, 80, 94, 108, 125, 138}
 
 func (i GraphType) String() string {
 	if i >= GraphType(len(_GraphType_index)-1) {

terraform/node_provider_eval.go

@@ -0,0 +1,20 @@
+package terraform
+
+// NodeEvalableProvider represents a provider during an "eval" walk.
+// This special provider node type just initializes a provider and
+// fetches its schema, without configuring it or otherwise interacting
+// with it.
+type NodeEvalableProvider struct {
+	*NodeAbstractProvider
+}
+
+// GraphNodeEvalable
+func (n *NodeEvalableProvider) EvalTree() EvalNode {
+	addr := n.Addr
+	relAddr := addr.ProviderConfig
+
+	return &EvalInitProvider{
+		TypeName: relAddr.Type,
+		Addr:     addr.ProviderConfig,
+	}
+}

terraform/transform_provider.go

@@ -478,9 +478,15 @@ func (t *ProviderConfigTransformer) transformSingle(g *Graph, c *configs.Config)
 		relAddr := p.Addr()
 		addr := relAddr.Absolute(path)
 
-		v := t.Concrete(&NodeAbstractProvider{
+		abstract := &NodeAbstractProvider{
 			Addr: addr,
-		})
+		}
+		var v dag.Vertex
+		if t.Concrete != nil {
+			v = t.Concrete(abstract)
+		} else {
+			v = abstract
+		}
 
 		// Add it to the graph
 		g.Add(v)

terraform/valuesourcetype_string.go

@@ -0,0 +1,41 @@
+// Code generated by "stringer -type ValueSourceType"; DO NOT EDIT.
+
+package terraform
+
+import "strconv"
+
+const (
+	_ValueSourceType_name_0 = "ValueFromUnknown"
+	_ValueSourceType_name_1 = "ValueFromCLIArg"
+	_ValueSourceType_name_2 = "ValueFromConfig"
+	_ValueSourceType_name_3 = "ValueFromEnvVarValueFromFile"
+	_ValueSourceType_name_4 = "ValueFromInput"
+	_ValueSourceType_name_5 = "ValueFromPlan"
+	_ValueSourceType_name_6 = "ValueFromCaller"
+)
+
+var (
+	_ValueSourceType_index_3 = [...]uint8{0, 15, 28}
+)
+
+func (i ValueSourceType) String() string {
+	switch {
+	case i == 0:
+		return _ValueSourceType_name_0
+	case i == 65:
+		return _ValueSourceType_name_1
+	case i == 67:
+		return _ValueSourceType_name_2
+	case 69 <= i && i <= 70:
+		i -= 69
+		return _ValueSourceType_name_3[_ValueSourceType_index_3[i]:_ValueSourceType_index_3[i+1]]
+	case i == 73:
+		return _ValueSourceType_name_4
+	case i == 80:
+		return _ValueSourceType_name_5
+	case i == 83:
+		return _ValueSourceType_name_6
+	default:
+		return "ValueSourceType(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+}

terraform/walkoperation_string.go

@@ -4,9 +4,9 @@ package terraform
 
 import "strconv"
 
-const _walkOperation_name = "walkInvalidwalkInputwalkApplywalkPlanwalkPlanDestroywalkRefreshwalkValidatewalkDestroywalkImport"
+const _walkOperation_name = "walkInvalidwalkInputwalkApplywalkPlanwalkPlanDestroywalkRefreshwalkValidatewalkDestroywalkImportwalkEval"
 
-var _walkOperation_index = [...]uint8{0, 11, 20, 29, 37, 52, 63, 75, 86, 96}
+var _walkOperation_index = [...]uint8{0, 11, 20, 29, 37, 52, 63, 75, 86, 96, 104}
 
 func (i walkOperation) String() string {
 	if i >= walkOperation(len(_walkOperation_index)-1) {