Merge branch 'master' into patch-1

.go-version

@@ -1 +1 @@
-1.12.1
+1.12.4

.tfdev

@@ -0,0 +1,8 @@
+version_info {
+  commit_var  =    "main.GitCommit"
+  version_var =    "github.com/hashicorp/terraform/version.Version"
+  prerelease_var = "github.com/hashicorp/terraform/version.Prerelease"
+}
+
+version_exec = false
+disable_provider_requirements = true

.travis.yml

@@ -4,7 +4,7 @@ services:
   - docker
 language: go
 go:
-- "1.12.1"
+- "1.12.4"
 
 # add TF_CONSUL_TEST=1 to run consul tests
 # they were causing timouts in travis

CHANGELOG.md

@@ -1,5 +1,32 @@
-## 0.12.0-rc1 (Unreleased)
+## 0.12.0 (Unreleased)
 
+The following are the significant changes since 0.12.0-rc1.
+
+BUG FIXES:
+
+* configs/configupgrade: preserve in-line comments on lists [#21299]
+
+## 0.12.0-rc1 (May 7, 2019)
+
+The following are the significant changes since 0.12.0-beta2.
+
+NEW FEATURES: 
+
+* New function `strrev`, for reversing unicode strings. ([#21091](https://github.com/hashicorp/terraform/issues/21091))
+
+IMPROVEMENTS:
+
+* backend/s3: Support for the new AWS region `ap-east-1` ([#21117](https://github.com/hashicorp/terraform/issues/21117))
+* backend/remote: Do not unlock a workspace after a failed state upload ([#21148](https://github.com/hashicorp/terraform/issues/21148))
+* command/init: Improve formatting of provider names during discovery ([#21094](https://github.com/hashicorp/terraform/issues/21094))
+* command/0.12upgrade: Upgrade indexing of splat syntax ([#21103](https://github.com/hashicorp/terraform/issues/21103))
+* command/0.12upgrade: Return error for invalid references (e.g. with initial digits) ([#21103](https://github.com/hashicorp/terraform/issues/21103))
+
+BUG FIXES:
+
+* core: Make sure UIInput keeps working after being canceled ([#21139](https://github.com/hashicorp/terraform/issues/21139))
+* lang/funcs: `flatten` fix handling of sets and tuples; return a tuple ([#21171](https://github.com/hashicorp/terraform/issues/21171))
+* states/statefile: properly upgrade dependency syntax ([#21159](https://github.com/hashicorp/terraform/issues/21159))
 
 ## 0.12.0-beta2 (Apr 18, 2019)
 
@@ -241,6 +268,8 @@ In addition to the high-level known issues above, please refer also to [the GitH
 
 ## 0.11.11 (December 14, 2018)
 
+**NOTE:** Subsequent releases in the v0.11.x line occurred after this branch pivoted to v0.12.0 development. For more information on these, see [the v0.11 maintenance changelog](https://github.com/hashicorp/terraform/blob/v0.11/CHANGELOG.md).
+
 IMPROVEMENTS:
 
 * backend/remote: Return detailed version (in)compatibility information ([#19660](https://github.com/hashicorp/terraform/issues/19660))

addrs/resourcemode_string.go

@@ -4,6 +4,15 @@ package addrs
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[InvalidResourceMode-0]
+	_ = x[ManagedResourceMode-77]
+	_ = x[DataResourceMode-68]
+}
+
 const (
 	_ResourceMode_name_0 = "InvalidResourceMode"
 	_ResourceMode_name_1 = "DataResourceMode"

backend/local/counthookaction_string.go

@@ -4,6 +4,15 @@ package local
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[countHookActionAdd-0]
+	_ = x[countHookActionChange-1]
+	_ = x[countHookActionRemove-2]
+}
+
 const _countHookAction_name = "countHookActionAddcountHookActionChangecountHookActionRemove"
 
 var _countHookAction_index = [...]uint8{0, 18, 39, 60}

backend/operationtype_string.go

@@ -4,6 +4,16 @@ package backend
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[OperationTypeInvalid-0]
+	_ = x[OperationTypeRefresh-1]
+	_ = x[OperationTypePlan-2]
+	_ = x[OperationTypeApply-3]
+}
+
 const _OperationType_name = "OperationTypeInvalidOperationTypeRefreshOperationTypePlanOperationTypeApply"
 
 var _OperationType_index = [...]uint8{0, 20, 40, 57, 75}

backend/remote-state/pg/backend.go

@@ -65,7 +65,7 @@ func (b *Backend) configure(ctx context.Context) error {
 	// Prepare database schema, tables, & indexes.
 	var query string
 	query = `CREATE SCHEMA IF NOT EXISTS %s`
-	if _, err := db.Query(fmt.Sprintf(query, b.schemaName)); err != nil {
+	if _, err := db.Exec(fmt.Sprintf(query, b.schemaName)); err != nil {
 		return err
 	}
 	query = `CREATE TABLE IF NOT EXISTS %s.%s (
@@ -73,11 +73,11 @@ func (b *Backend) configure(ctx context.Context) error {
 		name TEXT,
 		data TEXT
 	)`
-	if _, err := db.Query(fmt.Sprintf(query, b.schemaName, statesTableName)); err != nil {
+	if _, err := db.Exec(fmt.Sprintf(query, b.schemaName, statesTableName)); err != nil {
 		return err
 	}
 	query = `CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s.%s (name)`
-	if _, err := db.Query(fmt.Sprintf(query, statesIndexName, b.schemaName, statesTableName)); err != nil {
+	if _, err := db.Exec(fmt.Sprintf(query, statesIndexName, b.schemaName, statesTableName)); err != nil {
 		return err
 	}
 

backend/remote/backend_common.go

@@ -227,6 +227,57 @@ func (b *Remote) parseVariableValues(op *backend.Operation) (terraform.InputValu
 	return result, diags
 }
 
+func (b *Remote) costEstimation(stopCtx, cancelCtx context.Context, op *backend.Operation, r *tfe.Run) error {
+	if r.CostEstimation == nil {
+		return nil
+	}
+
+	if b.CLI != nil {
+		b.CLI.Output("\n------------------------------------------------------------------------\n")
+	}
+
+	logs, err := b.client.CostEstimations.Logs(stopCtx, r.CostEstimation.ID)
+	if err != nil {
+		return generalError("Failed to retrieve cost estimation logs", err)
+	}
+	scanner := bufio.NewScanner(logs)
+
+	// Retrieve the cost estimation to get its current status.
+	ce, err := b.client.CostEstimations.Read(stopCtx, r.CostEstimation.ID)
+	if err != nil {
+		return generalError("Failed to retrieve cost estimation", err)
+	}
+
+	msgPrefix := "Cost estimation"
+	if b.CLI != nil {
+		b.CLI.Output(b.Colorize().Color(msgPrefix + ":\n"))
+	}
+
+	for scanner.Scan() {
+		if b.CLI != nil {
+			b.CLI.Output(b.Colorize().Color(scanner.Text()))
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return generalError("Failed to read logs", err)
+	}
+
+	switch ce.Status {
+	case tfe.CostEstimationFinished:
+		if len(r.PolicyChecks) == 0 && r.HasChanges && op.Type == backend.OperationTypeApply && b.CLI != nil {
+			b.CLI.Output("\n------------------------------------------------------------------------")
+		}
+		return nil
+	case tfe.CostEstimationErrored:
+		return fmt.Errorf(msgPrefix + " errored.")
+	case tfe.CostEstimationCanceled:
+		return fmt.Errorf(msgPrefix + " canceled.")
+	default:
+		return fmt.Errorf("Unknown or unexpected cost estimation state: %s", ce.Status)
+	}
+}
+
 func (b *Remote) checkPolicy(stopCtx, cancelCtx context.Context, op *backend.Operation, r *tfe.Run) error {
 	if b.CLI != nil {
 		b.CLI.Output("\n------------------------------------------------------------------------\n")

backend/remote/backend_mock.go

@@ -21,6 +21,7 @@ import (
 type mockClient struct {
 	Applies               *mockApplies
 	ConfigurationVersions *mockConfigurationVersions
+	CostEstimations       *mockCostEstimations
 	Organizations         *mockOrganizations
 	Plans                 *mockPlans
 	PolicyChecks          *mockPolicyChecks
@@ -33,6 +34,7 @@ func newMockClient() *mockClient {
 	c := &mockClient{}
 	c.Applies = newMockApplies(c)
 	c.ConfigurationVersions = newMockConfigurationVersions(c)
+	c.CostEstimations = newMockCostEstimations(c)
 	c.Organizations = newMockOrganizations(c)
 	c.Plans = newMockPlans(c)
 	c.PolicyChecks = newMockPolicyChecks(c)
@@ -212,6 +214,84 @@ func (m *mockConfigurationVersions) Upload(ctx context.Context, url, path string
 	return nil
 }
 
+type mockCostEstimations struct {
+	client      *mockClient
+	estimations map[string]*tfe.CostEstimation
+	logs        map[string]string
+}
+
+func newMockCostEstimations(client *mockClient) *mockCostEstimations {
+	return &mockCostEstimations{
+		client:      client,
+		estimations: make(map[string]*tfe.CostEstimation),
+		logs:        make(map[string]string),
+	}
+}
+
+// create is a helper function to create a mock cost estimation that uses the
+// configured working directory to find the logfile.
+func (m *mockCostEstimations) create(cvID, workspaceID string) (*tfe.CostEstimation, error) {
+	id := generateID("ce-")
+
+	ce := &tfe.CostEstimation{
+		ID:     id,
+		Status: tfe.CostEstimationQueued,
+	}
+
+	w, ok := m.client.Workspaces.workspaceIDs[workspaceID]
+	if !ok {
+		return nil, tfe.ErrResourceNotFound
+	}
+
+	logfile := filepath.Join(
+		m.client.ConfigurationVersions.uploadPaths[cvID],
+		w.WorkingDirectory,
+		"ce.log",
+	)
+
+	if _, err := os.Stat(logfile); os.IsNotExist(err) {
+		return nil, nil
+	}
+
+	m.logs[ce.ID] = logfile
+	m.estimations[ce.ID] = ce
+
+	return ce, nil
+}
+
+func (m *mockCostEstimations) Read(ctx context.Context, costEstimationID string) (*tfe.CostEstimation, error) {
+	ce, ok := m.estimations[costEstimationID]
+	if !ok {
+		return nil, tfe.ErrResourceNotFound
+	}
+	return ce, nil
+}
+
+func (m *mockCostEstimations) Logs(ctx context.Context, costEstimationID string) (io.Reader, error) {
+	ce, ok := m.estimations[costEstimationID]
+	if !ok {
+		return nil, tfe.ErrResourceNotFound
+	}
+
+	logfile, ok := m.logs[ce.ID]
+	if !ok {
+		return nil, tfe.ErrResourceNotFound
+	}
+
+	if _, err := os.Stat(logfile); os.IsNotExist(err) {
+		return bytes.NewBufferString("logfile does not exist"), nil
+	}
+
+	logs, err := ioutil.ReadFile(logfile)
+	if err != nil {
+		return nil, err
+	}
+
+	ce.Status = tfe.CostEstimationFinished
+
+	return bytes.NewBuffer(logs), nil
+}
+
 // mockInput is a mock implementation of terraform.UIInput.
 type mockInput struct {
 	answers map[string]string
@@ -652,19 +732,25 @@ func (m *mockRuns) Create(ctx context.Context, options tfe.RunCreateOptions) (*t
 		return nil, err
 	}
 
+	ce, err := m.client.CostEstimations.create(options.ConfigurationVersion.ID, options.Workspace.ID)
+	if err != nil {
+		return nil, err
+	}
+
 	pc, err := m.client.PolicyChecks.create(options.ConfigurationVersion.ID, options.Workspace.ID)
 	if err != nil {
 		return nil, err
 	}
 
 	r := &tfe.Run{
-		ID:          generateID("run-"),
-		Actions:     &tfe.RunActions{IsCancelable: true},
-		Apply:       a,
-		HasChanges:  false,
-		Permissions: &tfe.RunPermissions{},
-		Plan:        p,
-		Status:      tfe.RunPending,
+		ID:             generateID("run-"),
+		Actions:        &tfe.RunActions{IsCancelable: true},
+		Apply:          a,
+		CostEstimation: ce,
+		HasChanges:     false,
+		Permissions:    &tfe.RunPermissions{},
+		Plan:           p,
+		Status:         tfe.RunPending,
 	}
 
 	if pc != nil {

backend/remote/backend_plan.go

@@ -290,6 +290,14 @@ func (b *Remote) plan(stopCtx, cancelCtx context.Context, op *backend.Operation,
 		return r, nil
 	}
 
+	// Show any cost estimation output.
+	if r.CostEstimation != nil {
+		err = b.costEstimation(stopCtx, cancelCtx, op, r)
+		if err != nil {
+			return r, err
+		}
+	}
+
 	// Check any configured sentinel policies.
 	if len(r.PolicyChecks) > 0 {
 		err = b.checkPolicy(stopCtx, cancelCtx, op, r)

backend/remote/backend_plan_test.go

@@ -655,6 +655,40 @@ func TestRemote_planWithWorkingDirectory(t *testing.T) {
 	}
 }
 
+func TestRemote_costEstimation(t *testing.T) {
+	b, bCleanup := testBackendDefault(t)
+	defer bCleanup()
+
+	op, configCleanup := testOperationPlan(t, "./test-fixtures/plan-cost-estimation")
+	defer configCleanup()
+
+	op.Workspace = backend.DefaultStateName
+
+	run, err := b.Operation(context.Background(), op)
+	if err != nil {
+		t.Fatalf("error starting operation: %v", err)
+	}
+
+	<-run.Done()
+	if run.Result != backend.OperationSuccess {
+		t.Fatalf("operation failed: %s", b.CLI.(*cli.MockUi).ErrorWriter.String())
+	}
+	if run.PlanEmpty {
+		t.Fatalf("expected a non-empty plan")
+	}
+
+	output := b.CLI.(*cli.MockUi).OutputWriter.String()
+	if !strings.Contains(output, "Running plan in the remote backend") {
+		t.Fatalf("expected remote backend header in output: %s", output)
+	}
+	if !strings.Contains(output, "SKU") {
+		t.Fatalf("expected cost estimation result in output: %s", output)
+	}
+	if !strings.Contains(output, "1 to add, 0 to change, 0 to destroy") {
+		t.Fatalf("expected plan summary in output: %s", output)
+	}
+}
+
 func TestRemote_planPolicyPass(t *testing.T) {
 	b, bCleanup := testBackendDefault(t)
 	defer bCleanup()
@@ -681,12 +715,12 @@ func TestRemote_planPolicyPass(t *testing.T) {
 	if !strings.Contains(output, "Running plan in the remote backend") {
 		t.Fatalf("expected remote backend header in output: %s", output)
 	}
-	if !strings.Contains(output, "1 to add, 0 to change, 0 to destroy") {
-		t.Fatalf("expected plan summery in output: %s", output)
-	}
 	if !strings.Contains(output, "Sentinel Result: true") {
 		t.Fatalf("expected policy check result in output: %s", output)
 	}
+	if !strings.Contains(output, "1 to add, 0 to change, 0 to destroy") {
+		t.Fatalf("expected plan summery in output: %s", output)
+	}
 }
 
 func TestRemote_planPolicyHardFail(t *testing.T) {
@@ -720,12 +754,12 @@ func TestRemote_planPolicyHardFail(t *testing.T) {
 	if !strings.Contains(output, "Running plan in the remote backend") {
 		t.Fatalf("expected remote backend header in output: %s", output)
 	}
-	if !strings.Contains(output, "1 to add, 0 to change, 0 to destroy") {
-		t.Fatalf("expected plan summery in output: %s", output)
-	}
 	if !strings.Contains(output, "Sentinel Result: false") {
 		t.Fatalf("expected policy check result in output: %s", output)
 	}
+	if !strings.Contains(output, "1 to add, 0 to change, 0 to destroy") {
+		t.Fatalf("expected plan summery in output: %s", output)
+	}
 }
 
 func TestRemote_planPolicySoftFail(t *testing.T) {
@@ -759,12 +793,12 @@ func TestRemote_planPolicySoftFail(t *testing.T) {
 	if !strings.Contains(output, "Running plan in the remote backend") {
 		t.Fatalf("expected remote backend header in output: %s", output)
 	}
-	if !strings.Contains(output, "1 to add, 0 to change, 0 to destroy") {
-		t.Fatalf("expected plan summery in output: %s", output)
-	}
 	if !strings.Contains(output, "Sentinel Result: false") {
 		t.Fatalf("expected policy check result in output: %s", output)
 	}
+	if !strings.Contains(output, "1 to add, 0 to change, 0 to destroy") {
+		t.Fatalf("expected plan summery in output: %s", output)
+	}
 }
 
 func TestRemote_planWithRemoteError(t *testing.T) {

backend/remote/backend_state.go

@@ -14,11 +14,12 @@ import (
 )
 
 type remoteClient struct {
-	client       *tfe.Client
-	lockInfo     *state.LockInfo
-	organization string
-	runID        string
-	workspace    *tfe.Workspace
+	client         *tfe.Client
+	lockInfo       *state.LockInfo
+	organization   string
+	runID          string
+	stateUploadErr bool
+	workspace      *tfe.Workspace
 }
 
 // Get the remote state.
@@ -31,12 +32,12 @@ func (r *remoteClient) Get() (*remote.Payload, error) {
 			// If no state exists, then return nil.
 			return nil, nil
 		}
-		return nil, fmt.Errorf("Error retrieving remote state: %v", err)
+		return nil, fmt.Errorf("Error retrieving state: %v", err)
 	}
 
 	state, err := r.client.StateVersions.Download(ctx, sv.DownloadURL)
 	if err != nil {
-		return nil, fmt.Errorf("Error downloading remote state: %v", err)
+		return nil, fmt.Errorf("Error downloading state: %v", err)
 	}
 
 	// If the state is empty, then return nil.
@@ -79,7 +80,8 @@ func (r *remoteClient) Put(state []byte) error {
 	// Create the new state.
 	_, err = r.client.StateVersions.Create(ctx, r.workspace.ID, options)
 	if err != nil {
-		return fmt.Errorf("Error creating remote state: %v", err)
+		r.stateUploadErr = true
+		return fmt.Errorf("Error uploading state: %v", err)
 	}
 
 	return nil
@@ -106,6 +108,9 @@ func (r *remoteClient) Lock(info *state.LockInfo) (string, error) {
 		Reason: tfe.String("Locked by Terraform"),
 	})
 	if err != nil {
+		if err == tfe.ErrWorkspaceLocked {
+			err = fmt.Errorf("%s (lock ID: \"%s/%s\")", err, r.organization, r.workspace.Name)
+		}
 		lockErr.Err = err
 		return "", lockErr
 	}
@@ -119,6 +124,13 @@ func (r *remoteClient) Lock(info *state.LockInfo) (string, error) {
 func (r *remoteClient) Unlock(id string) error {
 	ctx := context.Background()
 
+	// We first check if there was an error while uploading the latest
+	// state. If so, we will not unlock the workspace to prevent any
+	// changes from being applied until the correct state is uploaded.
+	if r.stateUploadErr {
+		return nil
+	}
+
 	lockErr := &state.LockError{Info: r.lockInfo}
 
 	// With lock info this should be treated as a normal unlock.
@@ -141,7 +153,12 @@ func (r *remoteClient) Unlock(id string) error {
 
 	// Verify the optional force-unlock lock ID.
 	if r.organization+"/"+r.workspace.Name != id {
-		lockErr.Err = fmt.Errorf("lock ID does not match existing lock")
+		lockErr.Err = fmt.Errorf(
+			"lock ID %q does not match existing lock ID \"%s/%s\"",
+			id,
+			r.organization,
+			r.workspace.Name,
+		)
 		return lockErr
 	}
 

backend/remote/test-fixtures/plan-cost-estimation/ce.log

@@ -0,0 +1,6 @@
++---------+------+-----+-------------+----------------------+
+| PRODUCT | NAME | SKU | DESCRIPTION |        DELTA         |
++---------+------+-----+-------------+----------------------+
++---------+------+-----+-------------+----------------------+
+|                           TOTAL    | $0.000 USD / 720 HRS |
++---------+------+-----+-------------+----------------------+

backend/remote/test-fixtures/plan-cost-estimation/main.tf

@@ -0,0 +1 @@
+resource "null_resource" "foo" {}

backend/remote/test-fixtures/plan-cost-estimation/plan.log

@@ -0,0 +1,21 @@
+Terraform v0.11.7
+
+Configuring remote state backend...
+Initializing Terraform configuration...
+Refreshing Terraform state in-memory prior to plan...
+The refreshed state will be used to calculate this plan, but will not be
+persisted to local or remote state storage.
+
+------------------------------------------------------------------------
+
+An execution plan has been generated and is shown below.
+Resource actions are indicated with the following symbols:
+  + create
+
+Terraform will perform the following actions:
+
+  + null_resource.foo
+      id: <computed>
+
+
+Plan: 1 to add, 0 to change, 0 to destroy.

backend/remote/testing.go

@@ -115,6 +115,7 @@ func testBackend(t *testing.T, obj cty.Value) (*Remote, func()) {
 	b.CLI = cli.NewMockUi()
 	b.client.Applies = mc.Applies
 	b.client.ConfigurationVersions = mc.ConfigurationVersions
+	b.client.CostEstimations = mc.CostEstimations
 	b.client.Organizations = mc.Organizations
 	b.client.Plans = mc.Plans
 	b.client.PolicyChecks = mc.PolicyChecks

builtin/providers/terraform/data_source_state.go

@@ -42,79 +42,65 @@ func dataSourceRemoteStateGetSchema() providers.Schema {
 	}
 }
 
-func dataSourceRemoteStateRead(d *cty.Value) (cty.Value, tfdiags.Diagnostics) {
+func dataSourceRemoteStateValidate(cfg cty.Value) tfdiags.Diagnostics {
 	var diags tfdiags.Diagnostics
 
+	// Getting the backend implicitly validates the configuration for it,
+	// but we can only do that if it's all known already.
+	if cfg.GetAttr("config").IsWhollyKnown() && cfg.GetAttr("backend").IsKnown() {
+		_, moreDiags := getBackend(cfg)
+		diags = diags.Append(moreDiags)
+	} else {
+		// Otherwise we'll just type-check the config object itself.
+		configTy := cfg.GetAttr("config").Type()
+		if configTy != cty.DynamicPseudoType && !(configTy.IsObjectType() || configTy.IsMapType()) {
+			diags = diags.Append(tfdiags.AttributeValue(
+				tfdiags.Error,
+				"Invalid backend configuration",
+				"The configuration must be an object value.",
+				cty.GetAttrPath("config"),
+			))
+		}
+	}
+
+	{
+		defaultsTy := cfg.GetAttr("defaults").Type()
+		if defaultsTy != cty.DynamicPseudoType && !(defaultsTy.IsObjectType() || defaultsTy.IsMapType()) {
+			diags = diags.Append(tfdiags.AttributeValue(
+				tfdiags.Error,
+				"Invalid default values",
+				"Defaults must be given in an object value.",
+				cty.GetAttrPath("defaults"),
+			))
+		}
+	}
+
+	return diags
+}
+
+func dataSourceRemoteStateRead(d cty.Value) (cty.Value, tfdiags.Diagnostics) {
+	var diags tfdiags.Diagnostics
+
+	b, moreDiags := getBackend(d)
+	diags = diags.Append(moreDiags)
+	if diags.HasErrors() {
+		return cty.NilVal, diags
+	}
+
 	newState := make(map[string]cty.Value)
 	newState["backend"] = d.GetAttr("backend")
+	newState["config"] = d.GetAttr("config")
 
-	backendType := d.GetAttr("backend").AsString()
-
-	// Don't break people using the old _local syntax - but note warning above
-	if backendType == "_local" {
-		log.Println(`[INFO] Switching old (unsupported) backend "_local" to "local"`)
-		backendType = "local"
-	}
-
-	// Create the client to access our remote state
-	log.Printf("[DEBUG] Initializing remote state backend: %s", backendType)
-	f := backendInit.Backend(backendType)
-	if f == nil {
-		diags = diags.Append(tfdiags.AttributeValue(
-			tfdiags.Error,
-			"Invalid backend configuration",
-			fmt.Sprintf("Unknown backend type: %s", backendType),
-			cty.Path(nil).GetAttr("backend"),
-		))
-		return cty.NilVal, diags
-	}
-	b := f()
-
-	config := d.GetAttr("config")
-	if config.IsNull() {
-		// We'll treat this as an empty configuration and see if the backend's
-		// schema and validation code will accept it.
-		config = cty.EmptyObjectVal
-	}
-	newState["config"] = config
-
-	schema := b.ConfigSchema()
-	// Try to coerce the provided value into the desired configuration type.
-	configVal, err := schema.CoerceValue(config)
-	if err != nil {
-		diags = diags.Append(tfdiags.AttributeValue(
-			tfdiags.Error,
-			"Invalid backend configuration",
-			fmt.Sprintf("The given configuration is not valid for backend %q: %s.", backendType,
-				tfdiags.FormatError(err)),
-			cty.Path(nil).GetAttr("config"),
-		))
-		return cty.NilVal, diags
-	}
-
-	newVal, validateDiags := b.PrepareConfig(configVal)
-	diags = diags.Append(validateDiags)
-	if validateDiags.HasErrors() {
-		return cty.NilVal, diags
-	}
-	configVal = newVal
-
-	configureDiags := b.Configure(configVal)
-	if configureDiags.HasErrors() {
-		diags = diags.Append(configureDiags.Err())
-		return cty.NilVal, diags
-	}
-
-	name := backend.DefaultStateName
+	workspaceName := backend.DefaultStateName
 
 	if workspaceVal := d.GetAttr("workspace"); !workspaceVal.IsNull() {
 		newState["workspace"] = workspaceVal
-		name = workspaceVal.AsString()
+		workspaceName = workspaceVal.AsString()
 	}
 
-	newState["workspace"] = cty.StringVal(name)
+	newState["workspace"] = cty.StringVal(workspaceName)
 
-	state, err := b.StateMgr(name)
+	state, err := b.StateMgr(workspaceName)
 	if err != nil {
 		diags = diags.Append(tfdiags.AttributeValue(
 			tfdiags.Error,
@@ -165,3 +151,69 @@ func dataSourceRemoteStateRead(d *cty.Value) (cty.Value, tfdiags.Diagnostics) {
 
 	return cty.ObjectVal(newState), diags
 }
+
+func getBackend(cfg cty.Value) (backend.Backend, tfdiags.Diagnostics) {
+	var diags tfdiags.Diagnostics
+
+	backendType := cfg.GetAttr("backend").AsString()
+
+	// Don't break people using the old _local syntax - but note warning above
+	if backendType == "_local" {
+		log.Println(`[INFO] Switching old (unsupported) backend "_local" to "local"`)
+		backendType = "local"
+	}
+
+	// Create the client to access our remote state
+	log.Printf("[DEBUG] Initializing remote state backend: %s", backendType)
+	f := backendInit.Backend(backendType)
+	if f == nil {
+		diags = diags.Append(tfdiags.AttributeValue(
+			tfdiags.Error,
+			"Invalid backend configuration",
+			fmt.Sprintf("There is no backend type named %q.", backendType),
+			cty.Path(nil).GetAttr("backend"),
+		))
+		return nil, diags
+	}
+	b := f()
+
+	config := cfg.GetAttr("config")
+	if config.IsNull() {
+		// We'll treat this as an empty configuration and see if the backend's
+		// schema and validation code will accept it.
+		config = cty.EmptyObjectVal
+	}
+
+	if config.Type().IsMapType() { // The code below expects an object type, so we'll convert
+		config = cty.ObjectVal(config.AsValueMap())
+	}
+
+	schema := b.ConfigSchema()
+	// Try to coerce the provided value into the desired configuration type.
+	configVal, err := schema.CoerceValue(config)
+	if err != nil {
+		diags = diags.Append(tfdiags.AttributeValue(
+			tfdiags.Error,
+			"Invalid backend configuration",
+			fmt.Sprintf("The given configuration is not valid for backend %q: %s.", backendType,
+				tfdiags.FormatError(err)),
+			cty.Path(nil).GetAttr("config"),
+		))
+		return nil, diags
+	}
+
+	newVal, validateDiags := b.PrepareConfig(configVal)
+	diags = diags.Append(validateDiags)
+	if validateDiags.HasErrors() {
+		return nil, diags
+	}
+	configVal = newVal
+
+	configureDiags := b.Configure(configVal)
+	if configureDiags.HasErrors() {
+		diags = diags.Append(configureDiags.Err())
+		return nil, diags
+	}
+
+	return b, diags
+}

builtin/providers/terraform/data_source_state_test.go

@@ -1,6 +1,7 @@
 package terraform
 
 import (
+	"github.com/hashicorp/terraform/tfdiags"
 	"testing"
 
 	"github.com/apparentlymart/go-dump/dump"
@@ -138,6 +139,80 @@ func TestState_basic(t *testing.T) {
 			}),
 			true,
 		},
+		"wrong type for config": {
+			cty.ObjectVal(map[string]cty.Value{
+				"backend": cty.StringVal("local"),
+				"config":  cty.StringVal("nope"),
+			}),
+			cty.NilVal,
+			true,
+		},
+		"wrong type for config with unknown backend": {
+			cty.ObjectVal(map[string]cty.Value{
+				"backend": cty.UnknownVal(cty.String),
+				"config":  cty.StringVal("nope"),
+			}),
+			cty.NilVal,
+			true,
+		},
+		"wrong type for config with unknown config": {
+			cty.ObjectVal(map[string]cty.Value{
+				"backend": cty.StringVal("local"),
+				"config":  cty.UnknownVal(cty.String),
+			}),
+			cty.NilVal,
+			true,
+		},
+		"wrong type for defaults": {
+			cty.ObjectVal(map[string]cty.Value{
+				"backend": cty.StringVal("local"),
+				"config": cty.ObjectVal(map[string]cty.Value{
+					"path": cty.StringVal("./test-fixtures/basic.tfstate"),
+				}),
+				"defaults": cty.StringVal("nope"),
+			}),
+			cty.NilVal,
+			true,
+		},
+		"config as map": {
+			cty.ObjectVal(map[string]cty.Value{
+				"backend": cty.StringVal("local"),
+				"config": cty.MapVal(map[string]cty.Value{
+					"path": cty.StringVal("./test-fixtures/empty.tfstate"),
+				}),
+			}),
+			cty.ObjectVal(map[string]cty.Value{
+				"backend": cty.StringVal("local"),
+				"config": cty.MapVal(map[string]cty.Value{
+					"path": cty.StringVal("./test-fixtures/empty.tfstate"),
+				}),
+				"defaults":  cty.NullVal(cty.DynamicPseudoType),
+				"outputs":   cty.EmptyObjectVal,
+				"workspace": cty.StringVal(backend.DefaultStateName),
+			}),
+			false,
+		},
+		"defaults as map": {
+			cty.ObjectVal(map[string]cty.Value{
+				"backend": cty.StringVal("local"),
+				"config": cty.ObjectVal(map[string]cty.Value{
+					"path": cty.StringVal("./test-fixtures/basic.tfstate"),
+				}),
+				"defaults": cty.MapValEmpty(cty.String),
+			}),
+			cty.ObjectVal(map[string]cty.Value{
+				"backend": cty.StringVal("local"),
+				"config": cty.ObjectVal(map[string]cty.Value{
+					"path": cty.StringVal("./test-fixtures/basic.tfstate"),
+				}),
+				"defaults": cty.MapValEmpty(cty.String),
+				"outputs": cty.ObjectVal(map[string]cty.Value{
+					"foo": cty.StringVal("bar"),
+				}),
+				"workspace": cty.StringVal(backend.DefaultStateName),
+			}),
+			false,
+		},
 	}
 	for name, test := range tests {
 		t.Run(name, func(t *testing.T) {
@@ -146,7 +221,15 @@ func TestState_basic(t *testing.T) {
 			if err != nil {
 				t.Fatalf("unexpected error: %s", err)
 			}
-			got, diags := dataSourceRemoteStateRead(&config)
+
+			diags := dataSourceRemoteStateValidate(config)
+
+			var got cty.Value
+			if !diags.HasErrors() && config.IsWhollyKnown() {
+				var moreDiags tfdiags.Diagnostics
+				got, moreDiags = dataSourceRemoteStateRead(config)
+				diags = diags.Append(moreDiags)
+			}
 
 			if test.Err {
 				if !diags.HasErrors() {
@@ -156,8 +239,8 @@ func TestState_basic(t *testing.T) {
 				t.Fatalf("unexpected errors: %s", diags.Err())
 			}
 
-			if !test.Want.RawEquals(got) {
-				t.Errorf("wrong result\nconfig: %sgot: %swant: %s", dump.Value(config), dump.Value(got), dump.Value(test.Want))
+			if test.Want != cty.NilVal && !test.Want.RawEquals(got) {
+				t.Errorf("wrong result\nconfig: %sgot:    %swant:   %s", dump.Value(config), dump.Value(got), dump.Value(test.Want))
 			}
 		})
 	}

builtin/providers/terraform/provider.go

@@ -40,11 +40,21 @@ func (p *Provider) PrepareProviderConfig(req providers.PrepareProviderConfigRequ
 }
 
 // ValidateDataSourceConfig is used to validate the data source configuration values.
-func (p *Provider) ValidateDataSourceConfig(providers.ValidateDataSourceConfigRequest) providers.ValidateDataSourceConfigResponse {
+func (p *Provider) ValidateDataSourceConfig(req providers.ValidateDataSourceConfigRequest) providers.ValidateDataSourceConfigResponse {
 	// FIXME: move the backend configuration validate call that's currently
 	// inside the read method  into here so that we can catch provider configuration
 	// errors in terraform validate as well as during terraform plan.
 	var res providers.ValidateDataSourceConfigResponse
+
+	// This should not happen
+	if req.TypeName != "terraform_remote_state" {
+		res.Diagnostics.Append(fmt.Errorf("Error: unsupported data source %s", req.TypeName))
+		return res
+	}
+
+	diags := dataSourceRemoteStateValidate(req.Config)
+	res.Diagnostics = diags
+
 	return res
 }
 
@@ -67,7 +77,7 @@ func (p *Provider) ReadDataSource(req providers.ReadDataSourceRequest) providers
 		return res
 	}
 
-	newState, diags := dataSourceRemoteStateRead(&req.Config)
+	newState, diags := dataSourceRemoteStateRead(req.Config)
 
 	res.State = newState
 	res.Diagnostics = diags

builtin/providers/test/data_source_test.go

@@ -239,3 +239,53 @@ data "test_data_source" "two" {
 		},
 	})
 }
+
+func TestDataSource_planUpdate(t *testing.T) {
+	resource.UnitTest(t, resource.TestCase{
+		Providers: testAccProviders,
+		Steps: []resource.TestStep{
+			{
+				Config: strings.TrimSpace(`
+resource "test_resource" "a" {
+	required = "first"
+	required_map = {
+	    key = "1"
+	}
+	optional_force_new = "first"
+}
+
+data "test_data_source" "a" {
+  input = "${test_resource.a.computed_from_required}"
+}
+
+output "out" {
+  value = "${data.test_data_source.a.output}"
+}
+				`),
+			},
+			{
+				Config: strings.TrimSpace(`
+resource "test_resource" "a" {
+	required = "second"
+	required_map = {
+	    key = "1"
+	}
+	optional_force_new = "second"
+}
+
+data "test_data_source" "a" {
+  input = "${test_resource.a.computed_from_required}"
+}
+
+output "out" {
+  value = "${data.test_data_source.a.output}"
+}
+				`),
+				Check: resource.ComposeAggregateTestCheckFunc(
+					resource.TestCheckResourceAttr("data.test_data_source.a", "output", "second"),
+					resource.TestCheckOutput("out", "second"),
+				),
+			},
+		},
+	})
+}

builtin/providers/test/diff_apply_test.go

@@ -131,7 +131,7 @@ func TestDiffApply_set(t *testing.T) {
 		"id":                                 "testID",
 	}
 
-	attrs, err := diff.Apply(priorAttrs, schema.LegacyResourceSchema(&schema.Resource{Schema: resSchema}).CoreConfigSchema())
+	attrs, err := diff.Apply(priorAttrs, (&schema.Resource{Schema: resSchema}).CoreConfigSchema())
 	if err != nil {
 		t.Fatal(err)
 	}

builtin/providers/test/resource.go

@@ -54,6 +54,11 @@ func testResource() *schema.Resource {
 				Computed: true,
 				ForceNew: true,
 			},
+			"optional_computed": {
+				Type:     schema.TypeString,
+				Optional: true,
+				Computed: true,
+			},
 			"computed_read_only": {
 				Type:     schema.TypeString,
 				Computed: true,
@@ -148,6 +153,10 @@ func testResource() *schema.Resource {
 				Optional:    true,
 				Description: "do not set in config",
 			},
+			"int": {
+				Type:     schema.TypeInt,
+				Optional: true,
+			},
 		},
 	}
 }

builtin/providers/test/resource_config_mode.go

@@ -28,21 +28,6 @@ func testResourceConfigMode() *schema.Resource {
 					},
 				},
 			},
-			"resource_as_attr_dynamic": {
-				Type:              schema.TypeList,
-				ConfigMode:        schema.SchemaConfigModeAttr,
-				SkipCoreTypeCheck: true,
-				Optional:          true,
-				Elem: &schema.Resource{
-					Schema: map[string]*schema.Schema{
-						"foo": {
-							Type:     schema.TypeString,
-							Optional: true,
-							Default:  "default",
-						},
-					},
-				},
-			},
 		},
 	}
 }
@@ -53,14 +38,12 @@ func testResourceConfigModeCreate(d *schema.ResourceData, meta interface{}) erro
 }
 
 func testResourceConfigModeRead(d *schema.ResourceData, meta interface{}) error {
-	for _, k := range []string{"resource_as_attr", "resource_as_attr_dynamic"} {
-		if l, ok := d.Get(k).([]interface{}); !ok {
-			return fmt.Errorf("%s should appear as []interface{}, not %T", k, l)
-		} else {
-			for i, item := range l {
-				if _, ok := item.(map[string]interface{}); !ok {
-					return fmt.Errorf("%s[%d] should appear as map[string]interface{}, not %T", k, i, item)
-				}
+	if l, ok := d.Get("resource_as_attr").([]interface{}); !ok {
+		return fmt.Errorf("resource_as_attr should appear as []interface{}, not %T", l)
+	} else {
+		for i, item := range l {
+			if _, ok := item.(map[string]interface{}); !ok {
+				return fmt.Errorf("resource_as_attr[%d] should appear as map[string]interface{}, not %T", i, item)
 			}
 		}
 	}

builtin/providers/test/resource_config_mode_test.go

@@ -23,22 +23,12 @@ resource "test_resource_config_mode" "foo" {
 			foo = "resource_as_attr 1"
 		},
 	]
-	resource_as_attr_dynamic = [
-		{
-			foo = "resource_as_attr_dynamic 0"
-		},
-		{
-		},
-	]
 }
 				`),
 				Check: resource.ComposeTestCheckFunc(
 					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr.#", "2"),
 					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr.0.foo", "resource_as_attr 0"),
 					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr.1.foo", "resource_as_attr 1"),
-					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr_dynamic.#", "2"),
-					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr_dynamic.0.foo", "resource_as_attr_dynamic 0"),
-					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr_dynamic.1.foo", "default"),
 				),
 			},
 			resource.TestStep{
@@ -58,22 +48,12 @@ resource "test_resource_config_mode" "foo" {
 	resource_as_attr {
 		foo = "resource_as_attr 1"
 	}
-	resource_as_attr_dynamic = [
-		{
-			foo = "resource_as_attr_dynamic 0"
-		},
-		{
-		},
-	]
 }
 				`),
 				Check: resource.ComposeTestCheckFunc(
 					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr.#", "2"),
 					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr.0.foo", "resource_as_attr 0"),
 					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr.1.foo", "resource_as_attr 1"),
-					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr_dynamic.#", "2"),
-					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr_dynamic.0.foo", "resource_as_attr_dynamic 0"),
-					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr_dynamic.1.foo", "default"),
 				),
 			},
 			resource.TestStep{
@@ -84,43 +64,21 @@ resource "test_resource_config_mode" "foo" {
 			foo = "resource_as_attr 0 updated"
 		},
 	]
-	resource_as_attr_dynamic = [
-		{
-		},
-	]
 }
 				`),
 				Check: resource.ComposeTestCheckFunc(
 					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr.#", "1"),
 					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr.0.foo", "resource_as_attr 0 updated"),
-					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr_dynamic.#", "1"),
-					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr_dynamic.0.foo", "default"),
-				),
-			},
-			resource.TestStep{
-				Config: strings.TrimSpace(`
-resource "test_resource_config_mode" "foo" {
-	resource_as_attr_dynamic = [
-		{
-		},
-	]
-}
-				`),
-				Check: resource.ComposeTestCheckFunc(
-					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr.#", "1"),
-					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr_dynamic.#", "1"),
 				),
 			},
 			resource.TestStep{
 				Config: strings.TrimSpace(`
 resource "test_resource_config_mode" "foo" {
 	resource_as_attr = []
-	resource_as_attr_dynamic = []
 }
 				`),
 				Check: resource.ComposeTestCheckFunc(
 					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr.#", "0"),
-					resource.TestCheckResourceAttr("test_resource_config_mode.foo", "resource_as_attr_dynamic.#", "0"),
 				),
 			},
 			resource.TestStep{
@@ -130,7 +88,6 @@ resource "test_resource_config_mode" "foo" {
 				`),
 				Check: resource.ComposeTestCheckFunc(
 					resource.TestCheckNoResourceAttr("test_resource_config_mode.foo", "resource_as_attr.#"),
-					resource.TestCheckNoResourceAttr("test_resource_config_mode.foo", "resource_as_attr_dynamic.#"),
 				),
 			},
 		},

builtin/providers/test/resource_list_test.go

@@ -447,3 +447,37 @@ resource "test_resource_list" "bar" {
 		},
 	})
 }
+
+func TestResourceList_dynamicList(t *testing.T) {
+	resource.UnitTest(t, resource.TestCase{
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckResourceDestroy,
+		Steps: []resource.TestStep{
+			resource.TestStep{
+				Config: strings.TrimSpace(`
+resource "test_resource_list" "a" {
+	dependent_list {
+		val = "a"
+	}
+
+	dependent_list {
+		val = "b"
+	}
+}
+resource "test_resource_list" "b" {
+	list_block {
+		string = "constant"
+	}
+	dynamic "list_block" {
+		for_each = test_resource_list.a.computed_list
+		content {
+		  string = list_block.value
+		}
+	}
+}
+				`),
+				Check: resource.ComposeTestCheckFunc(),
+			},
+		},
+	})
+}

builtin/providers/test/resource_test.go

@@ -992,3 +992,61 @@ resource "test_resource" "foo" {
 		},
 	})
 }
+
+func TestResource_replacedOptionalComputed(t *testing.T) {
+	resource.UnitTest(t, resource.TestCase{
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckResourceDestroy,
+		Steps: []resource.TestStep{
+			resource.TestStep{
+				Config: strings.TrimSpace(`
+resource "test_resource_nested" "a" {
+}
+
+resource "test_resource" "foo" {
+	required = "yep"
+	required_map = {
+	    key = "value"
+	}
+	optional_computed = test_resource_nested.a.id
+}
+				`),
+			},
+			resource.TestStep{
+				Config: strings.TrimSpace(`
+resource "test_resource_nested" "b" {
+}
+
+resource "test_resource" "foo" {
+	required = "yep"
+	required_map = {
+	    key = "value"
+	}
+	optional_computed = test_resource_nested.b.id
+}
+				`),
+			},
+		},
+	})
+}
+
+func TestResource_floatInIntAttr(t *testing.T) {
+	resource.UnitTest(t, resource.TestCase{
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckResourceDestroy,
+		Steps: []resource.TestStep{
+			resource.TestStep{
+				Config: strings.TrimSpace(`
+resource "test_resource" "foo" {
+	required = "yep"
+	required_map = {
+	    key = "value"
+	}
+	int = 40.2
+}
+				`),
+				ExpectError: regexp.MustCompile(`must be a whole number, got 40.2`),
+			},
+		},
+	})
+}

command/e2etest/automation_test.go

@@ -7,7 +7,6 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/davecgh/go-spew/spew"
 	"github.com/hashicorp/terraform/e2e"
 )
 
@@ -41,11 +40,11 @@ func TestPlanApplyInAutomation(t *testing.T) {
 
 	// Make sure we actually downloaded the plugins, rather than picking up
 	// copies that might be already installed globally on the system.
-	if !strings.Contains(stdout, "- Downloading plugin for provider \"template\"") {
+	if !strings.Contains(stdout, "- Downloading plugin for provider \"template") {
 		t.Errorf("template provider download message is missing from init output:\n%s", stdout)
 		t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)")
 	}
-	if !strings.Contains(stdout, "- Downloading plugin for provider \"null\"") {
+	if !strings.Contains(stdout, "- Downloading plugin for provider \"null") {
 		t.Errorf("null provider download message is missing from init output:\n%s", stdout)
 		t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)")
 	}
@@ -71,14 +70,11 @@ func TestPlanApplyInAutomation(t *testing.T) {
 		t.Fatalf("failed to read plan file: %s", err)
 	}
 
-	stateResources := plan.State.RootModule().Resources
-	diffResources := plan.Diff.RootModule().Resources
+	// stateResources := plan.Changes.Resources
+	diffResources := plan.Changes.Resources
 
-	if len(stateResources) != 1 || stateResources["data.template_file.test"] == nil {
-		t.Errorf("incorrect state in plan; want just data.template_file.test to have been rendered, but have:\n%s", spew.Sdump(stateResources))
-	}
-	if len(diffResources) != 1 || diffResources["null_resource.test"] == nil {
-		t.Errorf("incorrect diff in plan; want just null_resource.test to have been rendered, but have:\n%s", spew.Sdump(diffResources))
+	if len(diffResources) != 1 || diffResources[0].Addr.String() != "null_resource.test" {
+		t.Errorf("incorrect number of resources in plan")
 	}
 
 	//// APPLY
@@ -96,9 +92,9 @@ func TestPlanApplyInAutomation(t *testing.T) {
 		t.Fatalf("failed to read state file: %s", err)
 	}
 
-	stateResources = state.RootModule().Resources
+	stateResources := state.RootModule().Resources
 	var gotResources []string
-	for n := range stateResources {
+	for n, _ := range stateResources {
 		gotResources = append(gotResources, n)
 	}
 	sort.Strings(gotResources)
@@ -139,11 +135,11 @@ func TestAutoApplyInAutomation(t *testing.T) {
 
 	// Make sure we actually downloaded the plugins, rather than picking up
 	// copies that might be already installed globally on the system.
-	if !strings.Contains(stdout, "- Downloading plugin for provider \"template\"") {
+	if !strings.Contains(stdout, "- Downloading plugin for provider \"template") {
 		t.Errorf("template provider download message is missing from init output:\n%s", stdout)
 		t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)")
 	}
-	if !strings.Contains(stdout, "- Downloading plugin for provider \"null\"") {
+	if !strings.Contains(stdout, "- Downloading plugin for provider \"null") {
 		t.Errorf("null provider download message is missing from init output:\n%s", stdout)
 		t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)")
 	}
@@ -206,11 +202,11 @@ func TestPlanOnlyInAutomation(t *testing.T) {
 
 	// Make sure we actually downloaded the plugins, rather than picking up
 	// copies that might be already installed globally on the system.
-	if !strings.Contains(stdout, "- Downloading plugin for provider \"template\"") {
+	if !strings.Contains(stdout, "- Downloading plugin for provider \"template") {
 		t.Errorf("template provider download message is missing from init output:\n%s", stdout)
 		t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)")
 	}
-	if !strings.Contains(stdout, "- Downloading plugin for provider \"null\"") {
+	if !strings.Contains(stdout, "- Downloading plugin for provider \"null") {
 		t.Errorf("null provider download message is missing from init output:\n%s", stdout)
 		t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)")
 	}

command/e2etest/init_test.go

@@ -39,7 +39,7 @@ func TestInitProviders(t *testing.T) {
 		t.Errorf("success message is missing from output:\n%s", stdout)
 	}
 
-	if !strings.Contains(stdout, "- Downloading plugin for provider \"template\"") {
+	if !strings.Contains(stdout, "- Downloading plugin for provider \"template\" (terraform-providers/template)") {
 		t.Errorf("provider download message is missing from output:\n%s", stdout)
 		t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)")
 	}
@@ -112,10 +112,10 @@ func TestInitProviders_pluginCache(t *testing.T) {
 
 	stderr := cmd.Stderr.(*bytes.Buffer).String()
 	if stderr != "" {
-		t.Errorf("unexpected stderr output:\n%s", stderr)
+		t.Errorf("unexpected stderr output:\n%s\n", stderr)
 	}
 
-	path := fmt.Sprintf(".terraform/plugins/%s_%s/terraform-provider-template_v0.1.0_x4", runtime.GOOS, runtime.GOARCH)
+	path := fmt.Sprintf(".terraform/plugins/%s_%s/terraform-provider-template_v2.1.0_x4", runtime.GOOS, runtime.GOARCH)
 	content, err := tf.ReadFile(path)
 	if err != nil {
 		t.Fatalf("failed to read installed plugin from %s: %s", path, err)
@@ -124,11 +124,11 @@ func TestInitProviders_pluginCache(t *testing.T) {
 		t.Errorf("template plugin was not installed from local cache")
 	}
 
-	if !tf.FileExists(fmt.Sprintf(".terraform/plugins/%s_%s/terraform-provider-null_v0.1.0_x4", runtime.GOOS, runtime.GOARCH)) {
+	if !tf.FileExists(fmt.Sprintf(".terraform/plugins/%s_%s/terraform-provider-null_v2.1.0_x4", runtime.GOOS, runtime.GOARCH)) {
 		t.Errorf("null plugin was not installed")
 	}
 
-	if !tf.FileExists(fmt.Sprintf("cache/%s_%s/terraform-provider-null_v0.1.0_x4", runtime.GOOS, runtime.GOARCH)) {
+	if !tf.FileExists(fmt.Sprintf("cache/%s_%s/terraform-provider-null_v2.1.0_x4", runtime.GOOS, runtime.GOARCH)) {
 		t.Errorf("null plugin is not in cache after install")
 	}
 }

command/e2etest/main_test.go

@@ -56,10 +56,4 @@ func skipIfCannotAccessNetwork(t *testing.T) {
 	if !canAccessNetwork() {
 		t.Skip("network access not allowed; use TF_ACC=1 to enable")
 	}
-
-	// During the early part of the Terraform v0.12 release process, certain
-	// upstream resources are not yet ready to support it and so these
-	// tests cannot be run. These will be re-enabled prior to Terraform v0.12.0
-	// final.
-	t.Skip("all tests with external network access are temporarily disabled until upstream services are updated")
 }

command/e2etest/primary_test.go

@@ -38,11 +38,11 @@ func TestPrimarySeparatePlan(t *testing.T) {
 
 	// Make sure we actually downloaded the plugins, rather than picking up
 	// copies that might be already installed globally on the system.
-	if !strings.Contains(stdout, "- Downloading plugin for provider \"template\"") {
+	if !strings.Contains(stdout, "- Downloading plugin for provider \"template") {
 		t.Errorf("template provider download message is missing from init output:\n%s", stdout)
 		t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)")
 	}
-	if !strings.Contains(stdout, "- Downloading plugin for provider \"null\"") {
+	if !strings.Contains(stdout, "- Downloading plugin for provider \"null") {
 		t.Errorf("null provider download message is missing from init output:\n%s", stdout)
 		t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)")
 	}
@@ -69,13 +69,8 @@ func TestPrimarySeparatePlan(t *testing.T) {
 		t.Fatalf("failed to read plan file: %s", err)
 	}
 
-	stateResources := plan.State.RootModule().Resources
-	diffResources := plan.Diff.RootModule().Resources
-
-	if len(stateResources) != 1 || stateResources["data.template_file.test"] == nil {
-		t.Errorf("incorrect state in plan; want just data.template_file.test to have been rendered, but have:\n%s", spew.Sdump(stateResources))
-	}
-	if len(diffResources) != 1 || diffResources["null_resource.test"] == nil {
+	diffResources := plan.Changes.Resources
+	if len(diffResources) != 1 || diffResources[0].Addr.String() != "null_resource.test" {
 		t.Errorf("incorrect diff in plan; want just null_resource.test to have been rendered, but have:\n%s", spew.Sdump(diffResources))
 	}
 
@@ -94,9 +89,9 @@ func TestPrimarySeparatePlan(t *testing.T) {
 		t.Fatalf("failed to read state file: %s", err)
 	}
 
-	stateResources = state.RootModule().Resources
+	stateResources := state.RootModule().Resources
 	var gotResources []string
-	for n := range stateResources {
+	for n, _ := range stateResources {
 		gotResources = append(gotResources, n)
 	}
 	sort.Strings(gotResources)

command/e2etest/test-fixtures/plugin-cache/main.tf

@@ -1,7 +1,7 @@
 provider "template" {
-  version = "0.1.0"
+  version = "2.1.0"
 }
 
 provider "null" {
-  version = "0.1.0"
+  version = "2.1.0"
 }

command/format/diff.go

@@ -1101,8 +1101,8 @@ func ctySequenceDiff(old, new []cty.Value) []*plans.Change {
 		if lcsI < len(lcs) {
 			ret = append(ret, &plans.Change{
 				Action: plans.NoOp,
-				Before: new[newI],
-				After:  new[newI],
+				Before: lcs[lcsI],
+				After:  lcs[lcsI],
 			})
 
 			// All of our indexes advance together now, since the line

command/format/diff_test.go

@@ -2564,13 +2564,13 @@ func TestResourceChange_nestedSet(t *testing.T) {
       ~ ami = "ami-BEFORE" -> "ami-AFTER"
         id  = "i-02ae66f368e8518a9"
 
-      - root_block_device {
-          - volume_type = "gp2" -> null
-        }
       + root_block_device {
           + new_field   = "new_value"
           + volume_type = "gp2"
         }
+      - root_block_device {
+          - volume_type = "gp2" -> null
+        }
     }
 `,
 		},
@@ -2624,12 +2624,12 @@ func TestResourceChange_nestedSet(t *testing.T) {
       ~ ami = "ami-BEFORE" -> "ami-AFTER"
         id  = "i-02ae66f368e8518a9"
 
-      - root_block_device { # forces replacement
-          - volume_type = "gp2" -> null
-        }
       + root_block_device { # forces replacement
           + volume_type = "different"
         }
+      - root_block_device { # forces replacement
+          - volume_type = "gp2" -> null
+        }
     }
 `,
 		},
@@ -3006,6 +3006,49 @@ func TestResourceChange_nestedMap(t *testing.T) {
           - volume_type = "gp2" -> null
         }
     }
+`,
+		},
+		"in-place sequence update - deletion": {
+			Action: plans.Update,
+			Mode:   addrs.ManagedResourceMode,
+			Before: cty.ObjectVal(map[string]cty.Value{
+				"list": cty.ListVal([]cty.Value{
+					cty.ObjectVal(map[string]cty.Value{"attr": cty.StringVal("x")}),
+					cty.ObjectVal(map[string]cty.Value{"attr": cty.StringVal("y")}),
+				}),
+			}),
+			After: cty.ObjectVal(map[string]cty.Value{
+				"list": cty.ListVal([]cty.Value{
+					cty.ObjectVal(map[string]cty.Value{"attr": cty.StringVal("y")}),
+					cty.ObjectVal(map[string]cty.Value{"attr": cty.StringVal("z")}),
+				}),
+			}),
+			RequiredReplace: cty.NewPathSet(),
+			Tainted:         false,
+			Schema: &configschema.Block{
+				BlockTypes: map[string]*configschema.NestedBlock{
+					"list": {
+						Block: configschema.Block{
+							Attributes: map[string]*configschema.Attribute{
+								"attr": {
+									Type:     cty.String,
+									Required: true,
+								},
+							},
+						},
+						Nesting: configschema.NestingList,
+					},
+				},
+			},
+			ExpectedOutput: `  # test_instance.example will be updated in-place
+  ~ resource "test_instance" "example" {
+      ~ list {
+          ~ attr = "x" -> "y"
+        }
+      ~ list {
+          ~ attr = "y" -> "z"
+        }
+    }
 `,
 		},
 	}

command/init_test.go

@@ -1310,3 +1310,33 @@ func TestInit_012UpgradeNeededInAutomation(t *testing.T) {
 		t.Errorf("looks like we incorrectly gave an upgrade command to run:\n%s", output)
 	}
 }
+
+func TestInit_syntaxErrorVersionSniff(t *testing.T) {
+	// Create a temporary working directory that is empty
+	td := tempDir(t)
+	copy.CopyDir(testFixturePath("init-sniff-version-error"), td)
+	defer os.RemoveAll(td)
+	defer testChdir(t, td)()
+
+	ui := new(cli.MockUi)
+	c := &InitCommand{
+		Meta: Meta{
+			testingOverrides: metaOverridesForProvider(testProvider()),
+			Ui:               ui,
+		},
+	}
+
+	args := []string{}
+	if code := c.Run(args); code != 0 {
+		t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
+	}
+
+	// Check output.
+	// Currently, this lands in the "upgrade may be needed" codepath, because
+	// the intentional syntax error in our test fixture is something that
+	// "terraform 0.12upgrade" could fix.
+	output := ui.OutputWriter.String()
+	if got, want := output, "Terraform has initialized, but configuration upgrades may be needed"; !strings.Contains(got, want) {
+		t.Fatalf("wrong output\ngot:\n%s\n\nwant: message containing %q", got, want)
+	}
+}

command/test-fixtures/init-sniff-version-error/init-sniff-version-error.tf

@@ -0,0 +1,9 @@
+# The following is invalid because we don't permit multiple nested blocks
+# all one one line. Instead, we require the backend block to be on a line
+# of its own.
+# The purpose of this test case is to see that HCL still produces a valid-enough
+# AST that we can try to sniff in this block for a terraform_version argument
+# without crashing, since we do that during init to try to give a better
+# error message if we detect that the configuration is for a newer Terraform
+# version.
+terraform { backend "local" {} }

command/ui_input.go

@@ -12,6 +12,7 @@ import (
 	"os/signal"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"unicode"
 
 	"github.com/hashicorp/terraform/terraform"
@@ -30,10 +31,13 @@ type UIInput struct {
 	Colorize *colorstring.Colorize
 
 	// Reader and Writer for IO. If these aren't set, they will default to
-	// Stdout and Stderr respectively.
+	// Stdin and Stdout respectively.
 	Reader io.Reader
 	Writer io.Writer
 
+	listening int32
+	result    chan string
+
 	interrupted bool
 	l           sync.Mutex
 	once        sync.Once
@@ -117,20 +121,24 @@ func (i *UIInput) Input(ctx context.Context, opts *terraform.InputOpts) (string,
 	}
 
 	// Listen for the input in a goroutine. This will allow us to
-	// interrupt this if we are interrupted (SIGINT)
-	result := make(chan string, 1)
+	// interrupt this if we are interrupted (SIGINT).
 	go func() {
+		if !atomic.CompareAndSwapInt32(&i.listening, 0, 1) {
+			return // We are already listening for input.
+		}
+		defer atomic.CompareAndSwapInt32(&i.listening, 1, 0)
+
 		buf := bufio.NewReader(r)
 		line, err := buf.ReadString('\n')
 		if err != nil {
 			log.Printf("[ERR] UIInput scan err: %s", err)
 		}
 
-		result <- strings.TrimRightFunc(line, unicode.IsSpace)
+		i.result <- strings.TrimRightFunc(line, unicode.IsSpace)
 	}()
 
 	select {
-	case line := <-result:
+	case line := <-i.result:
 		fmt.Fprint(w, "\n")
 
 		if line == "" {
@@ -157,6 +165,8 @@ func (i *UIInput) Input(ctx context.Context, opts *terraform.InputOpts) (string,
 }
 
 func (i *UIInput) init() {
+	i.result = make(chan string)
+
 	if i.Colorize == nil {
 		i.Colorize = &colorstring.Colorize{
 			Colors:  colorstring.DefaultColors,

command/ui_input_test.go

@@ -3,7 +3,11 @@ package command
 import (
 	"bytes"
 	"context"
+	"fmt"
+	"io"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/terraform/terraform"
 )
@@ -20,11 +24,61 @@ func TestUIInputInput(t *testing.T) {
 
 	v, err := i.Input(context.Background(), &terraform.InputOpts{})
 	if err != nil {
-		t.Fatalf("err: %s", err)
+		t.Fatalf("unexpected error: %v", err)
 	}
 
 	if v != "foo" {
-		t.Fatalf("bad: %#v", v)
+		t.Fatalf("unexpected input: %s", v)
+	}
+}
+
+func TestUIInputInput_canceled(t *testing.T) {
+	r, w := io.Pipe()
+	i := &UIInput{
+		Reader: r,
+		Writer: bytes.NewBuffer(nil),
+	}
+
+	// Make a context that can be canceled.
+	ctx, cancel := context.WithCancel(context.Background())
+
+	go func() {
+		// Cancel the context after 2 seconds.
+		time.Sleep(2 * time.Second)
+		cancel()
+	}()
+
+	// Get input until the context is canceled.
+	v, err := i.Input(ctx, &terraform.InputOpts{})
+	if err != context.Canceled {
+		t.Fatalf("expected a context.Canceled error, got: %v", err)
+	}
+
+	// As the context was canceled v should be empty.
+	if v != "" {
+		t.Fatalf("unexpected input: %s", v)
+	}
+
+	// As the context was canceled we should still be listening.
+	listening := atomic.LoadInt32(&i.listening)
+	if listening != 1 {
+		t.Fatalf("expected listening to be 1, got: %d", listening)
+	}
+
+	go func() {
+		// Fake input is given after 1 second.
+		time.Sleep(time.Second)
+		fmt.Fprint(w, "foo\n")
+		w.Close()
+	}()
+
+	v, err = i.Input(context.Background(), &terraform.InputOpts{})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if v != "foo" {
+		t.Fatalf("unexpected input: %s", v)
 	}
 }
 
@@ -36,10 +90,10 @@ func TestUIInputInput_spaces(t *testing.T) {
 
 	v, err := i.Input(context.Background(), &terraform.InputOpts{})
 	if err != nil {
-		t.Fatalf("err: %s", err)
+		t.Fatalf("unexpected error: %v", err)
 	}
 
 	if v != "foo bar" {
-		t.Fatalf("bad: %#v", v)
+		t.Fatalf("unexpected input: %s", v)
 	}
 }

config/resource_mode_string.go

@@ -4,6 +4,14 @@ package config
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[ManagedResourceMode-0]
+	_ = x[DataResourceMode-1]
+}
+
 const _ResourceMode_name = "ManagedResourceModeDataResourceMode"
 
 var _ResourceMode_index = [...]uint8{0, 19, 35}

configs/configupgrade/analysis.go

@@ -232,9 +232,13 @@ func (u *Upgrader) analyze(ms ModuleSources) (*analysis, error) {
 		}
 	}
 
-	providerFactories, err := u.Providers.ResolveProviders(m.PluginRequirements())
-	if err != nil {
-		return nil, fmt.Errorf("error resolving providers: %s", err)
+	providerFactories, errs := u.Providers.ResolveProviders(m.PluginRequirements())
+	if len(errs) > 0 {
+		var errorsMsg string
+		for _, err := range errs {
+			errorsMsg += fmt.Sprintf("\n- %s", err)
+		}
+		return nil, fmt.Errorf("error resolving providers:\n%s", errorsMsg)
 	}
 
 	for name, fn := range providerFactories {

configs/configupgrade/test-fixtures/valid/indexed-splat/input/indexed-splat.tf

@@ -0,0 +1,16 @@
+resource "test_instance" "first_many" {
+  count = 2
+}
+
+resource "test_instance" "one" {
+  image = "${test_instance.first_many.*.id[0]}"
+}
+
+resource "test_instance" "splat_of_one" {
+  image = "${test_instance.one.*.id[0]}"
+}
+
+resource "test_instance" "second_many" {
+  count = "${length(test_instance.first_many)}"
+  security_groups = "${test_instance.first_many.*.id[count.index]}"
+}

configs/configupgrade/test-fixtures/valid/indexed-splat/want/indexed-splat.tf

@@ -0,0 +1,16 @@
+resource "test_instance" "first_many" {
+  count = 2
+}
+
+resource "test_instance" "one" {
+  image = test_instance.first_many[0].id
+}
+
+resource "test_instance" "splat_of_one" {
+  image = test_instance.one.*.id[0]
+}
+
+resource "test_instance" "second_many" {
+  count           = length(test_instance.first_many)
+  security_groups = test_instance.first_many[count.index].id
+}

configs/configupgrade/test-fixtures/valid/indexed-splat/want/versions.tf

@@ -0,0 +1,3 @@
+terraform {
+  required_version = ">= 0.12"
+}

configs/configupgrade/test-fixtures/valid/inline-comments/input/main.tf

@@ -0,0 +1,25 @@
+variable "list" {
+  type = "list"
+
+  default = [
+    "foo", # I am a comment
+    "bar", # I am also a comment
+    "baz",
+  ]
+}
+
+variable "list2" {
+  type = "list"
+
+  default = [
+    "foo",
+    "bar",
+    "baz",
+  ]
+}
+
+variable "list_the_third" {
+  type = "list"
+
+  default = ["foo", "bar", "baz"]
+}

configs/configupgrade/test-fixtures/valid/inline-comments/want/main.tf

@@ -0,0 +1,25 @@
+variable "list" {
+  type = list(string)
+
+  default = [
+    "foo", # I am a comment
+    "bar", # I am also a comment
+    "baz",
+  ]
+}
+
+variable "list2" {
+  type = list(string)
+
+  default = [
+    "foo",
+    "bar",
+    "baz",
+  ]
+}
+
+variable "list_the_third" {
+  type = list(string)
+
+  default = ["foo", "bar", "baz"]
+}

configs/configupgrade/test-fixtures/valid/inline-comments/want/versions.tf

@@ -0,0 +1,3 @@
+terraform {
+  required_version = ">= 0.12"
+}

configs/configupgrade/upgrade_expr.go

@@ -168,10 +168,17 @@ Value:
 			src, moreDiags := upgradeExpr(node, filename, interp, an)
 			diags = diags.Append(moreDiags)
 			buf.Write(src)
-			if multiline {
-				buf.WriteString(",\n")
-			} else if i < len(tv.List)-1 {
-				buf.WriteString(", ")
+			if lit, ok := node.(*hcl1ast.LiteralType); ok && lit.LineComment != nil {
+				for _, comment := range lit.LineComment.List {
+					buf.WriteString(", " + comment.Text)
+					buf.WriteString("\n")
+				}
+			} else {
+				if multiline {
+					buf.WriteString(",\n")
+				} else if i < len(tv.List)-1 {
+					buf.WriteString(", ")
+				}
 			}
 		}
 		buf.WriteString("]")
@@ -248,81 +255,21 @@ Value:
 		// safe to do so.
 		parts := strings.Split(tv.Name, ".")
 
-		// First we need to deal with the .count pseudo-attributes that 0.11 and
-		// prior allowed for resources. These no longer exist, because they
-		// don't do anything we can't do with the length(...) function.
-		if len(parts) > 0 {
-			var rAddr addrs.Resource
-			switch parts[0] {
-			case "data":
-				if len(parts) == 4 && parts[3] == "count" {
-					rAddr.Mode = addrs.DataResourceMode
-					rAddr.Type = parts[1]
-					rAddr.Name = parts[2]
-				}
-			default:
-				if len(parts) == 3 && parts[2] == "count" {
-					rAddr.Mode = addrs.ManagedResourceMode
-					rAddr.Type = parts[0]
-					rAddr.Name = parts[1]
-				}
-			}
-
-			// We need to check if the thing being referenced is actually an
-			// existing resource, because other three-part traversals might
-			// coincidentally end with "count".
-			if hasCount, exists := an.ResourceHasCount[rAddr]; exists {
-				if hasCount {
-					buf.WriteString("length(")
-					buf.WriteString(rAddr.String())
-					buf.WriteString(")")
-				} else {
-					// If the resource does not have count, the .count
-					// attr would've always returned 1 before.
-					buf.WriteString("1")
-				}
-				break Value
-			}
+		transformed := transformCountPseudoAttribute(&buf, parts, an)
+		if transformed {
+			break Value
 		}
 
 		parts = upgradeTraversalParts(parts, an) // might add/remove/change parts
-		first, remain := parts[0], parts[1:]
-		buf.WriteString(first)
-		seenSplat := false
-		for _, part := range remain {
-			if part == "*" {
-				seenSplat = true
-				buf.WriteString(".*")
-				continue
-			}
 
-			// Other special cases apply only if we've not previously
-			// seen a splat expression marker, since attribute vs. index
-			// syntax have different interpretations after a simple splat.
-			if !seenSplat {
-				if v, err := strconv.Atoi(part); err == nil {
-					// Looks like it's old-style index traversal syntax foo.0.bar
-					// so we'll replace with canonical index syntax foo[0].bar.
-					fmt.Fprintf(&buf, "[%d]", v)
-					continue
-				}
-				if !hcl2syntax.ValidIdentifier(part) {
-					// This should be rare since HIL's identifier syntax is _close_
-					// to HCL2's, but we'll get here if one of the intervening
-					// parts is not a valid identifier in isolation, since HIL
-					// did not consider these to be separate identifiers.
-					// e.g. foo.1bar would be invalid in HCL2; must instead be foo["1bar"].
-					buf.WriteByte('[')
-					printQuotedString(&buf, part)
-					buf.WriteByte(']')
-					continue
-				}
-			}
-
-			buf.WriteByte('.')
-			buf.WriteString(part)
+		vDiags := validateHilAddress(tv.Name, filename)
+		if len(vDiags) > 0 {
+			diags = diags.Append(vDiags)
+			break
 		}
 
+		printHilTraversalPartsAsHcl2(&buf, parts)
+
 	case *hilast.Arithmetic:
 		op, exists := hilArithmeticOpSyms[tv.Op]
 		if !exists {
@@ -553,14 +500,74 @@ Value:
 		buf.Write(falseSrc)
 
 	case *hilast.Index:
-		targetSrc, exprDiags := upgradeExpr(tv.Target, filename, true, an)
-		diags = diags.Append(exprDiags)
+		target, ok := tv.Target.(*hilast.VariableAccess)
+		if !ok {
+			panic(fmt.Sprintf("Index node with unsupported target type (%T)", tv.Target))
+		}
+		parts := strings.Split(target.Name, ".")
+
 		keySrc, exprDiags := upgradeExpr(tv.Key, filename, true, an)
 		diags = diags.Append(exprDiags)
-		buf.Write(targetSrc)
-		buf.WriteString("[")
-		buf.Write(keySrc)
-		buf.WriteString("]")
+
+		transformed := transformCountPseudoAttribute(&buf, parts, an)
+		if transformed {
+			break Value
+		}
+
+		parts = upgradeTraversalParts(parts, an) // might add/remove/change parts
+
+		vDiags := validateHilAddress(target.Name, filename)
+		if len(vDiags) > 0 {
+			diags = diags.Append(vDiags)
+			break
+		}
+
+		first, remain := parts[0], parts[1:]
+
+		var rAddr addrs.Resource
+		switch parts[0] {
+		case "data":
+			if len(parts) == 5 && parts[3] == "*" {
+				rAddr.Mode = addrs.DataResourceMode
+				rAddr.Type = parts[1]
+				rAddr.Name = parts[2]
+			}
+		default:
+			if len(parts) == 4 && parts[2] == "*" {
+				rAddr.Mode = addrs.ManagedResourceMode
+				rAddr.Type = parts[0]
+				rAddr.Name = parts[1]
+			}
+		}
+
+		// We need to check if the thing being referenced has count
+		// to retain backward compatibility
+		hasCount := false
+		if v, exists := an.ResourceHasCount[rAddr]; exists {
+			hasCount = v
+		}
+
+		hasSplat := false
+
+		buf.WriteString(first)
+		for _, part := range remain {
+			// Attempt to convert old-style splat indexing to new one
+			// e.g. res.label.*.attr[idx] to res.label[idx].attr
+			if part == "*" && hasCount {
+				hasSplat = true
+				buf.WriteString(fmt.Sprintf("[%s]", keySrc))
+				continue
+			}
+
+			buf.WriteByte('.')
+			buf.WriteString(part)
+		}
+
+		if !hasSplat {
+			buf.WriteString("[")
+			buf.Write(keySrc)
+			buf.WriteString("]")
+		}
 
 	case *hilast.Output:
 		if len(tv.Exprs) == 1 {
@@ -614,6 +621,122 @@ Value:
 	return buf.Bytes(), diags
 }
 
+func validateHilAddress(address, filename string) tfdiags.Diagnostics {
+	parts := strings.Split(address, ".")
+	var diags tfdiags.Diagnostics
+
+	label, ok := getResourceLabel(parts)
+	if ok && !hcl2syntax.ValidIdentifier(label) {
+		// We can't get any useful source location out of HIL unfortunately
+		diags = diags.Append(tfdiags.Sourceless(
+			tfdiags.Error,
+			fmt.Sprintf("Invalid address (%s) in ./%s", address, filename),
+			// The label could be invalid for another reason
+			// but this is the most likely, so we add it as hint
+			"Names of objects (resources, modules, etc) may no longer start with digits."))
+	}
+
+	return diags
+}
+
+func getResourceLabel(parts []string) (string, bool) {
+	if len(parts) < 1 {
+		return "", false
+	}
+
+	if parts[0] == "data" {
+		if len(parts) < 3 {
+			return "", false
+		}
+		return parts[2], true
+	}
+
+	if len(parts) < 2 {
+		return "", false
+	}
+
+	return parts[1], true
+}
+
+// transformCountPseudoAttribute deals with the .count pseudo-attributes
+// that 0.11 and prior allowed for resources. These no longer exist,
+// because they don't do anything we can't do with the length(...) function.
+func transformCountPseudoAttribute(buf *bytes.Buffer, parts []string, an *analysis) (transformed bool) {
+	if len(parts) > 0 {
+		var rAddr addrs.Resource
+		switch parts[0] {
+		case "data":
+			if len(parts) == 4 && parts[3] == "count" {
+				rAddr.Mode = addrs.DataResourceMode
+				rAddr.Type = parts[1]
+				rAddr.Name = parts[2]
+			}
+		default:
+			if len(parts) == 3 && parts[2] == "count" {
+				rAddr.Mode = addrs.ManagedResourceMode
+				rAddr.Type = parts[0]
+				rAddr.Name = parts[1]
+			}
+		}
+		// We need to check if the thing being referenced is actually an
+		// existing resource, because other three-part traversals might
+		// coincidentally end with "count".
+		if hasCount, exists := an.ResourceHasCount[rAddr]; exists {
+			if hasCount {
+				buf.WriteString("length(")
+				buf.WriteString(rAddr.String())
+				buf.WriteString(")")
+			} else {
+				// If the resource does not have count, the .count
+				// attr would've always returned 1 before.
+				buf.WriteString("1")
+			}
+			transformed = true
+			return
+		}
+	}
+	return
+}
+
+func printHilTraversalPartsAsHcl2(buf *bytes.Buffer, parts []string) {
+	first, remain := parts[0], parts[1:]
+	buf.WriteString(first)
+	seenSplat := false
+	for _, part := range remain {
+		if part == "*" {
+			seenSplat = true
+			buf.WriteString(".*")
+			continue
+		}
+
+		// Other special cases apply only if we've not previously
+		// seen a splat expression marker, since attribute vs. index
+		// syntax have different interpretations after a simple splat.
+		if !seenSplat {
+			if v, err := strconv.Atoi(part); err == nil {
+				// Looks like it's old-style index traversal syntax foo.0.bar
+				// so we'll replace with canonical index syntax foo[0].bar.
+				fmt.Fprintf(buf, "[%d]", v)
+				continue
+			}
+			if !hcl2syntax.ValidIdentifier(part) {
+				// This should be rare since HIL's identifier syntax is _close_
+				// to HCL2's, but we'll get here if one of the intervening
+				// parts is not a valid identifier in isolation, since HIL
+				// did not consider these to be separate identifiers.
+				// e.g. foo.1bar would be invalid in HCL2; must instead be foo["1bar"].
+				buf.WriteByte('[')
+				printQuotedString(buf, part)
+				buf.WriteByte(']')
+				continue
+			}
+		}
+
+		buf.WriteByte('.')
+		buf.WriteString(part)
+	}
+}
+
 func upgradeHeredocBody(buf *bytes.Buffer, val *hilast.Output, filename string, an *analysis) tfdiags.Diagnostics {
 	var diags tfdiags.Diagnostics
 

configs/configupgrade/upgrade_test.go

@@ -2,6 +2,7 @@ package configupgrade
 
 import (
 	"bytes"
+	"flag"
 	"io"
 	"io/ioutil"
 	"log"
@@ -286,6 +287,7 @@ func init() {
 }
 
 func TestMain(m *testing.M) {
+	flag.Parse()
 	if testing.Verbose() {
 		// if we're verbose, use the logging requested by TF_LOG
 		logging.SetOutput()

configs/provisioneronfailure_string.go

@@ -4,6 +4,15 @@ package configs
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[ProvisionerOnFailureInvalid-0]
+	_ = x[ProvisionerOnFailureContinue-1]
+	_ = x[ProvisionerOnFailureFail-2]
+}
+
 const _ProvisionerOnFailure_name = "ProvisionerOnFailureInvalidProvisionerOnFailureContinueProvisionerOnFailureFail"
 
 var _ProvisionerOnFailure_index = [...]uint8{0, 27, 55, 79}

configs/provisionerwhen_string.go

@@ -4,6 +4,15 @@ package configs
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[ProvisionerWhenInvalid-0]
+	_ = x[ProvisionerWhenCreate-1]
+	_ = x[ProvisionerWhenDestroy-2]
+}
+
 const _ProvisionerWhen_name = "ProvisionerWhenInvalidProvisionerWhenCreateProvisionerWhenDestroy"
 
 var _ProvisionerWhen_index = [...]uint8{0, 22, 43, 65}

configs/variabletypehint_string.go

@@ -4,6 +4,16 @@ package configs
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[TypeHintNone-0]
+	_ = x[TypeHintString-83]
+	_ = x[TypeHintList-76]
+	_ = x[TypeHintMap-77]
+}
+
 const (
 	_VariableTypeHint_name_0 = "TypeHintNone"
 	_VariableTypeHint_name_1 = "TypeHintListTypeHintMap"

e2e/e2e.go

@@ -9,7 +9,10 @@ import (
 	"os/exec"
 	"path/filepath"
 
-	tfcore "github.com/hashicorp/terraform/terraform"
+	"github.com/hashicorp/terraform/plans"
+	"github.com/hashicorp/terraform/plans/planfile"
+	"github.com/hashicorp/terraform/states"
+	"github.com/hashicorp/terraform/states/statefile"
 )
 
 // Type binary represents the combination of a compiled binary
@@ -48,6 +51,12 @@ func NewBinary(binaryPath, workingDir string) *binary {
 			return nil
 		}
 
+		if filepath.Base(path) == ".exists" {
+			// We use this file just to let git know the "empty" fixture
+			// exists. It is not used by any test.
+			return nil
+		}
+
 		srcFn := path
 
 		path, err = filepath.Rel(workingDir, path)
@@ -170,43 +179,52 @@ func (b *binary) FileExists(path ...string) bool {
 
 // LocalState is a helper for easily reading the local backend's state file
 // terraform.tfstate from the working directory.
-func (b *binary) LocalState() (*tfcore.State, error) {
+func (b *binary) LocalState() (*states.State, error) {
 	f, err := b.OpenFile("terraform.tfstate")
 	if err != nil {
 		return nil, err
 	}
 	defer f.Close()
-	return tfcore.ReadState(f)
+
+	stateFile, err := statefile.Read(f)
+	if err != nil {
+		return nil, fmt.Errorf("Error reading statefile: %s", err)
+	}
+	return stateFile.State, nil
 }
 
 // Plan is a helper for easily reading a plan file from the working directory.
-func (b *binary) Plan(path ...string) (*tfcore.Plan, error) {
-	f, err := b.OpenFile(path...)
+func (b *binary) Plan(path string) (*plans.Plan, error) {
+	path = b.Path(path)
+	pr, err := planfile.Open(path)
 	if err != nil {
 		return nil, err
 	}
-	defer f.Close()
-	return tfcore.ReadPlan(f)
+	plan, err := pr.ReadPlan()
+	if err != nil {
+		return nil, err
+	}
+	return plan, nil
 }
 
 // SetLocalState is a helper for easily writing to the file the local backend
 // uses for state in the working directory. This does not go through the
 // actual local backend code, so processing such as management of serials
 // does not apply and the given state will simply be written verbatim.
-func (b *binary) SetLocalState(state *tfcore.State) error {
+func (b *binary) SetLocalState(state *states.State) error {
 	path := b.Path("terraform.tfstate")
 	f, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, os.ModePerm)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to create temporary state file %s: %s", path, err)
 	}
-	defer func() {
-		err := f.Close()
-		if err != nil {
-			panic(fmt.Sprintf("failed to close state file after writing: %s", err))
-		}
-	}()
+	defer f.Close()
 
-	return tfcore.WriteState(state, f)
+	sf := &statefile.File{
+		Serial:  0,
+		Lineage: "fake-for-testing",
+		State:   state,
+	}
+	return statefile.Write(sf, f)
 }
 
 // Close cleans up the temporary resources associated with the object,

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2
 	github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
-	github.com/aws/aws-sdk-go v1.16.36
+	github.com/aws/aws-sdk-go v1.19.18
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/chzyer/logex v1.1.10 // indirect
@@ -46,21 +46,21 @@ require (
 	github.com/hashicorp/go-azure-helpers v0.0.0-20190129193224-166dfd221bb2
 	github.com/hashicorp/go-checkpoint v0.5.0
 	github.com/hashicorp/go-cleanhttp v0.5.0
-	github.com/hashicorp/go-getter v1.1.0
+	github.com/hashicorp/go-getter v1.3.0
 	github.com/hashicorp/go-hclog v0.0.0-20181001195459-61d530d6c27f
 	github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa // indirect
-	github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c // indirect
+	github.com/hashicorp/go-msgpack v0.5.4 // indirect
 	github.com/hashicorp/go-multierror v1.0.0
-	github.com/hashicorp/go-plugin v0.0.0-20190322172744-52e1c4730856
+	github.com/hashicorp/go-plugin v1.0.1-0.20190430211030-5692942914bb
 	github.com/hashicorp/go-retryablehttp v0.5.2
 	github.com/hashicorp/go-rootcerts v1.0.0
 	github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 // indirect
-	github.com/hashicorp/go-tfe v0.3.14
+	github.com/hashicorp/go-tfe v0.3.16
 	github.com/hashicorp/go-uuid v1.0.1
 	github.com/hashicorp/go-version v1.1.0
 	github.com/hashicorp/golang-lru v0.5.0 // indirect
 	github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f
-	github.com/hashicorp/hcl2 v0.0.0-20190416162332-2c5a4b7d729a
+	github.com/hashicorp/hcl2 v0.0.0-20190514214226-6a61d80ae3d0
 	github.com/hashicorp/hil v0.0.0-20190212112733-ab17b08d6590
 	github.com/hashicorp/logutils v1.0.0
 	github.com/hashicorp/memberlist v0.1.0 // indirect
@@ -107,16 +107,13 @@ require (
 	github.com/xanzy/ssh-agent v0.2.1
 	github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 // indirect
 	github.com/xlab/treeprint v0.0.0-20161029104018-1d6e34225557
-	github.com/zclconf/go-cty v0.0.0-20190320224746-fd76348b9329
+	github.com/zclconf/go-cty v0.0.0-20190430221426-d36a6f0dbffd
 	go.uber.org/atomic v1.3.2 // indirect
 	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.9.1 // indirect
-	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
-	golang.org/x/net v0.0.0-20190311183353-d8887717615a
+	golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734
+	golang.org/x/net v0.0.0-20190502183928-7f726cade0ab
 	golang.org/x/oauth2 v0.0.0-20190220154721-9b3c75971fc9
 	google.golang.org/api v0.1.0
 	google.golang.org/grpc v1.18.0
-	gopkg.in/vmihailenco/msgpack.v2 v2.9.1 // indirect
-	labix.org/v2/mgo v0.0.0-20140701140051-000000000287 // indirect
-	launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect
 )

go.sum

@@ -51,6 +51,8 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI
 github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM=
 github.com/aws/aws-sdk-go v1.16.36 h1:POeH34ZME++pr7GBGh+ZO6Y5kOwSMQpqp5BGUgooJ6k=
 github.com/aws/aws-sdk-go v1.16.36/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.19.18 h1:Hb3+b9HCqrOrbAtFstUWg7H5TQ+/EcklJtE8VShVs8o=
+github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
@@ -165,20 +167,20 @@ github.com/hashicorp/go-checkpoint v0.5.0 h1:MFYpPZCnQqQTE18jFwSII6eUQrD/oxMFp3m
 github.com/hashicorp/go-checkpoint v0.5.0/go.mod h1:7nfLNL10NsxqO4iWuW6tWW0HjZuDrwkBuEQsVcpCOgg=
 github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-getter v1.1.0 h1:iGVeg7L4V5FTFV3D6w+1NAyvth7BIWWSzD60pWloe2Q=
-github.com/hashicorp/go-getter v1.1.0/go.mod h1:q+PoBhh16brIKwJS9kt18jEtXHTg2EGkmrA9P7HVS+U=
+github.com/hashicorp/go-getter v1.3.0 h1:pFMSFlI9l5NaeuzkpE3L7BYk9qQ9juTAgXW/H0cqxcU=
+github.com/hashicorp/go-getter v1.3.0/go.mod h1:/O1k/AizTN0QmfEKknCYGvICeyKUDqCYA8vvWtGWDeQ=
 github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI=
 github.com/hashicorp/go-hclog v0.0.0-20181001195459-61d530d6c27f h1:Yv9YzBlAETjy6AOX9eLBZ3nshNVRREgerT/3nvxlGho=
 github.com/hashicorp/go-hclog v0.0.0-20181001195459-61d530d6c27f/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa h1:0nA8i+6Rwqaq9xlpmVxxTwk6rxiEhX+E6Wh4vPNHiS8=
 github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa/go.mod h1:6ij3Z20p+OhOkCSrA0gImAWoHYQRGbnlcuk6XYTiaRw=
-github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c h1:BTAbnbegUIMB6xmQCwWE8yRzbA4XSpnZY5hvRJC188I=
-github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-msgpack v0.5.4 h1:SFT72YqIkOcLdWJUYcriVX7hbrZpwc/f7h8aW2NUqrA=
+github.com/hashicorp/go-msgpack v0.5.4/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-plugin v0.0.0-20190322172744-52e1c4730856 h1:FHiCaU46W1WoqApsaGGIKbNkhQ6v71hJrOf2INQMLUo=
-github.com/hashicorp/go-plugin v0.0.0-20190322172744-52e1c4730856/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY=
+github.com/hashicorp/go-plugin v1.0.1-0.20190430211030-5692942914bb h1:Zg2pmmk0lrLFL85lQGt08bOUBpIBaVs6/psiAyx0c4w=
+github.com/hashicorp/go-plugin v1.0.1-0.20190430211030-5692942914bb/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY=
 github.com/hashicorp/go-retryablehttp v0.5.2 h1:AoISa4P4IsW0/m4T6St8Yw38gTl5GtBAgfkhYh1xAz4=
 github.com/hashicorp/go-retryablehttp v0.5.2/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI=
@@ -189,8 +191,8 @@ github.com/hashicorp/go-slug v0.3.0 h1:L0c+AvH/J64iMNF4VqRaRku2DMTEuHioPVS7kMjWI
 github.com/hashicorp/go-slug v0.3.0/go.mod h1:I5tq5Lv0E2xcNXNkmx7BSfzi1PsJ2cNjs3cC3LwyhK8=
 github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 h1:7YOlAIO2YWnJZkQp7B5eFykaIY7C9JndqAFQyVV5BhM=
 github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
-github.com/hashicorp/go-tfe v0.3.14 h1:1eWmq4RAICGufydNUWu7ahb0gtq24pN9jatD2FkdxdE=
-github.com/hashicorp/go-tfe v0.3.14/go.mod h1:SuPHR+OcxvzBZNye7nGPfwZTEyd3rWPfLVbCgyZPezM=
+github.com/hashicorp/go-tfe v0.3.16 h1:GS2yv580p0co4j3FBVaC6Zahd9mxdCGehhJ0qqzFMH0=
+github.com/hashicorp/go-tfe v0.3.16/go.mod h1:SuPHR+OcxvzBZNye7nGPfwZTEyd3rWPfLVbCgyZPezM=
 github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
@@ -202,8 +204,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f h1:UdxlrJz4JOnY8W+DbLISwf2B8WXEolNRA8BGCwI9jws=
 github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
 github.com/hashicorp/hcl2 v0.0.0-20181208003705-670926858200/go.mod h1:ShfpTh661oAaxo7VcNxg0zcZW6jvMa7Moy2oFx7e5dE=
-github.com/hashicorp/hcl2 v0.0.0-20190416162332-2c5a4b7d729a h1:doKt9ZBCYgYQrGK6CqJsEB+8xqm3WoFyKu4TPZlyymg=
-github.com/hashicorp/hcl2 v0.0.0-20190416162332-2c5a4b7d729a/go.mod h1:HtEzazM5AZ9fviNEof8QZB4T1Vz9UhHrGhnMPzl//Ek=
+github.com/hashicorp/hcl2 v0.0.0-20190514214226-6a61d80ae3d0 h1:5Hbw6YJOLDh8XV2z9woGJHyCpCthyYmaG6i97/jvw2g=
+github.com/hashicorp/hcl2 v0.0.0-20190514214226-6a61d80ae3d0/go.mod h1:4oI94iqF3GB10QScn46WqbG0kgTUpha97SAzzg2+2ec=
 github.com/hashicorp/hil v0.0.0-20190212112733-ab17b08d6590 h1:2yzhWGdgQUWZUCNK+AoO35V+HTsgEmcM4J9IkArh7PI=
 github.com/hashicorp/hil v0.0.0-20190212112733-ab17b08d6590/go.mod h1:n2TSygSNwsLJ76m8qFXTSc7beTb+auJxYdqrnoqwZWE=
 github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y=
@@ -399,10 +401,10 @@ github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q
 github.com/xlab/treeprint v0.0.0-20161029104018-1d6e34225557 h1:Jpn2j6wHkC9wJv5iMfJhKqrZJx3TahFx+7sbZ7zQdxs=
 github.com/xlab/treeprint v0.0.0-20161029104018-1d6e34225557/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/zclconf/go-cty v0.0.0-20181129180422-88fbe721e0f8/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
-github.com/zclconf/go-cty v0.0.0-20190124225737-a385d646c1e9 h1:hHCAGde+QfwbqXSAqOmBd4NlOrJ6nmjWp+Nu408ezD4=
-github.com/zclconf/go-cty v0.0.0-20190124225737-a385d646c1e9/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
-github.com/zclconf/go-cty v0.0.0-20190320224746-fd76348b9329 h1:ne520NlvoncW5zfBGkmP4EJhyd6ruSaSyhzobv0Vz9w=
-github.com/zclconf/go-cty v0.0.0-20190320224746-fd76348b9329/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
+github.com/zclconf/go-cty v0.0.0-20190426224007-b18a157db9e2 h1:Ai1LhlYNEqE39zGU07qHDNJ41iZVPZfZr1dSCoXrp1w=
+github.com/zclconf/go-cty v0.0.0-20190426224007-b18a157db9e2/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
+github.com/zclconf/go-cty v0.0.0-20190430221426-d36a6f0dbffd h1:NZOOU7h+pDtcKo6xlqm8PwnarS8nJ+6+I83jT8ZfLPI=
+github.com/zclconf/go-cty v0.0.0-20190430221426-d36a6f0dbffd/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
 go.opencensus.io v0.18.0 h1:Mk5rgZcggtbvtAun5aJzAtjKKN/t0R3jJPlWILlv938=
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
@@ -423,6 +425,8 @@ golang.org/x/crypto v0.0.0-20190222235706-ffb98f73852f h1:qWFY9ZxP3tfI37wYIs/MnI
 golang.org/x/crypto v0.0.0-20190222235706-ffb98f73852f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -438,8 +442,9 @@ golang.org/x/net v0.0.0-20181129055619-fae4c4e3ad76/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190502183928-7f726cade0ab h1:9RfW3ktsOZxgo9YNbBAjq1FWzc/igwEcUzZz8IXgSbk=
+golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE=
@@ -451,6 +456,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -464,16 +471,22 @@ golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0 h1:bzeyCHgoAyjZjAhvTpks+qM7s
 golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82 h1:vsphBvatvfbhlb4PO1BYSr9dzugGxJ/SQHoNufZJq1w=
+golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
@@ -505,8 +518,6 @@ gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/vmihailenco/msgpack.v2 v2.9.1 h1:kb0VV7NuIojvRfzwslQeP3yArBqJHW9tOl4t38VS1jM=
-gopkg.in/vmihailenco/msgpack.v2 v2.9.1/go.mod h1:/3Dn1Npt9+MYyLpYYXjInO/5jvMLamn+AEGwNEOatn8=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -514,9 +525,5 @@ grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJd
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
-labix.org/v2/mgo v0.0.0-20140701140051-000000000287 h1:L0cnkNl4TfAXzvdrqsYEmxOHOCv2p5I3taaReO8BWFs=
-labix.org/v2/mgo v0.0.0-20140701140051-000000000287/go.mod h1:Lg7AYkt1uXJoR9oeSZ3W/8IXLdvOfIITgZnommstyz4=
-launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54=
-launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM=
 sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=

helper/plugin/grpc_provider.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log"
 	"strconv"
 
 	"github.com/zclconf/go-cty/cty"
@@ -51,7 +52,7 @@ func (s *GRPCProviderServer) GetSchema(_ context.Context, req *proto.GetProvider
 	}
 
 	resp.Provider = &proto.Schema{
-		Block: convert.ConfigSchemaToProto(s.getProviderSchemaBlockForCore()),
+		Block: convert.ConfigSchemaToProto(s.getProviderSchemaBlock()),
 	}
 
 	for typ, res := range s.provider.ResourcesMap {
@@ -71,46 +72,26 @@ func (s *GRPCProviderServer) GetSchema(_ context.Context, req *proto.GetProvider
 	return resp, nil
 }
 
-func (s *GRPCProviderServer) getProviderSchemaBlockForCore() *configschema.Block {
+func (s *GRPCProviderServer) getProviderSchemaBlock() *configschema.Block {
 	return schema.InternalMap(s.provider.Schema).CoreConfigSchema()
 }
 
-func (s *GRPCProviderServer) getResourceSchemaBlockForCore(name string) *configschema.Block {
+func (s *GRPCProviderServer) getResourceSchemaBlock(name string) *configschema.Block {
 	res := s.provider.ResourcesMap[name]
 	return res.CoreConfigSchema()
 }
 
-func (s *GRPCProviderServer) getDatasourceSchemaBlockForCore(name string) *configschema.Block {
+func (s *GRPCProviderServer) getDatasourceSchemaBlock(name string) *configschema.Block {
 	dat := s.provider.DataSourcesMap[name]
 	return dat.CoreConfigSchema()
 }
 
-func (s *GRPCProviderServer) getProviderSchemaBlockForShimming() *configschema.Block {
-	newSchema := map[string]*schema.Schema{}
-	for attr, s := range s.provider.Schema {
-		newSchema[attr] = schema.LegacySchema(s)
-	}
-
-	return schema.InternalMap(newSchema).CoreConfigSchema()
-}
-
-func (s *GRPCProviderServer) getResourceSchemaBlockForShimming(name string) *configschema.Block {
-	res := s.provider.ResourcesMap[name]
-	return schema.LegacyResourceSchema(res).CoreConfigSchema()
-}
-
-func (s *GRPCProviderServer) getDatasourceSchemaBlockForShimming(name string) *configschema.Block {
-	dat := s.provider.DataSourcesMap[name]
-	return schema.LegacyResourceSchema(dat).CoreConfigSchema()
-}
-
 func (s *GRPCProviderServer) PrepareProviderConfig(_ context.Context, req *proto.PrepareProviderConfig_Request) (*proto.PrepareProviderConfig_Response, error) {
 	resp := &proto.PrepareProviderConfig_Response{}
 
-	blockForCore := s.getProviderSchemaBlockForCore()
-	blockForShimming := s.getProviderSchemaBlockForShimming()
+	schemaBlock := s.getProviderSchemaBlock()
 
-	configVal, err := msgpack.Unmarshal(req.Config.Msgpack, blockForCore.ImpliedType())
+	configVal, err := msgpack.Unmarshal(req.Config.Msgpack, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -181,7 +162,7 @@ func (s *GRPCProviderServer) PrepareProviderConfig(_ context.Context, req *proto
 		return resp, nil
 	}
 
-	configVal, err = blockForShimming.CoerceValue(configVal)
+	configVal, err = schemaBlock.CoerceValue(configVal)
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -193,12 +174,12 @@ func (s *GRPCProviderServer) PrepareProviderConfig(_ context.Context, req *proto
 		return resp, nil
 	}
 
-	config := terraform.NewResourceConfigShimmed(configVal, blockForShimming)
+	config := terraform.NewResourceConfigShimmed(configVal, schemaBlock)
 
 	warns, errs := s.provider.Validate(config)
 	resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, convert.WarnsAndErrsToProto(warns, errs))
 
-	preparedConfigMP, err := msgpack.Marshal(configVal, blockForCore.ImpliedType())
+	preparedConfigMP, err := msgpack.Marshal(configVal, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -212,16 +193,15 @@ func (s *GRPCProviderServer) PrepareProviderConfig(_ context.Context, req *proto
 func (s *GRPCProviderServer) ValidateResourceTypeConfig(_ context.Context, req *proto.ValidateResourceTypeConfig_Request) (*proto.ValidateResourceTypeConfig_Response, error) {
 	resp := &proto.ValidateResourceTypeConfig_Response{}
 
-	blockForCore := s.getResourceSchemaBlockForCore(req.TypeName)
-	blockForShimming := s.getResourceSchemaBlockForShimming(req.TypeName)
+	schemaBlock := s.getResourceSchemaBlock(req.TypeName)
 
-	configVal, err := msgpack.Unmarshal(req.Config.Msgpack, blockForCore.ImpliedType())
+	configVal, err := msgpack.Unmarshal(req.Config.Msgpack, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
 	}
 
-	config := terraform.NewResourceConfigShimmed(configVal, blockForShimming)
+	config := terraform.NewResourceConfigShimmed(configVal, schemaBlock)
 
 	warns, errs := s.provider.ValidateResource(req.TypeName, config)
 	resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, convert.WarnsAndErrsToProto(warns, errs))
@@ -232,10 +212,9 @@ func (s *GRPCProviderServer) ValidateResourceTypeConfig(_ context.Context, req *
 func (s *GRPCProviderServer) ValidateDataSourceConfig(_ context.Context, req *proto.ValidateDataSourceConfig_Request) (*proto.ValidateDataSourceConfig_Response, error) {
 	resp := &proto.ValidateDataSourceConfig_Response{}
 
-	blockForCore := s.getDatasourceSchemaBlockForCore(req.TypeName)
-	blockForShimming := s.getDatasourceSchemaBlockForShimming(req.TypeName)
+	schemaBlock := s.getDatasourceSchemaBlock(req.TypeName)
 
-	configVal, err := msgpack.Unmarshal(req.Config.Msgpack, blockForCore.ImpliedType())
+	configVal, err := msgpack.Unmarshal(req.Config.Msgpack, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -247,7 +226,7 @@ func (s *GRPCProviderServer) ValidateDataSourceConfig(_ context.Context, req *pr
 		return resp, nil
 	}
 
-	config := terraform.NewResourceConfigShimmed(configVal, blockForShimming)
+	config := terraform.NewResourceConfigShimmed(configVal, schemaBlock)
 
 	warns, errs := s.provider.ValidateDataSource(req.TypeName, config)
 	resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, convert.WarnsAndErrsToProto(warns, errs))
@@ -259,31 +238,32 @@ func (s *GRPCProviderServer) UpgradeResourceState(_ context.Context, req *proto.
 	resp := &proto.UpgradeResourceState_Response{}
 
 	res := s.provider.ResourcesMap[req.TypeName]
-	blockForCore := s.getResourceSchemaBlockForCore(req.TypeName)
-	blockForShimming := s.getResourceSchemaBlockForShimming(req.TypeName)
+	schemaBlock := s.getResourceSchemaBlock(req.TypeName)
 
 	version := int(req.Version)
 
-	var jsonMap map[string]interface{}
+	jsonMap := map[string]interface{}{}
 	var err error
 
-	// if there's a JSON state, we need to decode it.
-	if len(req.RawState.Json) > 0 {
-		err = json.Unmarshal(req.RawState.Json, &jsonMap)
-		if err != nil {
-			resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
-			return resp, nil
-		}
-	}
-
+	switch {
 	// We first need to upgrade a flatmap state if it exists.
 	// There should never be both a JSON and Flatmap state in the request.
-	if req.RawState.Flatmap != nil {
+	case len(req.RawState.Flatmap) > 0:
 		jsonMap, version, err = s.upgradeFlatmapState(version, req.RawState.Flatmap, res)
 		if err != nil {
 			resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 			return resp, nil
 		}
+	// if there's a JSON state, we need to decode it.
+	case len(req.RawState.Json) > 0:
+		err = json.Unmarshal(req.RawState.Json, &jsonMap)
+		if err != nil {
+			resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
+			return resp, nil
+		}
+	default:
+		log.Println("[DEBUG] no state provided to upgrade")
+		return resp, nil
 	}
 
 	// complete the upgrade of the JSON states
@@ -293,16 +273,19 @@ func (s *GRPCProviderServer) UpgradeResourceState(_ context.Context, req *proto.
 		return resp, nil
 	}
 
+	// The provider isn't required to clean out removed fields
+	s.removeAttributes(jsonMap, schemaBlock.ImpliedType())
+
 	// now we need to turn the state into the default json representation, so
 	// that it can be re-decoded using the actual schema.
-	val, err := schema.JSONMapToStateValue(jsonMap, blockForShimming)
+	val, err := schema.JSONMapToStateValue(jsonMap, schemaBlock)
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
 	}
 
 	// encode the final state to the expected msgpack format
-	newStateMP, err := msgpack.Marshal(val, blockForCore.ImpliedType())
+	newStateMP, err := msgpack.Marshal(val, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -325,7 +308,7 @@ func (s *GRPCProviderServer) upgradeFlatmapState(version int, m map[string]strin
 	// first determine if we need to call the legacy MigrateState func
 	requiresMigrate := version < res.SchemaVersion
 
-	schemaType := schema.LegacyResourceSchema(res).CoreConfigSchema().ImpliedType()
+	schemaType := res.CoreConfigSchema().ImpliedType()
 
 	// if there are any StateUpgraders, then we need to only compare
 	// against the first version there
@@ -404,6 +387,57 @@ func (s *GRPCProviderServer) upgradeJSONState(version int, m map[string]interfac
 	return m, nil
 }
 
+// Remove any attributes no longer present in the schema, so that the json can
+// be correctly decoded.
+func (s *GRPCProviderServer) removeAttributes(v interface{}, ty cty.Type) {
+	// we're only concerned with finding maps that corespond to object
+	// attributes
+	switch v := v.(type) {
+	case []interface{}:
+		// If these aren't blocks the next call will be a noop
+		if ty.IsListType() || ty.IsSetType() {
+			eTy := ty.ElementType()
+			for _, eV := range v {
+				s.removeAttributes(eV, eTy)
+			}
+		}
+		return
+	case map[string]interface{}:
+		// map blocks aren't yet supported, but handle this just in case
+		if ty.IsMapType() {
+			eTy := ty.ElementType()
+			for _, eV := range v {
+				s.removeAttributes(eV, eTy)
+			}
+			return
+		}
+
+		if ty == cty.DynamicPseudoType {
+			log.Printf("[DEBUG] ignoring dynamic block: %#v\n", v)
+			return
+		}
+
+		if !ty.IsObjectType() {
+			// This shouldn't happen, and will fail to decode further on, so
+			// there's no need to handle it here.
+			log.Printf("[WARN] unexpected type %#v for map in json state", ty)
+			return
+		}
+
+		attrTypes := ty.AttributeTypes()
+		for attr, attrV := range v {
+			attrTy, ok := attrTypes[attr]
+			if !ok {
+				log.Printf("[DEBUG] attribute %q no longer present in schema", attr)
+				delete(v, attr)
+				continue
+			}
+
+			s.removeAttributes(attrV, attrTy)
+		}
+	}
+}
+
 func (s *GRPCProviderServer) Stop(_ context.Context, _ *proto.Stop_Request) (*proto.Stop_Response, error) {
 	resp := &proto.Stop_Response{}
 
@@ -418,10 +452,9 @@ func (s *GRPCProviderServer) Stop(_ context.Context, _ *proto.Stop_Request) (*pr
 func (s *GRPCProviderServer) Configure(_ context.Context, req *proto.Configure_Request) (*proto.Configure_Response, error) {
 	resp := &proto.Configure_Response{}
 
-	blockForCore := s.getProviderSchemaBlockForCore()
-	blockForShimming := s.getProviderSchemaBlockForShimming()
+	schemaBlock := s.getProviderSchemaBlock()
 
-	configVal, err := msgpack.Unmarshal(req.Config.Msgpack, blockForCore.ImpliedType())
+	configVal, err := msgpack.Unmarshal(req.Config.Msgpack, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -435,7 +468,7 @@ func (s *GRPCProviderServer) Configure(_ context.Context, req *proto.Configure_R
 		return resp, nil
 	}
 
-	config := terraform.NewResourceConfigShimmed(configVal, blockForShimming)
+	config := terraform.NewResourceConfigShimmed(configVal, schemaBlock)
 	err = s.provider.Configure(config)
 	resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 
@@ -446,10 +479,9 @@ func (s *GRPCProviderServer) ReadResource(_ context.Context, req *proto.ReadReso
 	resp := &proto.ReadResource_Response{}
 
 	res := s.provider.ResourcesMap[req.TypeName]
-	blockForCore := s.getResourceSchemaBlockForCore(req.TypeName)
-	blockForShimming := s.getResourceSchemaBlockForShimming(req.TypeName)
+	schemaBlock := s.getResourceSchemaBlock(req.TypeName)
 
-	stateVal, err := msgpack.Unmarshal(req.CurrentState.Msgpack, blockForCore.ImpliedType())
+	stateVal, err := msgpack.Unmarshal(req.CurrentState.Msgpack, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -471,7 +503,7 @@ func (s *GRPCProviderServer) ReadResource(_ context.Context, req *proto.ReadReso
 		// The old provider API used an empty id to signal that the remote
 		// object appears to have been deleted, but our new protocol expects
 		// to see a null value (in the cty sense) in that case.
-		newStateMP, err := msgpack.Marshal(cty.NullVal(blockForCore.ImpliedType()), blockForCore.ImpliedType())
+		newStateMP, err := msgpack.Marshal(cty.NullVal(schemaBlock.ImpliedType()), schemaBlock.ImpliedType())
 		if err != nil {
 			resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		}
@@ -484,7 +516,7 @@ func (s *GRPCProviderServer) ReadResource(_ context.Context, req *proto.ReadReso
 	// helper/schema should always copy the ID over, but do it again just to be safe
 	newInstanceState.Attributes["id"] = newInstanceState.ID
 
-	newStateVal, err := hcl2shim.HCL2ValueFromFlatmap(newInstanceState.Attributes, blockForShimming.ImpliedType())
+	newStateVal, err := hcl2shim.HCL2ValueFromFlatmap(newInstanceState.Attributes, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -493,7 +525,7 @@ func (s *GRPCProviderServer) ReadResource(_ context.Context, req *proto.ReadReso
 	newStateVal = normalizeNullValues(newStateVal, stateVal, false)
 	newStateVal = copyTimeoutValues(newStateVal, stateVal)
 
-	newStateMP, err := msgpack.Marshal(newStateVal, blockForCore.ImpliedType())
+	newStateMP, err := msgpack.Marshal(newStateVal, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -518,10 +550,9 @@ func (s *GRPCProviderServer) PlanResourceChange(_ context.Context, req *proto.Pl
 	resp.LegacyTypeSystem = true
 
 	res := s.provider.ResourcesMap[req.TypeName]
-	blockForCore := s.getResourceSchemaBlockForCore(req.TypeName)
-	blockForShimming := s.getResourceSchemaBlockForShimming(req.TypeName)
+	schemaBlock := s.getResourceSchemaBlock(req.TypeName)
 
-	priorStateVal, err := msgpack.Unmarshal(req.PriorState.Msgpack, blockForCore.ImpliedType())
+	priorStateVal, err := msgpack.Unmarshal(req.PriorState.Msgpack, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -529,7 +560,7 @@ func (s *GRPCProviderServer) PlanResourceChange(_ context.Context, req *proto.Pl
 
 	create := priorStateVal.IsNull()
 
-	proposedNewStateVal, err := msgpack.Unmarshal(req.ProposedNewState.Msgpack, blockForCore.ImpliedType())
+	proposedNewStateVal, err := msgpack.Unmarshal(req.ProposedNewState.Msgpack, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -567,7 +598,7 @@ func (s *GRPCProviderServer) PlanResourceChange(_ context.Context, req *proto.Pl
 	}
 
 	// turn the proposed state into a legacy configuration
-	cfg := terraform.NewResourceConfigShimmed(proposedNewStateVal, blockForShimming)
+	cfg := terraform.NewResourceConfigShimmed(proposedNewStateVal, schemaBlock)
 
 	diff, err := s.provider.SimpleDiff(info, priorState, cfg)
 	if err != nil {
@@ -600,15 +631,15 @@ func (s *GRPCProviderServer) PlanResourceChange(_ context.Context, req *proto.Pl
 	}
 
 	// now we need to apply the diff to the prior state, so get the planned state
-	plannedAttrs, err := diff.Apply(priorState.Attributes, blockForShimming)
+	plannedAttrs, err := diff.Apply(priorState.Attributes, schemaBlock)
 
-	plannedStateVal, err := hcl2shim.HCL2ValueFromFlatmap(plannedAttrs, blockForShimming.ImpliedType())
+	plannedStateVal, err := hcl2shim.HCL2ValueFromFlatmap(plannedAttrs, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
 	}
 
-	plannedStateVal, err = blockForShimming.CoerceValue(plannedStateVal)
+	plannedStateVal, err = schemaBlock.CoerceValue(plannedStateVal)
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -640,10 +671,10 @@ func (s *GRPCProviderServer) PlanResourceChange(_ context.Context, req *proto.Pl
 	// if this was creating the resource, we need to set any remaining computed
 	// fields
 	if create {
-		plannedStateVal = SetUnknowns(plannedStateVal, blockForShimming)
+		plannedStateVal = SetUnknowns(plannedStateVal, schemaBlock)
 	}
 
-	plannedMP, err := msgpack.Marshal(plannedStateVal, blockForCore.ImpliedType())
+	plannedMP, err := msgpack.Marshal(plannedStateVal, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -696,7 +727,7 @@ func (s *GRPCProviderServer) PlanResourceChange(_ context.Context, req *proto.Pl
 		requiresNew = append(requiresNew, "id")
 	}
 
-	requiresReplace, err := hcl2shim.RequiresReplace(requiresNew, blockForShimming.ImpliedType())
+	requiresReplace, err := hcl2shim.RequiresReplace(requiresNew, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -717,16 +748,15 @@ func (s *GRPCProviderServer) ApplyResourceChange(_ context.Context, req *proto.A
 	}
 
 	res := s.provider.ResourcesMap[req.TypeName]
-	blockForCore := s.getResourceSchemaBlockForCore(req.TypeName)
-	blockForShimming := s.getResourceSchemaBlockForShimming(req.TypeName)
+	schemaBlock := s.getResourceSchemaBlock(req.TypeName)
 
-	priorStateVal, err := msgpack.Unmarshal(req.PriorState.Msgpack, blockForCore.ImpliedType())
+	priorStateVal, err := msgpack.Unmarshal(req.PriorState.Msgpack, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
 	}
 
-	plannedStateVal, err := msgpack.Unmarshal(req.PlannedState.Msgpack, blockForCore.ImpliedType())
+	plannedStateVal, err := msgpack.Unmarshal(req.PlannedState.Msgpack, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -814,13 +844,13 @@ func (s *GRPCProviderServer) ApplyResourceChange(_ context.Context, req *proto.A
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 	}
-	newStateVal := cty.NullVal(blockForShimming.ImpliedType())
+	newStateVal := cty.NullVal(schemaBlock.ImpliedType())
 
 	// Always return a null value for destroy.
 	// While this is usually indicated by a nil state, check for missing ID or
 	// attributes in the case of a provider failure.
 	if destroy || newInstanceState == nil || newInstanceState.Attributes == nil || newInstanceState.ID == "" {
-		newStateMP, err := msgpack.Marshal(newStateVal, blockForCore.ImpliedType())
+		newStateMP, err := msgpack.Marshal(newStateVal, schemaBlock.ImpliedType())
 		if err != nil {
 			resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 			return resp, nil
@@ -833,7 +863,7 @@ func (s *GRPCProviderServer) ApplyResourceChange(_ context.Context, req *proto.A
 
 	// We keep the null val if we destroyed the resource, otherwise build the
 	// entire object, even if the new state was nil.
-	newStateVal, err = schema.StateValueFromInstanceState(newInstanceState, blockForShimming.ImpliedType())
+	newStateVal, err = schema.StateValueFromInstanceState(newInstanceState, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -843,7 +873,7 @@ func (s *GRPCProviderServer) ApplyResourceChange(_ context.Context, req *proto.A
 
 	newStateVal = copyTimeoutValues(newStateVal, plannedStateVal)
 
-	newStateMP, err := msgpack.Marshal(newStateVal, blockForCore.ImpliedType())
+	newStateMP, err := msgpack.Marshal(newStateVal, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -892,15 +922,14 @@ func (s *GRPCProviderServer) ImportResourceState(_ context.Context, req *proto.I
 			resourceType = req.TypeName
 		}
 
-		blockForCore := s.getResourceSchemaBlockForCore(resourceType)
-		blockForShimming := s.getResourceSchemaBlockForShimming(resourceType)
-		newStateVal, err := hcl2shim.HCL2ValueFromFlatmap(is.Attributes, blockForShimming.ImpliedType())
+		schemaBlock := s.getResourceSchemaBlock(resourceType)
+		newStateVal, err := hcl2shim.HCL2ValueFromFlatmap(is.Attributes, schemaBlock.ImpliedType())
 		if err != nil {
 			resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 			return resp, nil
 		}
 
-		newStateMP, err := msgpack.Marshal(newStateVal, blockForCore.ImpliedType())
+		newStateMP, err := msgpack.Marshal(newStateVal, schemaBlock.ImpliedType())
 		if err != nil {
 			resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 			return resp, nil
@@ -929,10 +958,9 @@ func (s *GRPCProviderServer) ImportResourceState(_ context.Context, req *proto.I
 func (s *GRPCProviderServer) ReadDataSource(_ context.Context, req *proto.ReadDataSource_Request) (*proto.ReadDataSource_Response, error) {
 	resp := &proto.ReadDataSource_Response{}
 
-	blockForCore := s.getDatasourceSchemaBlockForCore(req.TypeName)
-	blockForShimming := s.getDatasourceSchemaBlockForShimming(req.TypeName)
+	schemaBlock := s.getDatasourceSchemaBlock(req.TypeName)
 
-	configVal, err := msgpack.Unmarshal(req.Config.Msgpack, blockForCore.ImpliedType())
+	configVal, err := msgpack.Unmarshal(req.Config.Msgpack, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -948,7 +976,7 @@ func (s *GRPCProviderServer) ReadDataSource(_ context.Context, req *proto.ReadDa
 		return resp, nil
 	}
 
-	config := terraform.NewResourceConfigShimmed(configVal, blockForShimming)
+	config := terraform.NewResourceConfigShimmed(configVal, schemaBlock)
 
 	// we need to still build the diff separately with the Read method to match
 	// the old behavior
@@ -965,7 +993,7 @@ func (s *GRPCProviderServer) ReadDataSource(_ context.Context, req *proto.ReadDa
 		return resp, nil
 	}
 
-	newStateVal, err := schema.StateValueFromInstanceState(newInstanceState, blockForShimming.ImpliedType())
+	newStateVal, err := schema.StateValueFromInstanceState(newInstanceState, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -973,7 +1001,7 @@ func (s *GRPCProviderServer) ReadDataSource(_ context.Context, req *proto.ReadDa
 
 	newStateVal = copyTimeoutValues(newStateVal, configVal)
 
-	newStateMP, err := msgpack.Marshal(newStateVal, blockForCore.ImpliedType())
+	newStateMP, err := msgpack.Marshal(newStateVal, schemaBlock.ImpliedType())
 	if err != nil {
 		resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err)
 		return resp, nil
@@ -1113,25 +1141,22 @@ func normalizeNullValues(dst, src cty.Value, apply bool) cty.Value {
 	if !src.IsNull() && !src.IsKnown() {
 		// Return src during plan to retain unknown interpolated placeholders,
 		// which could be lost if we're only updating a resource. If this is a
-		// read scenario, then there shouldn't be any unknowns all.
+		// read scenario, then there shouldn't be any unknowns at all.
 		if dst.IsNull() && !apply {
 			return src
 		}
 		return dst
 	}
 
-	// handle null/empty changes for collections
-	if ty.IsCollectionType() {
-		if src.IsNull() && !dst.IsNull() && dst.IsKnown() {
-			if dst.LengthInt() == 0 {
-				return src
-			}
-		}
+	// Handle null/empty changes for collections during apply.
+	// A change between null and empty values prefers src to make sure the state
+	// is consistent between plan and apply.
+	if ty.IsCollectionType() && apply {
+		dstEmpty := !dst.IsNull() && dst.IsKnown() && dst.LengthInt() == 0
+		srcEmpty := !src.IsNull() && src.IsKnown() && src.LengthInt() == 0
 
-		if dst.IsNull() && !src.IsNull() && src.IsKnown() {
-			if src.LengthInt() == 0 {
-				return src
-			}
+		if (src.IsNull() && dstEmpty) || (srcEmpty && dst.IsNull()) {
+			return src
 		}
 	}
 
@@ -1164,6 +1189,7 @@ func normalizeNullValues(dst, src cty.Value, apply bool) cty.Value {
 				}
 				dstVal = cty.NullVal(v.Type())
 			}
+
 			dstMap[key] = normalizeNullValues(dstVal, v, apply)
 		}
 

helper/plugin/grpc_provider_test.go

@@ -116,6 +116,144 @@ func TestUpgradeState_jsonState(t *testing.T) {
 	}
 }
 
+func TestUpgradeState_removedAttr(t *testing.T) {
+	r1 := &schema.Resource{
+		Schema: map[string]*schema.Schema{
+			"two": {
+				Type:     schema.TypeString,
+				Optional: true,
+			},
+		},
+	}
+
+	r2 := &schema.Resource{
+		Schema: map[string]*schema.Schema{
+			"multi": {
+				Type:     schema.TypeSet,
+				Optional: true,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"set": {
+							Type:     schema.TypeSet,
+							Optional: true,
+							Elem: &schema.Resource{
+								Schema: map[string]*schema.Schema{
+									"required": {
+										Type:     schema.TypeString,
+										Required: true,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	r3 := &schema.Resource{
+		Schema: map[string]*schema.Schema{
+			"config_mode_attr": {
+				Type:       schema.TypeList,
+				ConfigMode: schema.SchemaConfigModeAttr,
+				Optional:   true,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"foo": {
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	p := &schema.Provider{
+		ResourcesMap: map[string]*schema.Resource{
+			"r1": r1,
+			"r2": r2,
+			"r3": r3,
+		},
+	}
+
+	server := &GRPCProviderServer{
+		provider: p,
+	}
+
+	for _, tc := range []struct {
+		name     string
+		raw      string
+		expected cty.Value
+	}{
+		{
+			name: "r1",
+			raw:  `{"id":"bar","removed":"removed","two":"2"}`,
+			expected: cty.ObjectVal(map[string]cty.Value{
+				"id":  cty.StringVal("bar"),
+				"two": cty.StringVal("2"),
+			}),
+		},
+		{
+			name: "r2",
+			raw:  `{"id":"bar","multi":[{"set":[{"required":"ok","removed":"removed"}]}]}`,
+			expected: cty.ObjectVal(map[string]cty.Value{
+				"id": cty.StringVal("bar"),
+				"multi": cty.SetVal([]cty.Value{
+					cty.ObjectVal(map[string]cty.Value{
+						"set": cty.SetVal([]cty.Value{
+							cty.ObjectVal(map[string]cty.Value{
+								"required": cty.StringVal("ok"),
+							}),
+						}),
+					}),
+				}),
+			}),
+		},
+		{
+			name: "r3",
+			raw:  `{"id":"bar","config_mode_attr":[{"foo":"ok","removed":"removed"}]}`,
+			expected: cty.ObjectVal(map[string]cty.Value{
+				"id": cty.StringVal("bar"),
+				"config_mode_attr": cty.ListVal([]cty.Value{
+					cty.ObjectVal(map[string]cty.Value{
+						"foo": cty.StringVal("ok"),
+					}),
+				}),
+			}),
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			req := &proto.UpgradeResourceState_Request{
+				TypeName: tc.name,
+				Version:  0,
+				RawState: &proto.RawState{
+					Json: []byte(tc.raw),
+				},
+			}
+			resp, err := server.UpgradeResourceState(nil, req)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if len(resp.Diagnostics) > 0 {
+				for _, d := range resp.Diagnostics {
+					t.Errorf("%#v", d)
+				}
+				t.Fatal("error")
+			}
+			val, err := msgpack.Unmarshal(resp.UpgradedState.Msgpack, p.ResourcesMap[tc.name].CoreConfigSchema().ImpliedType())
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !tc.expected.RawEquals(val) {
+				t.Fatalf("\nexpected: %#v\ngot:      %#v\n", tc.expected, val)
+			}
+		})
+	}
+
+}
+
 func TestUpgradeState_flatmapState(t *testing.T) {
 	r := &schema.Resource{
 		SchemaVersion: 4,
@@ -1002,6 +1140,41 @@ func TestNormalizeNullValues(t *testing.T) {
 				}),
 			}),
 		},
+		{
+			Src: cty.ObjectVal(map[string]cty.Value{
+				"set": cty.NullVal(cty.Set(cty.Object(map[string]cty.Type{
+					"list": cty.List(cty.String),
+				}))),
+			}),
+			Dst: cty.ObjectVal(map[string]cty.Value{
+				"set": cty.SetValEmpty(cty.Object(map[string]cty.Type{
+					"list": cty.List(cty.String),
+				})),
+			}),
+			Expect: cty.ObjectVal(map[string]cty.Value{
+				"set": cty.SetValEmpty(cty.Object(map[string]cty.Type{
+					"list": cty.List(cty.String),
+				})),
+			}),
+		},
+		{
+			Src: cty.ObjectVal(map[string]cty.Value{
+				"set": cty.NullVal(cty.Set(cty.Object(map[string]cty.Type{
+					"list": cty.List(cty.String),
+				}))),
+			}),
+			Dst: cty.ObjectVal(map[string]cty.Value{
+				"set": cty.SetValEmpty(cty.Object(map[string]cty.Type{
+					"list": cty.List(cty.String),
+				})),
+			}),
+			Expect: cty.ObjectVal(map[string]cty.Value{
+				"set": cty.NullVal(cty.Set(cty.Object(map[string]cty.Type{
+					"list": cty.List(cty.String),
+				}))),
+			}),
+			Apply: true,
+		},
 	} {
 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
 			got := normalizeNullValues(tc.Dst, tc.Src, tc.Apply)

helper/resource/grpc_test_provider.go

@@ -37,7 +37,7 @@ func GRPCTestProvider(rp terraform.ResourceProvider) providers.Interface {
 	client, _ := pp.GRPCClient(context.Background(), nil, conn)
 
 	grpcClient := client.(*tfplugin.GRPCProvider)
-	grpcClient.TestListener = listener
+	grpcClient.TestServer = grpcServer
 
 	return grpcClient
 }

helper/schema/core_schema.go

@@ -174,14 +174,6 @@ func (s *Schema) coreConfigSchemaBlock() *configschema.NestedBlock {
 // coreConfigSchemaType determines the core config schema type that corresponds
 // to a particular schema's type.
 func (s *Schema) coreConfigSchemaType() cty.Type {
-	if s.SkipCoreTypeCheck {
-		// If we're preparing a schema for Terraform Core and the schema is
-		// asking us to skip the Core type-check then we'll tell core that this
-		// attribute is dynamically-typed, so it'll just pass through anything
-		// and let us validate it on the plugin side.
-		return cty.DynamicPseudoType
-	}
-
 	switch s.Type {
 	case TypeString:
 		return cty.String

helper/schema/core_schema_test.go

@@ -445,28 +445,6 @@ func TestSchemaMapCoreConfigSchema(t *testing.T) {
 				BlockTypes: map[string]*configschema.NestedBlock{},
 			}),
 		},
-		"skip core type check": {
-			map[string]*Schema{
-				"list": {
-					Type:              TypeList,
-					ConfigMode:        SchemaConfigModeAttr,
-					SkipCoreTypeCheck: true,
-					Optional:          true,
-					Elem: &Resource{
-						Schema: map[string]*Schema{},
-					},
-				},
-			},
-			testResource(&configschema.Block{
-				Attributes: map[string]*configschema.Attribute{
-					"list": {
-						Type:     cty.DynamicPseudoType,
-						Optional: true, // Just so we can progress to provider-driven validation and return the error there
-					},
-				},
-				BlockTypes: map[string]*configschema.NestedBlock{},
-			}),
-		},
 	}
 
 	for name, test := range tests {

helper/schema/field_reader_config.go

@@ -2,6 +2,7 @@ package schema
 
 import (
 	"fmt"
+	"log"
 	"strconv"
 	"strings"
 	"sync"
@@ -93,6 +94,22 @@ func (r *ConfigFieldReader) readField(
 		}
 	}
 
+	if protoVersion5 {
+		switch schema.Type {
+		case TypeList, TypeSet, TypeMap, typeObject:
+			// Check if the value itself is unknown.
+			// The new protocol shims will add unknown values to this list of
+			// ComputedKeys. This is the only way we have to indicate that a
+			// collection is unknown in the config
+			for _, unknown := range r.Config.ComputedKeys {
+				if k == unknown {
+					log.Printf("[DEBUG] setting computed for %q from ComputedKeys", k)
+					return FieldReadResult{Computed: true, Exists: true}, nil
+				}
+			}
+		}
+	}
+
 	switch schema.Type {
 	case TypeBool, TypeFloat, TypeInt, TypeString:
 		return r.readPrimitive(k, schema)

helper/schema/getsource_string.go

@@ -4,6 +4,18 @@ package schema
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[getSourceState-1]
+	_ = x[getSourceConfig-2]
+	_ = x[getSourceDiff-4]
+	_ = x[getSourceSet-8]
+	_ = x[getSourceExact-16]
+	_ = x[getSourceLevelMask-15]
+}
+
 const (
 	_getSource_name_0 = "getSourceStategetSourceConfig"
 	_getSource_name_1 = "getSourceDiff"

helper/schema/schema.go

@@ -95,34 +95,6 @@ type Schema struct {
 	// behavior, and SchemaConfigModeBlock is not permitted.
 	ConfigMode SchemaConfigMode
 
-	// SkipCoreTypeCheck, if set, will advertise this attribute to Terraform Core
-	// has being dynamically-typed rather than deriving a type from the schema.
-	// This has the effect of making Terraform Core skip all type-checking of
-	// the value, and thus leaves all type checking up to a combination of this
-	// SDK and the provider's own code.
-	//
-	// This flag does nothing for Terraform versions prior to v0.12, because
-	// in prior versions there was no Core-side typecheck anyway.
-	//
-	// The most practical effect of this flag is to allow object-typed schemas
-	// (specified with Elem: schema.Resource) to pass through Terraform Core
-	// even without all of the object type attributes specified, which may be
-	// useful when using ConfigMode: SchemaConfigModeAttr to achieve
-	// nested-block-like behaviors while using attribute syntax.
-	//
-	// However, by doing so we require type information to be sent and stored
-	// per-object rather than just once statically in the schema, and so this
-	// will change the wire serialization of a resource type in state. Changing
-	// the value of SkipCoreTypeCheck will therefore require a state migration
-	// if there has previously been any release of the provider compatible with
-	// Terraform v0.12.
-	//
-	// SkipCoreTypeCheck can only be set when ConfigMode is SchemaConfigModeAttr,
-	// because nested blocks cannot be decoded by Terraform Core without at
-	// least shallow information about the next level of nested attributes and
-	// blocks.
-	SkipCoreTypeCheck bool
-
 	// If one of these is set, then this item can come from the configuration.
 	// Both cannot be set. If Optional is set, the value is optional. If
 	// Required is set, the value is required.
@@ -735,8 +707,6 @@ func (m schemaMap) internalValidate(topSchemaMap schemaMap, attrsOnly bool) erro
 
 		computedOnly := v.Computed && !v.Optional
 
-		isBlock := false
-
 		switch v.ConfigMode {
 		case SchemaConfigModeBlock:
 			if _, ok := v.Elem.(*Resource); !ok {
@@ -748,7 +718,6 @@ func (m schemaMap) internalValidate(topSchemaMap schemaMap, attrsOnly bool) erro
 			if computedOnly {
 				return fmt.Errorf("%s: ConfigMode of block cannot be used for computed schema", k)
 			}
-			isBlock = true
 		case SchemaConfigModeAttr:
 			// anything goes
 		case SchemaConfigModeAuto:
@@ -756,7 +725,6 @@ func (m schemaMap) internalValidate(topSchemaMap schemaMap, attrsOnly bool) erro
 			// and that's impossible inside an attribute, we require it to be
 			// explicitly overridden as mode "Attr" for clarity.
 			if _, ok := v.Elem.(*Resource); ok {
-				isBlock = true
 				if attrsOnly {
 					return fmt.Errorf("%s: in *schema.Resource with ConfigMode of attribute, so must also have ConfigMode of attribute", k)
 				}
@@ -765,10 +733,6 @@ func (m schemaMap) internalValidate(topSchemaMap schemaMap, attrsOnly bool) erro
 			return fmt.Errorf("%s: invalid ConfigMode value", k)
 		}
 
-		if isBlock && v.SkipCoreTypeCheck {
-			return fmt.Errorf("%s: SkipCoreTypeCheck must be false unless ConfigMode is attribute", k)
-		}
-
 		if v.Computed && v.Default != nil {
 			return fmt.Errorf("%s: Default must be nil if computed", k)
 		}
@@ -1731,12 +1695,25 @@ func (m schemaMap) validatePrimitive(
 		}
 		decoded = n
 	case TypeInt:
-		// Verify that we can parse this as an int
-		var n int
-		if err := mapstructure.WeakDecode(raw, &n); err != nil {
-			return nil, []error{fmt.Errorf("%s: %s", k, err)}
+		switch {
+		case isProto5():
+			// We need to verify the type precisely, because WeakDecode will
+			// decode a float as an integer.
+
+			// the config shims only use int for integral number values
+			if v, ok := raw.(int); ok {
+				decoded = v
+			} else {
+				return nil, []error{fmt.Errorf("%s: must be a whole number, got %v", k, raw)}
+			}
+		default:
+			// Verify that we can parse this as an int
+			var n int
+			if err := mapstructure.WeakDecode(raw, &n); err != nil {
+				return nil, []error{fmt.Errorf("%s: %s", k, err)}
+			}
+			decoded = n
 		}
-		decoded = n
 	case TypeFloat:
 		// Verify that we can parse this as an int
 		var n float64

helper/schema/shims.go

@@ -113,45 +113,3 @@ func JSONMapToStateValue(m map[string]interface{}, block *configschema.Block) (c
 func StateValueFromInstanceState(is *terraform.InstanceState, ty cty.Type) (cty.Value, error) {
 	return is.AttrsAsObjectValue(ty)
 }
-
-// LegacyResourceSchema takes a *Resource and returns a deep copy with 0.12 specific
-// features removed. This is used by the shims to get a configschema that
-// directly matches the structure of the schema.Resource.
-func LegacyResourceSchema(r *Resource) *Resource {
-	if r == nil {
-		return nil
-	}
-	// start with a shallow copy
-	newResource := new(Resource)
-	*newResource = *r
-	newResource.Schema = map[string]*Schema{}
-
-	for k, s := range r.Schema {
-		newResource.Schema[k] = LegacySchema(s)
-	}
-
-	return newResource
-}
-
-// LegacySchema takes a *Schema and returns a deep copy with some 0.12-specific
-// features disabled. This is used by the shims to get a configschema that
-// better reflects the given schema.Resource, without any adjustments we
-// make for when sending a schema to Terraform Core.
-func LegacySchema(s *Schema) *Schema {
-	if s == nil {
-		return nil
-	}
-	// start with a shallow copy
-	newSchema := new(Schema)
-	*newSchema = *s
-	newSchema.SkipCoreTypeCheck = false
-
-	switch e := newSchema.Elem.(type) {
-	case *Schema:
-		newSchema.Elem = LegacySchema(e)
-	case *Resource:
-		newSchema.Elem = LegacyResourceSchema(e)
-	}
-
-	return newSchema
-}

helper/schema/valuetype_string.go

@@ -4,6 +4,21 @@ package schema
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[TypeInvalid-0]
+	_ = x[TypeBool-1]
+	_ = x[TypeInt-2]
+	_ = x[TypeFloat-3]
+	_ = x[TypeString-4]
+	_ = x[TypeList-5]
+	_ = x[TypeMap-6]
+	_ = x[TypeSet-7]
+	_ = x[typeObject-8]
+}
+
 const _ValueType_name = "TypeInvalidTypeBoolTypeIntTypeFloatTypeStringTypeListTypeMapTypeSettypeObject"
 
 var _ValueType_index = [...]uint8{0, 11, 19, 26, 35, 45, 53, 60, 67, 77}

helper/validation/validation.go

@@ -320,3 +320,22 @@ func ValidateRFC3339TimeString(v interface{}, k string) (ws []string, errors []e
 	}
 	return
 }
+
+// FloatBetween returns a SchemaValidateFunc which tests if the provided value
+// is of type float64 and is between min and max (inclusive).
+func FloatBetween(min, max float64) schema.SchemaValidateFunc {
+	return func(i interface{}, k string) (s []string, es []error) {
+		v, ok := i.(float64)
+		if !ok {
+			es = append(es, fmt.Errorf("expected type of %s to be float64", k))
+			return
+		}
+
+		if v < min || v > max {
+			es = append(es, fmt.Errorf("expected %s to be in the range (%f - %f), got %f", k, min, max, v))
+			return
+		}
+
+		return
+	}
+}

helper/validation/validation_test.go

@@ -455,3 +455,46 @@ func runTestCases(t *testing.T, cases []testCase) {
 		}
 	}
 }
+
+func TestFloatBetween(t *testing.T) {
+	cases := map[string]struct {
+		Value                  interface{}
+		ValidateFunc           schema.SchemaValidateFunc
+		ExpectValidationErrors bool
+	}{
+		"accept valid value": {
+			Value:                  1.5,
+			ValidateFunc:           FloatBetween(1.0, 2.0),
+			ExpectValidationErrors: false,
+		},
+		"accept valid value inclusive upper bound": {
+			Value:                  1.0,
+			ValidateFunc:           FloatBetween(0.0, 1.0),
+			ExpectValidationErrors: false,
+		},
+		"accept valid value inclusive lower bound": {
+			Value:                  0.0,
+			ValidateFunc:           FloatBetween(0.0, 1.0),
+			ExpectValidationErrors: false,
+		},
+		"reject out of range value": {
+			Value:                  -1.0,
+			ValidateFunc:           FloatBetween(0.0, 1.0),
+			ExpectValidationErrors: true,
+		},
+		"reject incorrectly typed value": {
+			Value:                  1,
+			ValidateFunc:           FloatBetween(0.0, 1.0),
+			ExpectValidationErrors: true,
+		},
+	}
+
+	for tn, tc := range cases {
+		_, errors := tc.ValidateFunc(tc.Value, tn)
+		if len(errors) > 0 && !tc.ExpectValidationErrors {
+			t.Errorf("%s: unexpected errors %s", tn, errors)
+		} else if len(errors) == 0 && tc.ExpectValidationErrors {
+			t.Errorf("%s: expected errors but got none", tn)
+		}
+	}
+}

internal/initwd/module_install.go

@@ -428,7 +428,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest,
 		diags = diags.Append(tfdiags.Sourceless(
 			tfdiags.Error,
 			"Failed to download module",
-			fmt.Sprintf("Error attempting to download module %q (%s:%d) source code from %q: %s.", req.Name, req.CallPos.Filename, req.CallPos.Line, dlAddr, err),
+			fmt.Sprintf("Could not download module %q (%s:%d) source code from %q: %s.", req.Name, req.CallPos.Filename, req.CallPos.Line, dlAddr, err),
 		))
 		return nil, nil, diags
 	}
@@ -482,7 +482,7 @@ func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest,
 
 	modDir, err := getter.getWithGoGetter(instPath, req.SourceAddr)
 	if err != nil {
-		if err, ok := err.(*MaybeRelativePathErr); ok {
+		if _, ok := err.(*MaybeRelativePathErr); ok {
 			log.Printf(
 				"[TRACE] ModuleInstaller: %s looks like a local path but is missing ./ or ../",
 				req.SourceAddr,
@@ -507,7 +507,7 @@ func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest,
 			diags = diags.Append(tfdiags.Sourceless(
 				tfdiags.Error,
 				"Failed to download module",
-				fmt.Sprintf("Error attempting to download module %q (%s:%d) source code from %q: %s", req.Name, req.CallPos.Filename, req.CallPos.Line, packageAddr, err),
+				fmt.Sprintf("Could not download module %q (%s:%d) source code from %q: %s", req.Name, req.CallPos.Filename, req.CallPos.Line, packageAddr, err),
 			))
 		}
 		return nil, diags

lang/funcs/collection.go

@@ -1,6 +1,7 @@
 package funcs
 
 import (
+	"errors"
 	"fmt"
 	"sort"
 
@@ -43,7 +44,7 @@ var ElementFunc = function.New(&function.Spec{
 				return cty.DynamicPseudoType, fmt.Errorf("invalid index: %s", err)
 			}
 			if len(etys) == 0 {
-				return cty.DynamicPseudoType, fmt.Errorf("cannot use element function with an empty list")
+				return cty.DynamicPseudoType, errors.New("cannot use element function with an empty list")
 			}
 			index = index % len(etys)
 			return etys[index], nil
@@ -65,7 +66,7 @@ var ElementFunc = function.New(&function.Spec{
 
 		l := args[0].LengthInt()
 		if l == 0 {
-			return cty.DynamicVal, fmt.Errorf("cannot use element function with an empty list")
+			return cty.DynamicVal, errors.New("cannot use element function with an empty list")
 		}
 		index = index % l
 
@@ -90,7 +91,7 @@ var LengthFunc = function.New(&function.Spec{
 		case collTy == cty.String || collTy.IsTupleType() || collTy.IsObjectType() || collTy.IsListType() || collTy.IsMapType() || collTy.IsSetType() || collTy == cty.DynamicPseudoType:
 			return cty.Number, nil
 		default:
-			return cty.Number, fmt.Errorf("argument must be a string, a collection type, or a structural type")
+			return cty.Number, errors.New("argument must be a string, a collection type, or a structural type")
 		}
 	},
 	Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
@@ -114,12 +115,12 @@ var LengthFunc = function.New(&function.Spec{
 			return coll.Length(), nil
 		default:
 			// Should never happen, because of the checks in our Type func above
-			return cty.UnknownVal(cty.Number), fmt.Errorf("impossible value type for length(...)")
+			return cty.UnknownVal(cty.Number), errors.New("impossible value type for length(...)")
 		}
 	},
 })
 
-// CoalesceFunc contructs a function that takes any number of arguments and
+// CoalesceFunc constructs a function that takes any number of arguments and
 // returns the first one that isn't empty. This function was copied from go-cty
 // stdlib and modified so that it returns the first *non-empty* non-null element
 // from a sequence, instead of merely the first non-null.
@@ -139,7 +140,7 @@ var CoalesceFunc = function.New(&function.Spec{
 		}
 		retType, _ := convert.UnifyUnsafe(argTypes)
 		if retType == cty.NilType {
-			return cty.NilType, fmt.Errorf("all arguments must have the same type")
+			return cty.NilType, errors.New("all arguments must have the same type")
 		}
 		return retType, nil
 	},
@@ -159,42 +160,53 @@ var CoalesceFunc = function.New(&function.Spec{
 
 			return argVal, nil
 		}
-		return cty.NilVal, fmt.Errorf("no non-null, non-empty-string arguments")
+		return cty.NilVal, errors.New("no non-null, non-empty-string arguments")
 	},
 })
 
-// CoalesceListFunc contructs a function that takes any number of list arguments
+// CoalesceListFunc constructs a function that takes any number of list arguments
 // and returns the first one that isn't empty.
 var CoalesceListFunc = function.New(&function.Spec{
 	Params: []function.Parameter{},
 	VarParam: &function.Parameter{
 		Name:             "vals",
-		Type:             cty.List(cty.DynamicPseudoType),
+		Type:             cty.DynamicPseudoType,
 		AllowUnknown:     true,
 		AllowDynamicType: true,
 		AllowNull:        true,
 	},
 	Type: func(args []cty.Value) (ret cty.Type, err error) {
 		if len(args) == 0 {
-			return cty.NilType, fmt.Errorf("at least one argument is required")
+			return cty.NilType, errors.New("at least one argument is required")
 		}
 
 		argTypes := make([]cty.Type, len(args))
 
 		for i, arg := range args {
+			// if any argument is unknown, we can't be certain know which type we will return
+			if !arg.IsKnown() {
+				return cty.DynamicPseudoType, nil
+			}
+			ty := arg.Type()
+
+			if !ty.IsListType() && !ty.IsTupleType() {
+				return cty.NilType, errors.New("coalescelist arguments must be lists or tuples")
+			}
+
 			argTypes[i] = arg.Type()
 		}
 
-		retType, _ := convert.UnifyUnsafe(argTypes)
-		if retType == cty.NilType {
-			return cty.NilType, fmt.Errorf("all arguments must have the same type")
+		last := argTypes[0]
+		// If there are mixed types, we have to return a dynamic type.
+		for _, next := range argTypes[1:] {
+			if !next.Equals(last) {
+				return cty.DynamicPseudoType, nil
+			}
 		}
 
-		return retType, nil
+		return last, nil
 	},
 	Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) {
-
-		vals := make([]cty.Value, 0, len(args))
 		for _, arg := range args {
 			if !arg.IsKnown() {
 				// If we run into an unknown list at some point, we can't
@@ -203,25 +215,16 @@ var CoalesceListFunc = function.New(&function.Spec{
 				return cty.UnknownVal(retType), nil
 			}
 
-			// We already know this will succeed because of the checks in our Type func above
-			arg, _ = convert.Convert(arg, retType)
-
-			it := arg.ElementIterator()
-			for it.Next() {
-				_, v := it.Element()
-				vals = append(vals, v)
-			}
-
-			if len(vals) > 0 {
-				return cty.ListVal(vals), nil
+			if arg.LengthInt() > 0 {
+				return arg, nil
 			}
 		}
 
-		return cty.NilVal, fmt.Errorf("no non-null arguments")
+		return cty.NilVal, errors.New("no non-null arguments")
 	},
 })
 
-// CompactFunc contructs a function that takes a list of strings and returns a new list
+// CompactFunc constructs a function that takes a list of strings and returns a new list
 // with any empty string elements removed.
 var CompactFunc = function.New(&function.Spec{
 	Params: []function.Parameter{
@@ -257,13 +260,13 @@ var CompactFunc = function.New(&function.Spec{
 	},
 })
 
-// ContainsFunc contructs a function that determines whether a given list contains
-// a given single value as one of its elements.
+// ContainsFunc constructs a function that determines whether a given list or
+// set contains a given single value as one of its elements.
 var ContainsFunc = function.New(&function.Spec{
 	Params: []function.Parameter{
 		{
 			Name: "list",
-			Type: cty.List(cty.DynamicPseudoType),
+			Type: cty.DynamicPseudoType,
 		},
 		{
 			Name: "value",
@@ -272,8 +275,14 @@ var ContainsFunc = function.New(&function.Spec{
 	},
 	Type: function.StaticReturnType(cty.Bool),
 	Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) {
+		arg := args[0]
+		ty := arg.Type()
 
-		_, err = Index(args[0], args[1])
+		if !ty.IsListType() && !ty.IsTupleType() && !ty.IsSetType() {
+			return cty.NilVal, errors.New("argument must be list, tuple, or set")
+		}
+
+		_, err = Index(cty.TupleVal(arg.AsValueSlice()), args[1])
 		if err != nil {
 			return cty.False, nil
 		}
@@ -282,7 +291,7 @@ var ContainsFunc = function.New(&function.Spec{
 	},
 })
 
-// IndexFunc contructs a function that finds the element index for a given value in a list.
+// IndexFunc constructs a function that finds the element index for a given value in a list.
 var IndexFunc = function.New(&function.Spec{
 	Params: []function.Parameter{
 		{
@@ -297,7 +306,7 @@ var IndexFunc = function.New(&function.Spec{
 	Type: function.StaticReturnType(cty.Number),
 	Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) {
 		if !(args[0].Type().IsListType() || args[0].Type().IsTupleType()) {
-			return cty.NilVal, fmt.Errorf("argument must be a list or tuple")
+			return cty.NilVal, errors.New("argument must be a list or tuple")
 		}
 
 		if !args[0].IsKnown() {
@@ -305,7 +314,7 @@ var IndexFunc = function.New(&function.Spec{
 		}
 
 		if args[0].LengthInt() == 0 { // Easy path
-			return cty.NilVal, fmt.Errorf("cannot search an empty list")
+			return cty.NilVal, errors.New("cannot search an empty list")
 		}
 
 		for it := args[0].ElementIterator(); it.Next(); {
@@ -321,12 +330,12 @@ var IndexFunc = function.New(&function.Spec{
 				return i, nil
 			}
 		}
-		return cty.NilVal, fmt.Errorf("item not found")
+		return cty.NilVal, errors.New("item not found")
 
 	},
 })
 
-// DistinctFunc contructs a function that takes a list and returns a new list
+// DistinctFunc constructs a function that takes a list and returns a new list
 // with any duplicate elements removed.
 var DistinctFunc = function.New(&function.Spec{
 	Params: []function.Parameter{
@@ -358,7 +367,7 @@ var DistinctFunc = function.New(&function.Spec{
 	},
 })
 
-// ChunklistFunc contructs a function that splits a single list into fixed-size chunks,
+// ChunklistFunc constructs a function that splits a single list into fixed-size chunks,
 // returning a list of lists.
 var ChunklistFunc = function.New(&function.Spec{
 	Params: []function.Parameter{
@@ -376,7 +385,7 @@ var ChunklistFunc = function.New(&function.Spec{
 	},
 	Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) {
 		listVal := args[0]
-		if !listVal.IsWhollyKnown() {
+		if !listVal.IsKnown() {
 			return cty.UnknownVal(retType), nil
 		}
 
@@ -387,7 +396,7 @@ var ChunklistFunc = function.New(&function.Spec{
 		}
 
 		if size < 0 {
-			return cty.NilVal, fmt.Errorf("the size argument must be positive")
+			return cty.NilVal, errors.New("the size argument must be positive")
 		}
 
 		output := make([]cty.Value, 0)
@@ -419,47 +428,76 @@ var ChunklistFunc = function.New(&function.Spec{
 	},
 })
 
-// FlattenFunc contructs a function that takes a list and replaces any elements
+// FlattenFunc constructs a function that takes a list and replaces any elements
 // that are lists with a flattened sequence of the list contents.
 var FlattenFunc = function.New(&function.Spec{
 	Params: []function.Parameter{
 		{
 			Name: "list",
-			Type: cty.List(cty.DynamicPseudoType),
+			Type: cty.DynamicPseudoType,
 		},
 	},
-	Type: function.StaticReturnType(cty.List(cty.DynamicPseudoType)),
+	Type: func(args []cty.Value) (cty.Type, error) {
+		if !args[0].IsWhollyKnown() {
+			return cty.DynamicPseudoType, nil
+		}
+
+		argTy := args[0].Type()
+		if !argTy.IsListType() && !argTy.IsSetType() && !argTy.IsTupleType() {
+			return cty.NilType, errors.New("can only flatten lists, sets and tuples")
+		}
+
+		retVal, known := flattener(args[0])
+		if !known {
+			return cty.DynamicPseudoType, nil
+		}
+
+		tys := make([]cty.Type, len(retVal))
+		for i, ty := range retVal {
+			tys[i] = ty.Type()
+		}
+		return cty.Tuple(tys), nil
+	},
 	Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) {
 		inputList := args[0]
-		if !inputList.IsWhollyKnown() {
+		if inputList.LengthInt() == 0 {
+			return cty.EmptyTupleVal, nil
+		}
+
+		out, known := flattener(inputList)
+		if !known {
 			return cty.UnknownVal(retType), nil
 		}
 
-		if inputList.LengthInt() == 0 {
-			return cty.ListValEmpty(retType.ElementType()), nil
-		}
-		outputList := make([]cty.Value, 0)
-
-		return cty.ListVal(flattener(outputList, inputList)), nil
+		return cty.TupleVal(out), nil
 	},
 })
 
-// Flatten until it's not a cty.List
-func flattener(finalList []cty.Value, flattenList cty.Value) []cty.Value {
-
+// Flatten until it's not a cty.List, and return whether the value is known.
+// We can flatten lists with unknown values, as long as they are not
+// lists themselves.
+func flattener(flattenList cty.Value) ([]cty.Value, bool) {
+	out := make([]cty.Value, 0)
 	for it := flattenList.ElementIterator(); it.Next(); {
 		_, val := it.Element()
+		if val.Type().IsListType() || val.Type().IsSetType() || val.Type().IsTupleType() {
+			if !val.IsKnown() {
+				return out, false
+			}
 
-		if val.Type().IsListType() {
-			finalList = flattener(finalList, val)
+			res, known := flattener(val)
+			if !known {
+				return res, known
+			}
+			out = append(out, res...)
 		} else {
-			finalList = append(finalList, val)
+			out = append(out, val)
 		}
 	}
-	return finalList
+	return out, true
 }
 
-// KeysFunc contructs a function that takes a map and returns a sorted list of the map keys.
+// KeysFunc constructs a function that takes a map and returns a sorted list of the map keys.
 var KeysFunc = function.New(&function.Spec{
 	Params: []function.Parameter{
 		{
@@ -529,7 +567,7 @@ var KeysFunc = function.New(&function.Spec{
 	},
 })
 
-// ListFunc contructs a function that takes an arbitrary number of arguments
+// ListFunc constructs a function that takes an arbitrary number of arguments
 // and returns a list containing those values in the same order.
 //
 // This function is deprecated in Terraform v0.12
@@ -544,7 +582,7 @@ var ListFunc = function.New(&function.Spec{
 	},
 	Type: func(args []cty.Value) (ret cty.Type, err error) {
 		if len(args) == 0 {
-			return cty.NilType, fmt.Errorf("at least one argument is required")
+			return cty.NilType, errors.New("at least one argument is required")
 		}
 
 		argTypes := make([]cty.Type, len(args))
@@ -555,7 +593,7 @@ var ListFunc = function.New(&function.Spec{
 
 		retType, _ := convert.UnifyUnsafe(argTypes)
 		if retType == cty.NilType {
-			return cty.NilType, fmt.Errorf("all arguments must have the same type")
+			return cty.NilType, errors.New("all arguments must have the same type")
 		}
 
 		return cty.List(retType), nil
@@ -573,7 +611,7 @@ var ListFunc = function.New(&function.Spec{
 	},
 })
 
-// LookupFunc contructs a function that performs dynamic lookups of map types.
+// LookupFunc constructs a function that performs dynamic lookups of map types.
 var LookupFunc = function.New(&function.Spec{
 	Params: []function.Parameter{
 		{
@@ -649,7 +687,7 @@ var LookupFunc = function.New(&function.Spec{
 				case ty.Equals(cty.Number):
 					return cty.NumberVal(v.AsBigFloat()), nil
 				default:
-					return cty.NilVal, fmt.Errorf("lookup() can only be used with flat lists")
+					return cty.NilVal, errors.New("lookup() can only be used with flat lists")
 				}
 			}
 		}
@@ -667,7 +705,7 @@ var LookupFunc = function.New(&function.Spec{
 	},
 })
 
-// MapFunc contructs a function that takes an even number of arguments and
+// MapFunc constructs a function that takes an even number of arguments and
 // returns a map whose elements are constructed from consecutive pairs of arguments.
 //
 // This function is deprecated in Terraform v0.12
@@ -695,7 +733,7 @@ var MapFunc = function.New(&function.Spec{
 
 		valType, _ := convert.UnifyUnsafe(argTypes)
 		if valType == cty.NilType {
-			return cty.NilType, fmt.Errorf("all arguments must have the same type")
+			return cty.NilType, errors.New("all arguments must have the same type")
 		}
 
 		return cty.Map(valType), nil
@@ -740,7 +778,7 @@ var MapFunc = function.New(&function.Spec{
 	},
 })
 
-// MatchkeysFunc contructs a function that constructs a new list by taking a
+// MatchkeysFunc constructs a function that constructs a new list by taking a
 // subset of elements from one list whose indexes match the corresponding
 // indexes of values in another list.
 var MatchkeysFunc = function.New(&function.Spec{
@@ -760,7 +798,7 @@ var MatchkeysFunc = function.New(&function.Spec{
 	},
 	Type: func(args []cty.Value) (cty.Type, error) {
 		if !args[1].Type().Equals(args[2].Type()) {
-			return cty.NilType, fmt.Errorf("lists must be of the same type")
+			return cty.NilType, errors.New("lists must be of the same type")
 		}
 
 		return args[0].Type(), nil
@@ -771,7 +809,7 @@ var MatchkeysFunc = function.New(&function.Spec{
 		}
 
 		if args[0].LengthInt() != args[1].LengthInt() {
-			return cty.ListValEmpty(retType.ElementType()), fmt.Errorf("length of keys and values should be equal")
+			return cty.ListValEmpty(retType.ElementType()), errors.New("length of keys and values should be equal")
 		}
 
 		output := make([]cty.Value, 0)
@@ -818,7 +856,7 @@ var MatchkeysFunc = function.New(&function.Spec{
 	},
 })
 
-// MergeFunc contructs a function that takes an arbitrary number of maps and
+// MergeFunc constructs a function that takes an arbitrary number of maps and
 // returns a single map that contains a merged set of elements from all of the maps.
 //
 // If more than one given map defines the same key then the one that is later in
@@ -906,7 +944,7 @@ var SetProductFunc = function.New(&function.Spec{
 	},
 	Type: func(args []cty.Value) (retType cty.Type, err error) {
 		if len(args) < 2 {
-			return cty.NilType, fmt.Errorf("at least two arguments are required")
+			return cty.NilType, errors.New("at least two arguments are required")
 		}
 
 		listCount := 0
@@ -1017,13 +1055,13 @@ var SetProductFunc = function.New(&function.Spec{
 	},
 })
 
-// SliceFunc contructs a function that extracts some consecutive elements
+// SliceFunc constructs a function that extracts some consecutive elements
 // from within a list.
 var SliceFunc = function.New(&function.Spec{
 	Params: []function.Parameter{
 		{
 			Name: "list",
-			Type: cty.List(cty.DynamicPseudoType),
+			Type: cty.DynamicPseudoType,
 		},
 		{
 			Name: "startIndex",
@@ -1035,51 +1073,73 @@ var SliceFunc = function.New(&function.Spec{
 		},
 	},
 	Type: func(args []cty.Value) (cty.Type, error) {
-		return args[0].Type(), nil
+		arg := args[0]
+		argTy := arg.Type()
+
+		if !argTy.IsListType() && !argTy.IsTupleType() {
+			return cty.NilType, errors.New("cannot slice a set, because its elements do not have indices; use the tolist function to force conversion to list if the ordering of the result is not important")
+		}
+
+		if argTy.IsListType() {
+			return argTy, nil
+		}
+
+		startIndex, endIndex, err := sliceIndexes(args, args[0].LengthInt())
+		if err != nil {
+			return cty.NilType, err
+		}
+
+		return cty.Tuple(argTy.TupleElementTypes()[startIndex:endIndex]), nil
 	},
 	Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) {
 		inputList := args[0]
-		if !inputList.IsWhollyKnown() {
-			return cty.UnknownVal(retType), nil
-		}
-		var startIndex, endIndex int
 
-		if err = gocty.FromCtyValue(args[1], &startIndex); err != nil {
-			return cty.NilVal, fmt.Errorf("invalid start index: %s", err)
-		}
-		if err = gocty.FromCtyValue(args[2], &endIndex); err != nil {
-			return cty.NilVal, fmt.Errorf("invalid start index: %s", err)
+		startIndex, endIndex, err := sliceIndexes(args, inputList.LengthInt())
+		if err != nil {
+			return cty.NilVal, err
 		}
 
-		if startIndex < 0 {
-			return cty.NilVal, fmt.Errorf("from index must be >= 0")
-		}
-		if endIndex > inputList.LengthInt() {
-			return cty.NilVal, fmt.Errorf("to index must be <= length of the input list")
-		}
-		if startIndex > endIndex {
-			return cty.NilVal, fmt.Errorf("from index must be <= to index")
-		}
-
-		var outputList []cty.Value
-
-		i := 0
-		for it := inputList.ElementIterator(); it.Next(); {
-			_, v := it.Element()
-			if i >= startIndex && i < endIndex {
-				outputList = append(outputList, v)
+		if endIndex-startIndex == 0 {
+			if retType.IsTupleType() {
+				return cty.EmptyTupleVal, nil
 			}
-			i++
-		}
-
-		if len(outputList) == 0 {
 			return cty.ListValEmpty(retType.ElementType()), nil
 		}
+
+		outputList := inputList.AsValueSlice()[startIndex:endIndex]
+
+		if retType.IsTupleType() {
+			return cty.TupleVal(outputList), nil
+		}
+
 		return cty.ListVal(outputList), nil
 	},
 })
 
+func sliceIndexes(args []cty.Value, max int) (int, int, error) {
+	var startIndex, endIndex int
+
+	if err := gocty.FromCtyValue(args[1], &startIndex); err != nil {
+		return 0, 0, fmt.Errorf("invalid start index: %s", err)
+	}
+	if err := gocty.FromCtyValue(args[2], &endIndex); err != nil {
+		return 0, 0, fmt.Errorf("invalid start index: %s", err)
+	}
+
+	if startIndex < 0 {
+		return 0, 0, errors.New("from index must be greater than or equal to 0")
+	}
+	if endIndex > max {
+		return 0, 0, errors.New("to index must be less than or equal to the length of the input list")
+	}
+	if startIndex > endIndex {
+		return 0, 0, errors.New("from index must be less than or equal to index")
+	}
+	return startIndex, endIndex, nil
+}
+
 // TransposeFunc contructs a function that takes a map of lists of strings and
+// TransposeFunc constructs a function that takes a map of lists of strings and
 // swaps the keys and values to produce a new map of lists of strings.
 var TransposeFunc = function.New(&function.Spec{
 	Params: []function.Parameter{
@@ -1103,7 +1163,7 @@ var TransposeFunc = function.New(&function.Spec{
 			for iter := inVal.ElementIterator(); iter.Next(); {
 				_, val := iter.Element()
 				if !val.Type().Equals(cty.String) {
-					return cty.MapValEmpty(cty.List(cty.String)), fmt.Errorf("input must be a map of lists of strings")
+					return cty.MapValEmpty(cty.List(cty.String)), errors.New("input must be a map of lists of strings")
 				}
 
 				outKey := val.AsString()
@@ -1129,7 +1189,7 @@ var TransposeFunc = function.New(&function.Spec{
 	},
 })
 
-// ValuesFunc contructs a function that returns a list of the map values,
+// ValuesFunc constructs a function that returns a list of the map values,
 // in the order of the sorted keys.
 var ValuesFunc = function.New(&function.Spec{
 	Params: []function.Parameter{
@@ -1163,7 +1223,7 @@ var ValuesFunc = function.New(&function.Spec{
 			}
 			return cty.Tuple(tys), nil
 		}
-		return cty.NilType, fmt.Errorf("values() requires a map as the first argument")
+		return cty.NilType, errors.New("values() requires a map as the first argument")
 	},
 	Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) {
 		mapVar := args[0]
@@ -1186,7 +1246,7 @@ var ValuesFunc = function.New(&function.Spec{
 	},
 })
 
-// ZipmapFunc contructs a function that constructs a map from a list of keys
+// ZipmapFunc constructs a function that constructs a map from a list of keys
 // and a corresponding list of values.
 var ZipmapFunc = function.New(&function.Spec{
 	Params: []function.Parameter{
@@ -1230,7 +1290,7 @@ var ZipmapFunc = function.New(&function.Spec{
 			return cty.Object(atys), nil
 
 		default:
-			return cty.NilType, fmt.Errorf("values argument must be a list or tuple value")
+			return cty.NilType, errors.New("values argument must be a list or tuple value")
 		}
 	},
 	Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) {

lang/funcs/collection_test.go

@@ -417,7 +417,7 @@ func TestCoalesceList(t *testing.T) {
 				}),
 			},
 			cty.ListVal([]cty.Value{
-				cty.StringVal("1"), cty.StringVal("2"),
+				cty.NumberIntVal(1), cty.NumberIntVal(2),
 			}),
 			false,
 		},
@@ -476,7 +476,7 @@ func TestCoalesceList(t *testing.T) {
 				cty.ListValEmpty(cty.String),
 				cty.UnknownVal(cty.List(cty.String)),
 			},
-			cty.UnknownVal(cty.List(cty.String)),
+			cty.DynamicVal,
 			false,
 		},
 		{ // unknown list
@@ -486,13 +486,63 @@ func TestCoalesceList(t *testing.T) {
 					cty.StringVal("third"), cty.StringVal("fourth"),
 				}),
 			},
-			cty.UnknownVal(cty.List(cty.String)),
+			cty.DynamicVal,
 			false,
 		},
+		{ // unknown tuple
+			[]cty.Value{
+				cty.UnknownVal(cty.Tuple([]cty.Type{cty.String})),
+				cty.ListVal([]cty.Value{
+					cty.StringVal("third"), cty.StringVal("fourth"),
+				}),
+			},
+			cty.DynamicVal,
+			false,
+		},
+		{ // empty tuple
+			[]cty.Value{
+				cty.EmptyTupleVal,
+				cty.ListVal([]cty.Value{
+					cty.StringVal("third"), cty.StringVal("fourth"),
+				}),
+			},
+			cty.ListVal([]cty.Value{
+				cty.StringVal("third"), cty.StringVal("fourth"),
+			}),
+			false,
+		},
+		{ // tuple value
+			[]cty.Value{
+				cty.TupleVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.NumberIntVal(2),
+				}),
+				cty.ListVal([]cty.Value{
+					cty.StringVal("third"), cty.StringVal("fourth"),
+				}),
+			},
+			cty.TupleVal([]cty.Value{
+				cty.StringVal("a"),
+				cty.NumberIntVal(2),
+			}),
+			false,
+		},
+		{ // reject set value
+			[]cty.Value{
+				cty.SetVal([]cty.Value{
+					cty.StringVal("a"),
+				}),
+				cty.ListVal([]cty.Value{
+					cty.StringVal("third"), cty.StringVal("fourth"),
+				}),
+			},
+			cty.NilVal,
+			true,
+		},
 	}
 
-	for _, test := range tests {
-		t.Run(fmt.Sprintf("coalescelist(%#v)", test.Values), func(t *testing.T) {
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d-coalescelist(%#v)", i, test.Values), func(t *testing.T) {
 			got, err := CoalesceList(test.Values...)
 
 			if test.Err {
@@ -671,6 +721,26 @@ func TestContains(t *testing.T) {
 			cty.BoolVal(true),
 			false,
 		},
+		{ // set val
+			cty.SetVal([]cty.Value{
+				cty.StringVal("quick"),
+				cty.StringVal("brown"),
+				cty.StringVal("fox"),
+			}),
+			cty.StringVal("quick"),
+			cty.BoolVal(true),
+			false,
+		},
+		{ // tuple val
+			cty.TupleVal([]cty.Value{
+				cty.StringVal("quick"),
+				cty.StringVal("brown"),
+				cty.NumberIntVal(3),
+			}),
+			cty.NumberIntVal(3),
+			cty.BoolVal(true),
+			false,
+		},
 	}
 
 	for _, test := range tests {
@@ -993,13 +1063,29 @@ func TestChunklist(t *testing.T) {
 				cty.UnknownVal(cty.String),
 			}),
 			cty.NumberIntVal(1),
+			cty.ListVal([]cty.Value{
+				cty.ListVal([]cty.Value{
+					cty.StringVal("a"),
+				}),
+				cty.ListVal([]cty.Value{
+					cty.StringVal("b"),
+				}),
+				cty.ListVal([]cty.Value{
+					cty.UnknownVal(cty.String),
+				}),
+			}),
+			false,
+		},
+		{
+			cty.UnknownVal(cty.List(cty.String)),
+			cty.NumberIntVal(1),
 			cty.UnknownVal(cty.List(cty.List(cty.String))),
 			false,
 		},
 	}
 
-	for _, test := range tests {
-		t.Run(fmt.Sprintf("chunklist(%#v, %#v)", test.List, test.Size), func(t *testing.T) {
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d-chunklist(%#v, %#v)", i, test.List, test.Size), func(t *testing.T) {
 			got, err := Chunklist(test.List, test.Size)
 
 			if test.Err {
@@ -1035,7 +1121,7 @@ func TestFlatten(t *testing.T) {
 					cty.StringVal("d"),
 				}),
 			}),
-			cty.ListVal([]cty.Value{
+			cty.TupleVal([]cty.Value{
 				cty.StringVal("a"),
 				cty.StringVal("b"),
 				cty.StringVal("c"),
@@ -1043,6 +1129,44 @@ func TestFlatten(t *testing.T) {
 			}),
 			false,
 		},
+		// handle single elements as arguments
+		{
+			cty.TupleVal([]cty.Value{
+				cty.ListVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+				}),
+				cty.StringVal("c"),
+			}),
+			cty.TupleVal([]cty.Value{
+				cty.StringVal("a"),
+				cty.StringVal("b"),
+				cty.StringVal("c"),
+			}), false,
+		},
+		// handle single elements and mixed primitive types as arguments
+		{
+			cty.TupleVal([]cty.Value{
+				cty.ListVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+				}),
+				cty.StringVal("c"),
+				cty.TupleVal([]cty.Value{
+					cty.StringVal("x"),
+					cty.NumberIntVal(1),
+				}),
+			}),
+			cty.TupleVal([]cty.Value{
+				cty.StringVal("a"),
+				cty.StringVal("b"),
+				cty.StringVal("c"),
+				cty.StringVal("x"),
+				cty.NumberIntVal(1),
+			}),
+			false,
+		},
+		// Primitive unknowns should still be flattened to a tuple
 		{
 			cty.ListVal([]cty.Value{
 				cty.ListVal([]cty.Value{
@@ -1054,18 +1178,74 @@ func TestFlatten(t *testing.T) {
 					cty.StringVal("d"),
 				}),
 			}),
-			cty.UnknownVal(cty.List(cty.DynamicPseudoType)),
-			false,
+			cty.TupleVal([]cty.Value{
+				cty.StringVal("a"),
+				cty.StringVal("b"),
+				cty.UnknownVal(cty.String),
+				cty.StringVal("d"),
+			}), false,
+		},
+		// An unknown series should return an unknown dynamic value
+		{
+			cty.TupleVal([]cty.Value{
+				cty.ListVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+				}),
+				cty.TupleVal([]cty.Value{
+					cty.UnknownVal(cty.List(cty.String)),
+					cty.StringVal("d"),
+				}),
+			}),
+			cty.UnknownVal(cty.DynamicPseudoType), false,
 		},
 		{
 			cty.ListValEmpty(cty.String),
-			cty.ListValEmpty(cty.DynamicPseudoType),
+			cty.EmptyTupleVal,
+			false,
+		},
+		{
+			cty.SetVal([]cty.Value{
+				cty.SetVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+				}),
+				cty.SetVal([]cty.Value{
+					cty.StringVal("c"),
+					cty.StringVal("d"),
+				}),
+			}),
+			cty.TupleVal([]cty.Value{
+				cty.StringVal("a"),
+				cty.StringVal("b"),
+				cty.StringVal("c"),
+				cty.StringVal("d"),
+			}),
+			false,
+		},
+		{
+			cty.TupleVal([]cty.Value{
+				cty.SetVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+				}),
+				cty.ListVal([]cty.Value{
+					cty.StringVal("c"),
+					cty.StringVal("d"),
+				}),
+			}),
+			cty.TupleVal([]cty.Value{
+				cty.StringVal("a"),
+				cty.StringVal("b"),
+				cty.StringVal("c"),
+				cty.StringVal("d"),
+			}),
 			false,
 		},
 	}
 
-	for _, test := range tests {
-		t.Run(fmt.Sprintf("flatten(%#v)", test.List), func(t *testing.T) {
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d-flatten(%#v)", i, test.List), func(t *testing.T) {
 			got, err := Flatten(test.List)
 
 			if test.Err {
@@ -2005,12 +2185,12 @@ func TestReverse(t *testing.T) {
 		},
 		{
 			cty.SetVal([]cty.Value{cty.StringVal("a"), cty.StringVal("b")}),
-			cty.ListVal([]cty.Value{cty.StringVal("a"), cty.StringVal("b")}),
+			cty.ListVal([]cty.Value{cty.StringVal("b"), cty.StringVal("a")}), // set-of-string iterates in lexicographical order
 			"",
 		},
 		{
-			cty.SetVal([]cty.Value{cty.StringVal("a"), cty.StringVal("b"), cty.StringVal("c")}),
-			cty.ListVal([]cty.Value{cty.StringVal("a"), cty.StringVal("c"), cty.StringVal("b")}),
+			cty.SetVal([]cty.Value{cty.StringVal("b"), cty.StringVal("a"), cty.StringVal("c")}),
+			cty.ListVal([]cty.Value{cty.StringVal("c"), cty.StringVal("b"), cty.StringVal("a")}), // set-of-string iterates in lexicographical order
 			"",
 		},
 		{
@@ -2316,6 +2496,11 @@ func TestSlice(t *testing.T) {
 		cty.StringVal("a"),
 		cty.UnknownVal(cty.String),
 	})
+	tuple := cty.TupleVal([]cty.Value{
+		cty.StringVal("a"),
+		cty.NumberIntVal(1),
+		cty.UnknownVal(cty.List(cty.String)),
+	})
 	tests := []struct {
 		List       cty.Value
 		StartIndex cty.Value
@@ -2332,10 +2517,24 @@ func TestSlice(t *testing.T) {
 			}),
 			false,
 		},
-		{ // unknowns in the list
+		{ // slice only an unknown value
 			listWithUnknowns,
 			cty.NumberIntVal(1),
 			cty.NumberIntVal(2),
+			cty.ListVal([]cty.Value{cty.UnknownVal(cty.String)}),
+			false,
+		},
+		{ // slice multiple values, which contain an unknown
+			listWithUnknowns,
+			cty.NumberIntVal(0),
+			cty.NumberIntVal(2),
+			listWithUnknowns,
+			false,
+		},
+		{ // an unknown list should be slicable, returning an unknown list
+			cty.UnknownVal(cty.List(cty.String)),
+			cty.NumberIntVal(0),
+			cty.NumberIntVal(2),
 			cty.UnknownVal(cty.List(cty.String)),
 			false,
 		},
@@ -2376,10 +2575,44 @@ func TestSlice(t *testing.T) {
 			cty.NilVal,
 			true,
 		},
+		{ // sets are not slice-able
+			cty.SetVal([]cty.Value{
+				cty.StringVal("x"),
+				cty.StringVal("y"),
+			}),
+			cty.NumberIntVal(0),
+			cty.NumberIntVal(0),
+			cty.NilVal,
+			true,
+		},
+		{ // tuple slice
+			tuple,
+			cty.NumberIntVal(1),
+			cty.NumberIntVal(3),
+			cty.TupleVal([]cty.Value{
+				cty.NumberIntVal(1),
+				cty.UnknownVal(cty.List(cty.String)),
+			}),
+			false,
+		},
+		{ // empty list slice
+			listOfStrings,
+			cty.NumberIntVal(2),
+			cty.NumberIntVal(2),
+			cty.ListValEmpty(cty.String),
+			false,
+		},
+		{ // empty tuple slice
+			tuple,
+			cty.NumberIntVal(3),
+			cty.NumberIntVal(3),
+			cty.EmptyTupleVal,
+			false,
+		},
 	}
 
-	for _, test := range tests {
-		t.Run(fmt.Sprintf("slice(%#v, %#v, %#v)", test.List, test.StartIndex, test.EndIndex), func(t *testing.T) {
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d-slice(%#v, %#v, %#v)", i, test.List, test.StartIndex, test.EndIndex), func(t *testing.T) {
 			got, err := Slice(test.List, test.StartIndex, test.EndIndex)
 
 			if test.Err {

lang/functions.go

@@ -88,7 +88,6 @@ func (s *Scope) Functions() map[string]function.Function {
 			"replace":          funcs.ReplaceFunc,
 			"reverse":          funcs.ReverseFunc,
 			"rsadecrypt":       funcs.RsaDecryptFunc,
-			"sethaselement":    stdlib.SetHasElementFunc,
 			"setintersection":  stdlib.SetIntersectionFunc,
 			"setproduct":       funcs.SetProductFunc,
 			"setunion":         stdlib.SetUnionFunc,
@@ -99,6 +98,7 @@ func (s *Scope) Functions() map[string]function.Function {
 			"slice":            funcs.SliceFunc,
 			"sort":             funcs.SortFunc,
 			"split":            funcs.SplitFunc,
+			"strrev":           stdlib.ReverseFunc,
 			"substr":           stdlib.SubstrFunc,
 			"timestamp":        funcs.TimestampFunc,
 			"timeadd":          funcs.TimeAddFunc,

lang/functions_test.go

@@ -0,0 +1,853 @@
+package lang
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/hashicorp/hcl2/hcl"
+	"github.com/hashicorp/hcl2/hcl/hclsyntax"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/zclconf/go-cty/cty"
+)
+
+// TestFunctions tests that functions are callable through the functionality
+// in the langs package, via HCL.
+//
+// These tests are primarily here to assert that the functions are properly
+// registered in the functions table, rather than to test all of the details
+// of the functions. Each function should only have one or two tests here,
+// since the main set of unit tests for a function should live alongside that
+// function either in the "funcs" subdirectory here or over in the cty
+// function/stdlib package.
+//
+// One exception to that is we can use this test mechanism to assert common
+// patterns that are used in real-world configurations which rely on behaviors
+// implemented either in this lang package or in HCL itself, such as automatic
+// type conversions. The function unit tests don't cover those things because
+// they call directly into the functions.
+//
+// With that said then, this test function should contain at least one simple
+// test case per function registered in the functions table (just to prove
+// it really is registered correctly) and possibly a small set of additional
+// functions showing real-world use-cases that rely on type conversion
+// behaviors.
+func TestFunctions(t *testing.T) {
+	// used in `pathexpand()` test
+	homePath, err := homedir.Dir()
+	if err != nil {
+		t.Fatalf("Error getting home directory: %v", err)
+	}
+
+	tests := map[string][]struct {
+		src  string
+		want cty.Value
+	}{
+		// Please maintain this list in alphabetical order by function, with
+		// a blank line between the group of tests for each function.
+
+		"abs": {
+			{
+				`abs(-1)`,
+				cty.NumberIntVal(1),
+			},
+		},
+
+		"base64decode": {
+			{
+				`base64decode("YWJjMTIzIT8kKiYoKSctPUB+")`,
+				cty.StringVal("abc123!?$*&()'-=@~"),
+			},
+		},
+
+		"base64encode": {
+			{
+				`base64encode("abc123!?$*&()'-=@~")`,
+				cty.StringVal("YWJjMTIzIT8kKiYoKSctPUB+"),
+			},
+		},
+
+		"base64gzip": {
+			{
+				`base64gzip("test")`,
+				cty.StringVal("H4sIAAAAAAAA/ypJLS4BAAAA//8BAAD//wx+f9gEAAAA"),
+			},
+		},
+
+		"base64sha256": {
+			{
+				`base64sha256("test")`,
+				cty.StringVal("n4bQgYhMfWWaL+qgxVrQFaO/TxsrC4Is0V1sFbDwCgg="),
+			},
+		},
+
+		"base64sha512": {
+			{
+				`base64sha512("test")`,
+				cty.StringVal("7iaw3Ur350mqGo7jwQrpkj9hiYB3Lkc/iBml1JQODbJ6wYX4oOHV+E+IvIh/1nsUNzLDBMxfqa2Ob1f1ACio/w=="),
+			},
+		},
+
+		"basename": {
+			{
+				`basename("testdata/hello.txt")`,
+				cty.StringVal("hello.txt"),
+			},
+		},
+
+		"ceil": {
+			{
+				`ceil(1.2)`,
+				cty.NumberIntVal(2),
+			},
+		},
+
+		"chomp": {
+			{
+				`chomp("goodbye\ncruel\nworld\n")`,
+				cty.StringVal("goodbye\ncruel\nworld"),
+			},
+		},
+
+		"chunklist": {
+			{
+				`chunklist(["a", "b", "c"], 1)`,
+				cty.ListVal([]cty.Value{
+					cty.ListVal([]cty.Value{
+						cty.StringVal("a"),
+					}),
+					cty.ListVal([]cty.Value{
+						cty.StringVal("b"),
+					}),
+					cty.ListVal([]cty.Value{
+						cty.StringVal("c"),
+					}),
+				}),
+			},
+		},
+
+		"cidrhost": {
+			{
+				`cidrhost("192.168.1.0/24", 5)`,
+				cty.StringVal("192.168.1.5"),
+			},
+		},
+
+		"cidrnetmask": {
+			{
+				`cidrnetmask("192.168.1.0/24")`,
+				cty.StringVal("255.255.255.0"),
+			},
+		},
+
+		"cidrsubnet": {
+			{
+				`cidrsubnet("192.168.2.0/20", 4, 6)`,
+				cty.StringVal("192.168.6.0/24"),
+			},
+		},
+
+		"coalesce": {
+			{
+				`coalesce("first", "second", "third")`,
+				cty.StringVal("first"),
+			},
+
+			{
+				`coalescelist(["first", "second"], ["third", "fourth"])`,
+				cty.TupleVal([]cty.Value{
+					cty.StringVal("first"), cty.StringVal("second"),
+				}),
+			},
+		},
+
+		"coalescelist": {
+			{
+				`coalescelist(list("a", "b"), list("c", "d"))`,
+				cty.ListVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+				}),
+			},
+			{
+				`coalescelist(["a", "b"], ["c", "d"])`,
+				cty.TupleVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+				}),
+			},
+		},
+
+		"compact": {
+			{
+				`compact(["test", "", "test"])`,
+				cty.ListVal([]cty.Value{
+					cty.StringVal("test"), cty.StringVal("test"),
+				}),
+			},
+		},
+
+		"concat": {
+			{
+				`concat(["a", ""], ["b", "c"])`,
+				cty.TupleVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal(""),
+					cty.StringVal("b"),
+					cty.StringVal("c"),
+				}),
+			},
+		},
+
+		"contains": {
+			{
+				`contains(["a", "b"], "a")`,
+				cty.True,
+			},
+			{ // Should also work with sets, due to automatic conversion
+				`contains(toset(["a", "b"]), "a")`,
+				cty.True,
+			},
+		},
+
+		"csvdecode": {
+			{
+				`csvdecode("a,b,c\n1,2,3\n4,5,6")`,
+				cty.ListVal([]cty.Value{
+					cty.ObjectVal(map[string]cty.Value{
+						"a": cty.StringVal("1"),
+						"b": cty.StringVal("2"),
+						"c": cty.StringVal("3"),
+					}),
+					cty.ObjectVal(map[string]cty.Value{
+						"a": cty.StringVal("4"),
+						"b": cty.StringVal("5"),
+						"c": cty.StringVal("6"),
+					}),
+				}),
+			},
+		},
+
+		"dirname": {
+			{
+				`dirname("testdata/hello.txt")`,
+				cty.StringVal("testdata"),
+			},
+		},
+
+		"distinct": {
+			{
+				`distinct(["a", "b", "a", "b"])`,
+				cty.ListVal([]cty.Value{
+					cty.StringVal("a"), cty.StringVal("b"),
+				}),
+			},
+		},
+
+		"element": {
+			{
+				`element(["hello"], 0)`,
+				cty.StringVal("hello"),
+			},
+		},
+
+		"file": {
+			{
+				`file("hello.txt")`,
+				cty.StringVal("hello!"),
+			},
+		},
+
+		"fileexists": {
+			{
+				`fileexists("hello.txt")`,
+				cty.BoolVal(true),
+			},
+		},
+
+		"filebase64": {
+			{
+				`filebase64("hello.txt")`,
+				cty.StringVal("aGVsbG8h"),
+			},
+		},
+
+		"filebase64sha256": {
+			{
+				`filebase64sha256("hello.txt")`,
+				cty.StringVal("zgYJL7lI2f+sfRo3bkBLJrdXW8wR7gWkYV/vT+w6MIs="),
+			},
+		},
+
+		"filebase64sha512": {
+			{
+				`filebase64sha512("hello.txt")`,
+				cty.StringVal("xvgdsOn4IGyXHJ5YJuO6gj/7saOpAPgEdlKov3jqmP38dFhVo4U6Y1Z1RY620arxIJ6I6tLRkjgrXEy91oUOAg=="),
+			},
+		},
+
+		"filemd5": {
+			{
+				`filemd5("hello.txt")`,
+				cty.StringVal("5a8dd3ad0756a93ded72b823b19dd877"),
+			},
+		},
+
+		"filesha1": {
+			{
+				`filesha1("hello.txt")`,
+				cty.StringVal("8f7d88e901a5ad3a05d8cc0de93313fd76028f8c"),
+			},
+		},
+
+		"filesha256": {
+			{
+				`filesha256("hello.txt")`,
+				cty.StringVal("ce06092fb948d9ffac7d1a376e404b26b7575bcc11ee05a4615fef4fec3a308b"),
+			},
+		},
+
+		"filesha512": {
+			{
+				`filesha512("hello.txt")`,
+				cty.StringVal("c6f81db0e9f8206c971c9e5826e3ba823ffbb1a3a900f8047652a8bf78ea98fdfc745855a3853a635675458eb6d1aaf1209e88ead2d192382b5c4cbdd6850e02"),
+			},
+		},
+
+		"flatten": {
+			{
+				`flatten([["a", "b"], ["c", "d"]])`,
+				cty.TupleVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+					cty.StringVal("c"),
+					cty.StringVal("d"),
+				}),
+			},
+		},
+
+		"floor": {
+			{
+				`floor(-1.8)`,
+				cty.NumberFloatVal(-2),
+			},
+		},
+
+		"format": {
+			{
+				`format("Hello, %s!", "Ander")`,
+				cty.StringVal("Hello, Ander!"),
+			},
+		},
+
+		"formatlist": {
+			{
+				`formatlist("Hello, %s!", ["Valentina", "Ander", "Olivia", "Sam"])`,
+				cty.ListVal([]cty.Value{
+					cty.StringVal("Hello, Valentina!"),
+					cty.StringVal("Hello, Ander!"),
+					cty.StringVal("Hello, Olivia!"),
+					cty.StringVal("Hello, Sam!"),
+				}),
+			},
+		},
+
+		"formatdate": {
+			{
+				`formatdate("DD MMM YYYY hh:mm ZZZ", "2018-01-04T23:12:01Z")`,
+				cty.StringVal("04 Jan 2018 23:12 UTC"),
+			},
+		},
+
+		"indent": {
+			{
+				fmt.Sprintf("indent(4, %#v)", Poem),
+				cty.StringVal("Fleas:\n    Adam\n    Had'em\n    \n    E.E. Cummings"),
+			},
+		},
+
+		"index": {
+			{
+				`index(["a", "b", "c"], "a")`,
+				cty.NumberIntVal(0),
+			},
+		},
+
+		"join": {
+			{
+				`join(" ", ["Hello", "World"])`,
+				cty.StringVal("Hello World"),
+			},
+		},
+
+		"jsondecode": {
+			{
+				`jsondecode("{\"hello\": \"world\"}")`,
+				cty.ObjectVal(map[string]cty.Value{
+					"hello": cty.StringVal("world"),
+				}),
+			},
+		},
+
+		"jsonencode": {
+			{
+				`jsonencode({"hello"="world"})`,
+				cty.StringVal("{\"hello\":\"world\"}"),
+			},
+		},
+
+		"keys": {
+			{
+				`keys({"hello"=1, "goodbye"=42})`,
+				cty.TupleVal([]cty.Value{
+					cty.StringVal("goodbye"),
+					cty.StringVal("hello"),
+				}),
+			},
+		},
+
+		"length": {
+			{
+				`length(["the", "quick", "brown", "bear"])`,
+				cty.NumberIntVal(4),
+			},
+		},
+
+		"list": {
+			{
+				`list("hello")`,
+				cty.ListVal([]cty.Value{
+					cty.StringVal("hello"),
+				}),
+			},
+		},
+
+		"log": {
+			{
+				`log(1, 10)`,
+				cty.NumberFloatVal(0),
+			},
+		},
+
+		"lookup": {
+			{
+				`lookup({hello=1, goodbye=42}, "goodbye")`,
+				cty.NumberIntVal(42),
+			},
+		},
+
+		"lower": {
+			{
+				`lower("HELLO")`,
+				cty.StringVal("hello"),
+			},
+		},
+
+		"map": {
+			{
+				`map("hello", "world")`,
+				cty.MapVal(map[string]cty.Value{
+					"hello": cty.StringVal("world"),
+				}),
+			},
+		},
+
+		"matchkeys": {
+			{
+				`matchkeys(["a", "b", "c"], ["ref1", "ref2", "ref3"], ["ref1"])`,
+				cty.ListVal([]cty.Value{
+					cty.StringVal("a"),
+				}),
+			},
+		},
+
+		"max": {
+			{
+				`max(12, 54, 3)`,
+				cty.NumberIntVal(54),
+			},
+		},
+
+		"md5": {
+			{
+				`md5("tada")`,
+				cty.StringVal("ce47d07243bb6eaf5e1322c81baf9bbf"),
+			},
+		},
+
+		"merge": {
+			{
+				`merge({"a"="b"}, {"c"="d"})`,
+				cty.ObjectVal(map[string]cty.Value{
+					"a": cty.StringVal("b"),
+					"c": cty.StringVal("d"),
+				}),
+			},
+		},
+
+		"min": {
+			{
+				`min(12, 54, 3)`,
+				cty.NumberIntVal(3),
+			},
+		},
+
+		"pathexpand": {
+			{
+				`pathexpand("~/test-file")`,
+				cty.StringVal(filepath.Join(homePath, "test-file")),
+			},
+		},
+
+		"pow": {
+			{
+				`pow(1,0)`,
+				cty.NumberFloatVal(1),
+			},
+		},
+
+		"replace": {
+			{
+				`replace("hello", "hel", "bel")`,
+				cty.StringVal("bello"),
+			},
+		},
+
+		"reverse": {
+			{
+				`reverse(["a", true, 0])`,
+				cty.TupleVal([]cty.Value{cty.Zero, cty.True, cty.StringVal("a")}),
+			},
+		},
+
+		"rsadecrypt": {
+			{
+				fmt.Sprintf("rsadecrypt(%#v, %#v)", CipherBase64, PrivateKey),
+				cty.StringVal("message"),
+			},
+		},
+
+		"setintersection": {
+			{
+				`setintersection(["a", "b"], ["b", "c"], ["b", "d"])`,
+				cty.SetVal([]cty.Value{
+					cty.StringVal("b"),
+				}),
+			},
+		},
+
+		"setproduct": {
+			{
+				`setproduct(["development", "staging", "production"], ["app1", "app2"])`,
+				cty.ListVal([]cty.Value{
+					cty.TupleVal([]cty.Value{cty.StringVal("development"), cty.StringVal("app1")}),
+					cty.TupleVal([]cty.Value{cty.StringVal("development"), cty.StringVal("app2")}),
+					cty.TupleVal([]cty.Value{cty.StringVal("staging"), cty.StringVal("app1")}),
+					cty.TupleVal([]cty.Value{cty.StringVal("staging"), cty.StringVal("app2")}),
+					cty.TupleVal([]cty.Value{cty.StringVal("production"), cty.StringVal("app1")}),
+					cty.TupleVal([]cty.Value{cty.StringVal("production"), cty.StringVal("app2")}),
+				}),
+			},
+		},
+
+		"setunion": {
+			{
+				`setunion(["a", "b"], ["b", "c"], ["d"])`,
+				cty.SetVal([]cty.Value{
+					cty.StringVal("d"),
+					cty.StringVal("b"),
+					cty.StringVal("a"),
+					cty.StringVal("c"),
+				}),
+			},
+		},
+
+		"sha1": {
+			{
+				`sha1("test")`,
+				cty.StringVal("a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"),
+			},
+		},
+
+		"sha256": {
+			{
+				`sha256("test")`,
+				cty.StringVal("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
+			},
+		},
+
+		"sha512": {
+			{
+				`sha512("test")`,
+				cty.StringVal("ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff"),
+			},
+		},
+
+		"signum": {
+			{
+				`signum(12)`,
+				cty.NumberFloatVal(1),
+			},
+		},
+
+		"slice": {
+			{
+				// force a list type here for testing
+				`slice(list("a", "b", "c", "d"), 1, 3)`,
+				cty.ListVal([]cty.Value{
+					cty.StringVal("b"), cty.StringVal("c"),
+				}),
+			},
+			{
+				`slice(["a", "b", 3, 4], 1, 3)`,
+				cty.TupleVal([]cty.Value{
+					cty.StringVal("b"), cty.NumberIntVal(3),
+				}),
+			},
+		},
+
+		"sort": {
+			{
+				`sort(["banana", "apple"])`,
+				cty.ListVal([]cty.Value{
+					cty.StringVal("apple"),
+					cty.StringVal("banana"),
+				}),
+			},
+		},
+
+		"split": {
+			{
+				`split(" ", "Hello World")`,
+				cty.ListVal([]cty.Value{
+					cty.StringVal("Hello"),
+					cty.StringVal("World"),
+				}),
+			},
+		},
+
+		"strrev": {
+			{
+				`strrev("hello world")`,
+				cty.StringVal("dlrow olleh"),
+			},
+		},
+
+		"substr": {
+			{
+				`substr("hello world", 1, 4)`,
+				cty.StringVal("ello"),
+			},
+		},
+
+		"templatefile": {
+			{
+				`templatefile("hello.tmpl", {name = "Jodie"})`,
+				cty.StringVal("Hello, Jodie!"),
+			},
+		},
+
+		"timeadd": {
+			{
+				`timeadd("2017-11-22T00:00:00Z", "1s")`,
+				cty.StringVal("2017-11-22T00:00:01Z"),
+			},
+		},
+
+		"title": {
+			{
+				`title("hello")`,
+				cty.StringVal("Hello"),
+			},
+		},
+
+		"tobool": {
+			{
+				`tobool("false")`,
+				cty.False,
+			},
+		},
+
+		"tolist": {
+			{
+				`tolist(["a", "b", "c"])`,
+				cty.ListVal([]cty.Value{
+					cty.StringVal("a"), cty.StringVal("b"), cty.StringVal("c"),
+				}),
+			},
+		},
+
+		"tomap": {
+			{
+				`tomap({"a" = 1, "b" = 2})`,
+				cty.MapVal(map[string]cty.Value{
+					"a": cty.NumberIntVal(1),
+					"b": cty.NumberIntVal(2),
+				}),
+			},
+		},
+
+		"tonumber": {
+			{
+				`tonumber("42")`,
+				cty.NumberIntVal(42),
+			},
+		},
+
+		"toset": {
+			{
+				`toset(["a", "b", "c"])`,
+				cty.SetVal([]cty.Value{
+					cty.StringVal("a"), cty.StringVal("b"), cty.StringVal("c"),
+				}),
+			},
+		},
+
+		"tostring": {
+			{
+				`tostring("a")`,
+				cty.StringVal("a"),
+			},
+		},
+
+		"transpose": {
+			{
+				`transpose({"a" = ["1", "2"], "b" = ["2", "3"]})`,
+				cty.MapVal(map[string]cty.Value{
+					"1": cty.ListVal([]cty.Value{cty.StringVal("a")}),
+					"2": cty.ListVal([]cty.Value{cty.StringVal("a"), cty.StringVal("b")}),
+					"3": cty.ListVal([]cty.Value{cty.StringVal("b")}),
+				}),
+			},
+		},
+
+		"trimspace": {
+			{
+				`trimspace(" hello ")`,
+				cty.StringVal("hello"),
+			},
+		},
+
+		"upper": {
+			{
+				`upper("hello")`,
+				cty.StringVal("HELLO"),
+			},
+		},
+
+		"urlencode": {
+			{
+				`urlencode("foo:bar@localhost?foo=bar&bar=baz")`,
+				cty.StringVal("foo%3Abar%40localhost%3Ffoo%3Dbar%26bar%3Dbaz"),
+			},
+		},
+
+		"values": {
+			{
+				`values({"hello"="world", "what's"="up"})`,
+				cty.TupleVal([]cty.Value{
+					cty.StringVal("world"),
+					cty.StringVal("up"),
+				}),
+			},
+		},
+
+		"zipmap": {
+			{
+				`zipmap(["hello", "bar"], ["world", "baz"])`,
+				cty.ObjectVal(map[string]cty.Value{
+					"hello": cty.StringVal("world"),
+					"bar":   cty.StringVal("baz"),
+				}),
+			},
+		},
+	}
+
+	data := &dataForTests{} // no variables available; we only need literals here
+	scope := &Scope{
+		Data:    data,
+		BaseDir: "./testdata/functions-test", // for the functions that read from the filesystem
+	}
+
+	// Check that there is at least one test case for each function, omitting
+	// those functions that do not return consistent values
+	allFunctions := scope.Functions()
+
+	// TODO: we can test the impure functions partially by configuring the scope
+	// with PureOnly: true and then verify that they return unknown values of a
+	// suitable type.
+	for _, impureFunc := range impureFunctions {
+		delete(allFunctions, impureFunc)
+	}
+	for f, _ := range scope.Functions() {
+		if _, ok := tests[f]; !ok {
+			t.Errorf("Missing test for function %s\n", f)
+		}
+	}
+
+	for funcName, funcTests := range tests {
+		t.Run(funcName, func(t *testing.T) {
+			for _, test := range funcTests {
+				t.Run(test.src, func(t *testing.T) {
+					expr, parseDiags := hclsyntax.ParseExpression([]byte(test.src), "test.hcl", hcl.Pos{Line: 1, Column: 1})
+					if parseDiags.HasErrors() {
+						for _, diag := range parseDiags {
+							t.Error(diag.Error())
+						}
+						return
+					}
+
+					got, diags := scope.EvalExpr(expr, cty.DynamicPseudoType)
+					if diags.HasErrors() {
+						for _, diag := range diags {
+							t.Errorf("%s: %s", diag.Description().Summary, diag.Description().Detail)
+						}
+						return
+					}
+
+					if !test.want.RawEquals(got) {
+						t.Errorf("wrong result\nexpr: %s\ngot:  %#v\nwant: %#v", test.src, got, test.want)
+					}
+				})
+			}
+		})
+	}
+}
+
+const (
+	CipherBase64 = "eczGaDhXDbOFRZGhjx2etVzWbRqWDlmq0bvNt284JHVbwCgObiuyX9uV0LSAMY707IEgMkExJqXmsB4OWKxvB7epRB9G/3+F+pcrQpODlDuL9oDUAsa65zEpYF0Wbn7Oh7nrMQncyUPpyr9WUlALl0gRWytOA23S+y5joa4M34KFpawFgoqTu/2EEH4Xl1zo+0fy73fEto+nfkUY+meuyGZ1nUx/+DljP7ZqxHBFSlLODmtuTMdswUbHbXbWneW51D7Jm7xB8nSdiA2JQNK5+Sg5x8aNfgvFTt/m2w2+qpsyFa5Wjeu6fZmXSl840CA07aXbk9vN4I81WmJyblD/ZA=="
+	PrivateKey   = `
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAgUElV5mwqkloIrM8ZNZ72gSCcnSJt7+/Usa5G+D15YQUAdf9
+c1zEekTfHgDP+04nw/uFNFaE5v1RbHaPxhZYVg5ZErNCa/hzn+x10xzcepeS3KPV
+Xcxae4MR0BEegvqZqJzN9loXsNL/c3H/B+2Gle3hTxjlWFb3F5qLgR+4Mf4ruhER
+1v6eHQa/nchi03MBpT4UeJ7MrL92hTJYLdpSyCqmr8yjxkKJDVC2uRrr+sTSxfh7
+r6v24u/vp/QTmBIAlNPgadVAZw17iNNb7vjV7Gwl/5gHXonCUKURaV++dBNLrHIZ
+pqcAM8wHRph8mD1EfL9hsz77pHewxolBATV+7QIDAQABAoIBAC1rK+kFW3vrAYm3
++8/fQnQQw5nec4o6+crng6JVQXLeH32qXShNf8kLLG/Jj0vaYcTPPDZw9JCKkTMQ
+0mKj9XR/5DLbBMsV6eNXXuvJJ3x4iKW5eD9WkLD4FKlNarBRyO7j8sfPTqXW7uat
+NxWdFH7YsSRvNh/9pyQHLWA5OituidMrYbc3EUx8B1GPNyJ9W8Q8znNYLfwYOjU4
+Wv1SLE6qGQQH9Q0WzA2WUf8jklCYyMYTIywAjGb8kbAJlKhmj2t2Igjmqtwt1PYc
+pGlqbtQBDUiWXt5S4YX/1maIQ/49yeNUajjpbJiH3DbhJbHwFTzP3pZ9P9GHOzlG
+kYR+wSECgYEAw/Xida8kSv8n86V3qSY/I+fYQ5V+jDtXIE+JhRnS8xzbOzz3v0WS
+Oo5H+o4nJx5eL3Ghb3Gcm0Jn46dHrxinHbm+3RjXv/X6tlbxIYjRSQfHOTSMCTvd
+qcliF5vC6RCLXuc7R+IWR1Ky6eDEZGtrvt3DyeYABsp9fRUFR/6NluUCgYEAqNsw
+1aSl7WJa27F0DoJdlU9LWerpXcazlJcIdOz/S9QDmSK3RDQTdqfTxRmrxiYI9LEs
+mkOkvzlnnOBMpnZ3ZOU5qIRfprecRIi37KDAOHWGnlC0EWGgl46YLb7/jXiWf0AG
+Y+DfJJNd9i6TbIDWu8254/erAS6bKMhW/3q7f2kCgYAZ7Id/BiKJAWRpqTRBXlvw
+BhXoKvjI2HjYP21z/EyZ+PFPzur/lNaZhIUlMnUfibbwE9pFggQzzf8scM7c7Sf+
+mLoVSdoQ/Rujz7CqvQzi2nKSsM7t0curUIb3lJWee5/UeEaxZcmIufoNUrzohAWH
+BJOIPDM4ssUTLRq7wYM9uQKBgHCBau5OP8gE6mjKuXsZXWUoahpFLKwwwmJUp2vQ
+pOFPJ/6WZOlqkTVT6QPAcPUbTohKrF80hsZqZyDdSfT3peFx4ZLocBrS56m6NmHR
+UYHMvJ8rQm76T1fryHVidz85g3zRmfBeWg8yqT5oFg4LYgfLsPm1gRjOhs8LfPvI
+OLlRAoGBAIZ5Uv4Z3s8O7WKXXUe/lq6j7vfiVkR1NW/Z/WLKXZpnmvJ7FgxN4e56
+RXT7GwNQHIY8eDjDnsHxzrxd+raOxOZeKcMHj3XyjCX3NHfTscnsBPAGYpY/Wxzh
+T8UYnFu6RzkixElTf2rseEav7rkdKkI3LAeIZy7B0HulKKsmqVQ7
+-----END RSA PRIVATE KEY-----
+`
+	Poem = `Fleas:
+Adam
+Had'em
+
+E.E. Cummings`
+)

lang/testdata/functions-test/hello.tmpl

@@ -0,0 +1 @@
+Hello, ${name}!

lang/testdata/functions-test/hello.txt

@@ -0,0 +1 @@
+hello!

plans/action_string.go

@@ -4,6 +4,19 @@ package plans
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[NoOp-0]
+	_ = x[Create-43]
+	_ = x[Read-8592]
+	_ = x[Update-126]
+	_ = x[DeleteThenCreate-8723]
+	_ = x[CreateThenDelete-177]
+	_ = x[Delete-45]
+}
+
 const (
 	_Action_name_0 = "NoOp"
 	_Action_name_1 = "Create"

plans/objchange/compatible.go

@@ -61,30 +61,22 @@ func assertObjectCompatible(schema *configschema.Block, planned, actual cty.Valu
 		plannedV := planned.GetAttr(name)
 		actualV := actual.GetAttr(name)
 
-		// As a special case, we permit a "planned" block with exactly one
-		// element where all of the "leaf" values are unknown, since that's
-		// what HCL's dynamic block extension generates if the for_each
-		// expression is itself unknown and thus it cannot predict how many
-		// child blocks will get created.
-		switch blockS.Nesting {
-		case configschema.NestingSingle, configschema.NestingGroup:
-			if allLeafValuesUnknown(plannedV) && !plannedV.IsNull() {
-				return errs
-			}
-		case configschema.NestingList, configschema.NestingMap, configschema.NestingSet:
-			if plannedV.IsKnown() && !plannedV.IsNull() && plannedV.LengthInt() == 1 {
-				elemVs := plannedV.AsValueSlice()
-				if allLeafValuesUnknown(elemVs[0]) {
-					return errs
-				}
-			}
-		default:
-			panic(fmt.Sprintf("unsupported nesting mode %s", blockS.Nesting))
-		}
+		// As a special case, if there were any blocks whose leaf attributes
+		// are all unknown then we assume (possibly incorrectly) that the
+		// HCL dynamic block extension is in use with an unknown for_each
+		// argument, and so we will do looser validation here that allows
+		// for those blocks to have expanded into a different number of blocks
+		// if the for_each value is now known.
+		maybeUnknownBlocks := couldHaveUnknownBlockPlaceholder(plannedV, blockS, false)
 
 		path := append(path, cty.GetAttrStep{Name: name})
 		switch blockS.Nesting {
 		case configschema.NestingSingle, configschema.NestingGroup:
+			// If an unknown block placeholder was present then the placeholder
+			// may have expanded out into zero blocks, which is okay.
+			if maybeUnknownBlocks && actualV.IsNull() {
+				continue
+			}
 			moreErrs := assertObjectCompatible(&blockS.Block, plannedV, actualV, path)
 			errs = append(errs, moreErrs...)
 		case configschema.NestingList:
@@ -96,6 +88,14 @@ func assertObjectCompatible(schema *configschema.Block, planned, actual cty.Valu
 				continue
 			}
 
+			if maybeUnknownBlocks {
+				// When unknown blocks are present the final blocks may be
+				// at different indices than the planned blocks, so unfortunately
+				// we can't do our usual checks in this case without generating
+				// false negatives.
+				continue
+			}
+
 			plannedL := plannedV.LengthInt()
 			actualL := actualV.LengthInt()
 			if plannedL != actualL {
@@ -130,10 +130,12 @@ func assertObjectCompatible(schema *configschema.Block, planned, actual cty.Valu
 					moreErrs := assertObjectCompatible(&blockS.Block, plannedEV, actualEV, append(path, cty.GetAttrStep{Name: k}))
 					errs = append(errs, moreErrs...)
 				}
-				for k := range actualAtys {
-					if _, ok := plannedAtys[k]; !ok {
-						errs = append(errs, path.NewErrorf("new block key %q has appeared", k))
-						continue
+				if !maybeUnknownBlocks { // new blocks may appear if unknown blocks were present in the plan
+					for k := range actualAtys {
+						if _, ok := plannedAtys[k]; !ok {
+							errs = append(errs, path.NewErrorf("new block key %q has appeared", k))
+							continue
+						}
 					}
 				}
 			} else {
@@ -142,7 +144,7 @@ func assertObjectCompatible(schema *configschema.Block, planned, actual cty.Valu
 				}
 				plannedL := plannedV.LengthInt()
 				actualL := actualV.LengthInt()
-				if plannedL != actualL {
+				if plannedL != actualL && !maybeUnknownBlocks { // new blocks may appear if unknown blocks were persent in the plan
 					errs = append(errs, path.NewErrorf("block count changed from %d to %d", plannedL, actualL))
 					continue
 				}
@@ -302,18 +304,77 @@ func indexStrForErrors(v cty.Value) string {
 	}
 }
 
-func allLeafValuesUnknown(v cty.Value) bool {
-	seenKnownValue := false
-	cty.Walk(v, func(path cty.Path, cv cty.Value) (bool, error) {
-		if cv.IsNull() {
-			seenKnownValue = true
+// couldHaveUnknownBlockPlaceholder is a heuristic that recognizes how the
+// HCL dynamic block extension behaves when it's asked to expand a block whose
+// for_each argument is unknown. In such cases, it generates a single placeholder
+// block with all leaf attribute values unknown, and once the for_each
+// expression becomes known the placeholder may be replaced with any number
+// of blocks, so object compatibility checks would need to be more liberal.
+//
+// Set "nested" if testing a block that is nested inside a candidate block
+// placeholder; this changes the interpretation of there being no blocks of
+// a type to allow for there being zero nested blocks.
+func couldHaveUnknownBlockPlaceholder(v cty.Value, blockS *configschema.NestedBlock, nested bool) bool {
+	switch blockS.Nesting {
+	case configschema.NestingSingle, configschema.NestingGroup:
+		if nested && v.IsNull() {
+			return true // for nested blocks, a single block being unset doesn't disqualify from being an unknown block placeholder
 		}
-		if cv.Type().IsPrimitiveType() && cv.IsKnown() {
-			seenKnownValue = true
+		return couldBeUnknownBlockPlaceholderElement(v, &blockS.Block)
+	default:
+		// These situations should be impossible for correct providers, but
+		// we permit the legacy SDK to produce some incorrect outcomes
+		// for compatibility with its existing logic, and so we must be
+		// tolerant here.
+		if !v.IsKnown() {
+			return true
 		}
-		return true, nil
-	})
-	return !seenKnownValue
+		if v.IsNull() {
+			return false // treated as if the list were empty, so we would see zero iterations below
+		}
+
+		// For all other nesting modes, our value should be something iterable.
+		for it := v.ElementIterator(); it.Next(); {
+			_, ev := it.Element()
+			if couldBeUnknownBlockPlaceholderElement(ev, &blockS.Block) {
+				return true
+			}
+		}
+
+		// Our default changes depending on whether we're testing the candidate
+		// block itself or something nested inside of it: zero blocks of a type
+		// can never contain a dynamic block placeholder, but a dynamic block
+		// placeholder might contain zero blocks of one of its own nested block
+		// types, if none were set in the config at all.
+		return nested
+	}
+}
+
+func couldBeUnknownBlockPlaceholderElement(v cty.Value, schema *configschema.Block) bool {
+	if v.IsNull() {
+		return false // null value can never be a placeholder element
+	}
+	if !v.IsKnown() {
+		return true // this should never happen for well-behaved providers, but can happen with the legacy SDK opt-outs
+	}
+	for name := range schema.Attributes {
+		av := v.GetAttr(name)
+
+		// Unknown block placeholders contain only unknown or null attribute
+		// values, depending on whether or not a particular attribute was set
+		// explicitly inside the content block. Note that this is imprecise:
+		// non-placeholders can also match this, so this function can generate
+		// false positives.
+		if av.IsKnown() && !av.IsNull() {
+			return false
+		}
+	}
+	for name, blockS := range schema.BlockTypes {
+		if !couldHaveUnknownBlockPlaceholder(v.GetAttr(name), blockS, true) {
+			return false
+		}
+	}
+	return true
 }
 
 // assertSetValuesCompatible checks that each of the elements in a can

plans/objchange/compatible_test.go

@@ -12,6 +12,29 @@ import (
 )
 
 func TestAssertObjectCompatible(t *testing.T) {
+	schemaWithFoo := configschema.Block{
+		Attributes: map[string]*configschema.Attribute{
+			"foo": {Type: cty.String, Optional: true},
+		},
+	}
+	fooBlockValue := cty.ObjectVal(map[string]cty.Value{
+		"foo": cty.StringVal("bar"),
+	})
+	schemaWithFooBar := configschema.Block{
+		Attributes: map[string]*configschema.Attribute{
+			"foo": {Type: cty.String, Optional: true},
+			"bar": {Type: cty.String, Optional: true},
+		},
+	}
+	fooBarBlockValue := cty.ObjectVal(map[string]cty.Value{
+		"foo": cty.StringVal("bar"),
+		"bar": cty.NullVal(cty.String), // simulating the situation where bar isn't set in the config at all
+	})
+	fooBarBlockDynamicPlaceholder := cty.ObjectVal(map[string]cty.Value{
+		"foo": cty.UnknownVal(cty.String),
+		"bar": cty.NullVal(cty.String), // simulating the situation where bar isn't set in the config at all
+	})
+
 	tests := []struct {
 		Schema   *configschema.Block
 		Planned  cty.Value
@@ -681,11 +704,9 @@ func TestAssertObjectCompatible(t *testing.T) {
 				},
 			},
 			cty.ObjectVal(map[string]cty.Value{
-				"id":  cty.UnknownVal(cty.String),
 				"key": cty.EmptyObjectVal,
 			}),
 			cty.ObjectVal(map[string]cty.Value{
-				"id":  cty.UnknownVal(cty.String),
 				"key": cty.EmptyObjectVal,
 			}),
 			nil,
@@ -700,11 +721,9 @@ func TestAssertObjectCompatible(t *testing.T) {
 				},
 			},
 			cty.ObjectVal(map[string]cty.Value{
-				"id":  cty.UnknownVal(cty.String),
 				"key": cty.UnknownVal(cty.EmptyObject),
 			}),
 			cty.ObjectVal(map[string]cty.Value{
-				"id":  cty.UnknownVal(cty.String),
 				"key": cty.EmptyObjectVal,
 			}),
 			nil,
@@ -806,20 +825,18 @@ func TestAssertObjectCompatible(t *testing.T) {
 				BlockTypes: map[string]*configschema.NestedBlock{
 					"key": {
 						Nesting: configschema.NestingList,
-						Block:   configschema.Block{},
+						Block:   schemaWithFoo,
 					},
 				},
 			},
 			cty.ObjectVal(map[string]cty.Value{
-				"id": cty.UnknownVal(cty.String),
 				"key": cty.ListVal([]cty.Value{
-					cty.EmptyObjectVal,
+					fooBlockValue,
 				}),
 			}),
 			cty.ObjectVal(map[string]cty.Value{
-				"id": cty.UnknownVal(cty.String),
 				"key": cty.ListVal([]cty.Value{
-					cty.EmptyObjectVal,
+					fooBlockValue,
 				}),
 			}),
 			nil,
@@ -829,21 +846,19 @@ func TestAssertObjectCompatible(t *testing.T) {
 				BlockTypes: map[string]*configschema.NestedBlock{
 					"key": {
 						Nesting: configschema.NestingList,
-						Block:   configschema.Block{},
+						Block:   schemaWithFoo,
 					},
 				},
 			},
 			cty.ObjectVal(map[string]cty.Value{
-				"id": cty.UnknownVal(cty.String),
 				"key": cty.TupleVal([]cty.Value{
-					cty.EmptyObjectVal,
-					cty.EmptyObjectVal,
+					fooBlockValue,
+					fooBlockValue,
 				}),
 			}),
 			cty.ObjectVal(map[string]cty.Value{
-				"id": cty.UnknownVal(cty.String),
 				"key": cty.TupleVal([]cty.Value{
-					cty.EmptyObjectVal,
+					fooBlockValue,
 				}),
 			}),
 			[]string{
@@ -855,25 +870,77 @@ func TestAssertObjectCompatible(t *testing.T) {
 				BlockTypes: map[string]*configschema.NestedBlock{
 					"key": {
 						Nesting: configschema.NestingList,
-						Block:   configschema.Block{},
+						Block:   schemaWithFoo,
 					},
 				},
 			},
 			cty.ObjectVal(map[string]cty.Value{
-				"id":  cty.UnknownVal(cty.String),
 				"key": cty.TupleVal([]cty.Value{}),
 			}),
 			cty.ObjectVal(map[string]cty.Value{
-				"id": cty.UnknownVal(cty.String),
 				"key": cty.TupleVal([]cty.Value{
-					cty.EmptyObjectVal,
-					cty.EmptyObjectVal,
+					fooBlockValue,
+					fooBlockValue,
 				}),
 			}),
 			[]string{
 				`.key: block count changed from 0 to 2`,
 			},
 		},
+		{
+			&configschema.Block{
+				BlockTypes: map[string]*configschema.NestedBlock{
+					"key": {
+						Nesting: configschema.NestingList,
+						Block:   schemaWithFooBar,
+					},
+				},
+			},
+			cty.ObjectVal(map[string]cty.Value{
+				"key": cty.ListVal([]cty.Value{
+					fooBarBlockDynamicPlaceholder, // the presence of this disables some of our checks
+				}),
+			}),
+			cty.ObjectVal(map[string]cty.Value{
+				"key": cty.ListVal([]cty.Value{
+					cty.ObjectVal(map[string]cty.Value{
+						"foo": cty.StringVal("hello"),
+					}),
+					cty.ObjectVal(map[string]cty.Value{
+						"foo": cty.StringVal("world"),
+					}),
+				}),
+			}),
+			nil, // a single block whose attrs are all unknown is allowed to expand into multiple, because that's how dynamic blocks behave when for_each is unknown
+		},
+		{
+			&configschema.Block{
+				BlockTypes: map[string]*configschema.NestedBlock{
+					"key": {
+						Nesting: configschema.NestingList,
+						Block:   schemaWithFooBar,
+					},
+				},
+			},
+			cty.ObjectVal(map[string]cty.Value{
+				"key": cty.ListVal([]cty.Value{
+					fooBarBlockValue,              // the presence of one static block does not negate that the following element looks like a dynamic placeholder
+					fooBarBlockDynamicPlaceholder, // the presence of this disables some of our checks
+				}),
+			}),
+			cty.ObjectVal(map[string]cty.Value{
+				"key": cty.ListVal([]cty.Value{
+					fooBlockValue,
+					cty.ObjectVal(map[string]cty.Value{
+						"foo": cty.StringVal("hello"),
+					}),
+					cty.ObjectVal(map[string]cty.Value{
+						"foo": cty.StringVal("world"),
+					}),
+				}),
+			}),
+			nil, // as above, the presence of a block whose attrs are all unknown indicates dynamic block expansion, so our usual count checks don't apply
+		},
 
 		// NestingSet blocks
 		{
@@ -881,14 +948,7 @@ func TestAssertObjectCompatible(t *testing.T) {
 				BlockTypes: map[string]*configschema.NestedBlock{
 					"block": {
 						Nesting: configschema.NestingSet,
-						Block: configschema.Block{
-							Attributes: map[string]*configschema.Attribute{
-								"foo": {
-									Type:     cty.String,
-									Optional: true,
-								},
-							},
-						},
+						Block:   schemaWithFoo,
 					},
 				},
 			},
@@ -919,14 +979,7 @@ func TestAssertObjectCompatible(t *testing.T) {
 				BlockTypes: map[string]*configschema.NestedBlock{
 					"block": {
 						Nesting: configschema.NestingSet,
-						Block: configschema.Block{
-							Attributes: map[string]*configschema.Attribute{
-								"foo": {
-									Type:     cty.String,
-									Optional: true,
-								},
-							},
-						},
+						Block:   schemaWithFoo,
 					},
 				},
 			},
@@ -957,14 +1010,7 @@ func TestAssertObjectCompatible(t *testing.T) {
 				BlockTypes: map[string]*configschema.NestedBlock{
 					"block": {
 						Nesting: configschema.NestingSet,
-						Block: configschema.Block{
-							Attributes: map[string]*configschema.Attribute{
-								"foo": {
-									Type:     cty.String,
-									Optional: true,
-								},
-							},
-						},
+						Block:   schemaWithFoo,
 					},
 				},
 			},
@@ -995,14 +1041,7 @@ func TestAssertObjectCompatible(t *testing.T) {
 				BlockTypes: map[string]*configschema.NestedBlock{
 					"block": {
 						Nesting: configschema.NestingSet,
-						Block: configschema.Block{
-							Attributes: map[string]*configschema.Attribute{
-								"foo": {
-									Type:     cty.String,
-									Optional: true,
-								},
-							},
-						},
+						Block:   schemaWithFoo,
 					},
 				},
 			},
@@ -1038,14 +1077,7 @@ func TestAssertObjectCompatible(t *testing.T) {
 				BlockTypes: map[string]*configschema.NestedBlock{
 					"block": {
 						Nesting: configschema.NestingSet,
-						Block: configschema.Block{
-							Attributes: map[string]*configschema.Attribute{
-								"foo": {
-									Type:     cty.String,
-									Optional: true,
-								},
-							},
-						},
+						Block:   schemaWithFoo,
 					},
 				},
 			},
@@ -1070,7 +1102,7 @@ func TestAssertObjectCompatible(t *testing.T) {
 				}),
 			}),
 			[]string{
-				`.block: planned set element cty.Value{ty: cty.Object(map[string]cty.Type{"foo":cty.String}), v: map[string]interface {}{"foo":"hello"}} does not correlate with any element in actual`,
+				`.block: planned set element cty.ObjectVal(map[string]cty.Value{"foo":cty.StringVal("hello")}) does not correlate with any element in actual`,
 			},
 		},
 		{
@@ -1082,14 +1114,7 @@ func TestAssertObjectCompatible(t *testing.T) {
 				BlockTypes: map[string]*configschema.NestedBlock{
 					"block": {
 						Nesting: configschema.NestingSet,
-						Block: configschema.Block{
-							Attributes: map[string]*configschema.Attribute{
-								"foo": {
-									Type:     cty.String,
-									Optional: true,
-								},
-							},
-						},
+						Block:   schemaWithFoo,
 					},
 				},
 			},
@@ -1112,8 +1137,8 @@ func TestAssertObjectCompatible(t *testing.T) {
 		},
 	}
 
-	for _, test := range tests {
-		t.Run(fmt.Sprintf("%#v and %#v", test.Planned, test.Actual), func(t *testing.T) {
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%02d: %#v and %#v", i, test.Planned, test.Actual), func(t *testing.T) {
 			errs := AssertObjectCompatible(test.Schema, test.Planned, test.Actual)
 
 			wantErrs := make(map[string]struct{})

plugin/discovery/get.go

@@ -228,9 +228,9 @@ func (i *ProviderInstaller) Get(provider string, req Constraints) (PluginMeta, t
 		}
 	}
 
-	printedProviderName := fmt.Sprintf("%s (%s)", provider, providerSource)
-	i.Ui.Info(fmt.Sprintf("- Downloading plugin for provider %q (%s)...", printedProviderName, versionMeta.Version))
-	log.Printf("[DEBUG] getting provider %q version %q", printedProviderName, versionMeta.Version)
+	printedProviderName := fmt.Sprintf("%q (%s)", provider, providerSource)
+	i.Ui.Info(fmt.Sprintf("- Downloading plugin for provider %s %s...", printedProviderName, versionMeta.Version))
+	log.Printf("[DEBUG] getting provider %s version %q", printedProviderName, versionMeta.Version)
 	err = i.install(provider, v, providerURL)
 	if err != nil {
 		return PluginMeta{}, diags, err
@@ -298,7 +298,7 @@ func (i *ProviderInstaller) install(provider string, version Version, url string
 		// check if the target dir exists, and create it if not
 		var err error
 		if _, StatErr := os.Stat(i.Dir); os.IsNotExist(StatErr) {
-			err = os.Mkdir(i.Dir, 0700)
+			err = os.MkdirAll(i.Dir, 0700)
 		}
 		if err != nil {
 			return err

plugin/grpc_provider.go

@@ -3,7 +3,6 @@ package plugin
 import (
 	"context"
 	"errors"
-	"io"
 	"log"
 	"sync"
 
@@ -45,9 +44,9 @@ type GRPCProvider struct {
 	// This allows the GRPCProvider a way to shutdown the plugin process.
 	PluginClient *plugin.Client
 
-	// TestListener contains a net.Conn to close when the GRPCProvider is being
+	// TestServer contains a grpc.Server to close when the GRPCProvider is being
 	// used in an end to end test of a provider.
-	TestListener io.Closer
+	TestServer *grpc.Server
 
 	// Proto client use to make the grpc service calls.
 	client proto.ProviderClient
@@ -544,9 +543,9 @@ func (p *GRPCProvider) ReadDataSource(r providers.ReadDataSourceRequest) (resp p
 func (p *GRPCProvider) Close() error {
 	log.Printf("[TRACE] GRPCProvider: Close")
 
-	// close the remote listener if we're running within a test
-	if p.TestListener != nil {
-		p.TestListener.Close()
+	// Make sure to stop the server if we're not running within go-plugin.
+	if p.TestServer != nil {
+		p.TestServer.Stop()
 	}
 
 	// Check this since it's not automatically inserted during plugin creation.

states/eachmode_string.go

@@ -4,6 +4,15 @@ package states
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[NoEach-0]
+	_ = x[EachList-76]
+	_ = x[EachMap-77]
+}
+
 const (
 	_EachMode_name_0 = "NoEach"
 	_EachMode_name_1 = "EachListEachMap"

states/objectstatus_string.go

@@ -4,6 +4,15 @@ package states
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[ObjectReady-82]
+	_ = x[ObjectTainted-84]
+	_ = x[ObjectPlanned-80]
+}
+
 const (
 	_ObjectStatus_name_0 = "ObjectPlanned"
 	_ObjectStatus_name_1 = "ObjectReady"

states/statefile/testdata/roundtrip/v3-simple.in.tfstate

@@ -20,7 +20,8 @@
                     "type": "null_resource",
                     "depends_on": [
                         "null_resource.foo.*",
-                        "null_resource.foobar"
+                        "null_resource.foobar",
+                        "null_resource.foobar.1"
                     ],
                     "primary": {
                         "id": "5388490630832483079",
@@ -87,4 +88,4 @@
             "depends_on": []
         }
     ]
-}
+}

states/statefile/testdata/roundtrip/v3-simple.out.tfstate

@@ -25,7 +25,8 @@
                     },
                     "depends_on": [
                         "null_resource.foo",
-                        "null_resource.foobar"
+                        "null_resource.foobar",
+                        "null_resource.foobar[1]"
                     ]
                 }
             ]
@@ -74,4 +75,4 @@
             ]
         }
     ]
-}
+}

states/statefile/version3_upgrade.go

@@ -301,7 +301,7 @@ func upgradeInstanceObjectV3ToV4(rsOld *resourceStateV2, isOld *instanceStateV2,
 
 	dependencies := make([]string, len(rsOld.Dependencies))
 	for i, v := range rsOld.Dependencies {
-		dependencies[i] = strings.TrimSuffix(v, ".*")
+		dependencies[i] = parseLegacyDependency(v)
 	}
 
 	return &instanceObjectStateV4{
@@ -413,3 +413,19 @@ func simplifyImpliedValueType(ty cty.Type) cty.Type {
 		return ty
 	}
 }
+
+func parseLegacyDependency(s string) string {
+	parts := strings.Split(s, ".")
+	ret := parts[0]
+	for _, part := range parts[1:] {
+		if part == "*" {
+			break
+		}
+		if i, err := strconv.Atoi(part); err == nil {
+			ret = ret + fmt.Sprintf("[%d]", i)
+			break
+		}
+		ret = ret + "." + part
+	}
+	return ret
+}

states/statemgr/snapshotmetarel_string.go

@@ -4,6 +4,17 @@ package statemgr
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[SnapshotOlder-60]
+	_ = x[SnapshotNewer-62]
+	_ = x[SnapshotEqual-61]
+	_ = x[SnapshotUnrelated-33]
+	_ = x[SnapshotLegacy-63]
+}
+
 const (
 	_SnapshotMetaRel_name_0 = "SnapshotUnrelated"
 	_SnapshotMetaRel_name_1 = "SnapshotOlderSnapshotEqualSnapshotNewerSnapshotLegacy"

terraform/context_refresh_test.go

@@ -976,6 +976,65 @@ func TestContext2Refresh_stateBasic(t *testing.T) {
 	}
 }
 
+func TestContext2Refresh_dataCount(t *testing.T) {
+	p := testProvider("test")
+	m := testModule(t, "refresh-data-count")
+
+	// This test is verifying that a data resource count can refer to a
+	// resource attribute that can't be known yet during refresh (because
+	// the resource in question isn't in the state at all). In that case,
+	// we skip the data resource during refresh and process it during the
+	// subsequent plan step instead.
+	//
+	// Normally it's an error for "count" to be computed, but during the
+	// refresh step we allow it because we _expect_ to be working with an
+	// incomplete picture of the world sometimes, particularly when we're
+	// creating object for the first time against an empty state.
+	//
+	// For more information, see:
+	//    https://github.com/hashicorp/terraform/issues/21047
+
+	p.GetSchemaReturn = &ProviderSchema{
+		ResourceTypes: map[string]*configschema.Block{
+			"test": {
+				Attributes: map[string]*configschema.Attribute{
+					"things": {Type: cty.List(cty.String), Optional: true},
+				},
+			},
+		},
+		DataSources: map[string]*configschema.Block{
+			"test": {},
+		},
+	}
+
+	ctx := testContext2(t, &ContextOpts{
+		ProviderResolver: providers.ResolverFixed(
+			map[string]providers.Factory{
+				"test": testProviderFuncFixed(p),
+			},
+		),
+		Config: m,
+	})
+
+	s, diags := ctx.Refresh()
+	if p.ReadResourceCalled {
+		// The managed resource doesn't exist in the state yet, so there's
+		// nothing to refresh.
+		t.Errorf("ReadResource was called, but should not have been")
+	}
+	if p.ReadDataSourceCalled {
+		// The data resource should've been skipped because its count cannot
+		// be determined yet.
+		t.Errorf("ReadDataSource was called, but should not have been")
+	}
+
+	if diags.HasErrors() {
+		t.Fatalf("refresh errors: %s", diags.Err())
+	}
+
+	checkStateString(t, s, `<no state>`)
+}
+
 func TestContext2Refresh_dataOrphan(t *testing.T) {
 	p := testProvider("null")
 	state := MustShimLegacyState(&State{

terraform/eval_count.go

@@ -23,36 +23,10 @@ import (
 // the "count" behavior should not be enabled for this resource at all.
 //
 // If error diagnostics are returned then the result is always the meaningless
-// placeholder value -1, except in one case: if the count expression evaluates
-// to an unknown number value then the result is zero, allowing this situation
-// to be treated by the caller as special if needed. For example, an early
-// graph walk may wish to just silently skip resources with unknown counts
-// to allow them to be dealt with in a later graph walk where more information
-// is available.
+// placeholder value -1.
 func evaluateResourceCountExpression(expr hcl.Expression, ctx EvalContext) (int, tfdiags.Diagnostics) {
-	if expr == nil {
-		return -1, nil
-	}
-
-	var diags tfdiags.Diagnostics
-	var count int
-
-	countVal, countDiags := ctx.EvaluateExpr(expr, cty.Number, nil)
-	diags = diags.Append(countDiags)
-	if diags.HasErrors() {
-		return -1, diags
-	}
-
-	switch {
-	case countVal.IsNull():
-		diags = diags.Append(&hcl.Diagnostic{
-			Severity: hcl.DiagError,
-			Summary:  "Invalid count argument",
-			Detail:   `The given "count" argument value is null. An integer is required.`,
-			Subject:  expr.Range().Ptr(),
-		})
-		return -1, diags
-	case !countVal.IsKnown():
+	count, known, diags := evaluateResourceCountExpressionKnown(expr, ctx)
+	if !known {
 		// Currently this is a rather bad outcome from a UX standpoint, since we have
 		// no real mechanism to deal with this situation and all we can do is produce
 		// an error message.
@@ -65,11 +39,36 @@ func evaluateResourceCountExpression(expr hcl.Expression, ctx EvalContext) (int,
 			Detail:   `The "count" value depends on resource attributes that cannot be determined until apply, so Terraform cannot predict how many instances will be created. To work around this, use the -target argument to first apply only the resources that the count depends on.`,
 			Subject:  expr.Range().Ptr(),
 		})
-		// We return zero+errors in this one case to allow callers to handle
-		// an unknown count as special. This is rarely necessary, but is used
-		// by the validate walk in particular so that it can just skip
-		// validation in this case, assuming a later walk will take care of it.
-		return 0, diags
+	}
+	return count, diags
+}
+
+// evaluateResourceCountExpressionKnown is like evaluateResourceCountExpression
+// except that it handles an unknown result by returning count = 0 and
+// a known = false, rather than by reporting the unknown value as an error
+// diagnostic.
+func evaluateResourceCountExpressionKnown(expr hcl.Expression, ctx EvalContext) (count int, known bool, diags tfdiags.Diagnostics) {
+	if expr == nil {
+		return -1, true, nil
+	}
+
+	countVal, countDiags := ctx.EvaluateExpr(expr, cty.Number, nil)
+	diags = diags.Append(countDiags)
+	if diags.HasErrors() {
+		return -1, true, diags
+	}
+
+	switch {
+	case countVal.IsNull():
+		diags = diags.Append(&hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Invalid count argument",
+			Detail:   `The given "count" argument value is null. An integer is required.`,
+			Subject:  expr.Range().Ptr(),
+		})
+		return -1, true, diags
+	case !countVal.IsKnown():
+		return 0, false, diags
 	}
 
 	err := gocty.FromCtyValue(countVal, &count)
@@ -80,7 +79,7 @@ func evaluateResourceCountExpression(expr hcl.Expression, ctx EvalContext) (int,
 			Detail:   fmt.Sprintf(`The given "count" argument value is unsuitable: %s.`, err),
 			Subject:  expr.Range().Ptr(),
 		})
-		return -1, diags
+		return -1, true, diags
 	}
 	if count < 0 {
 		diags = diags.Append(&hcl.Diagnostic{
@@ -89,10 +88,10 @@ func evaluateResourceCountExpression(expr hcl.Expression, ctx EvalContext) (int,
 			Detail:   `The given "count" argument value is unsuitable: negative numbers are not supported.`,
 			Subject:  expr.Range().Ptr(),
 		})
-		return -1, diags
+		return -1, true, diags
 	}
 
-	return count, diags
+	return count, true, diags
 }
 
 // fixResourceCountSetTransition is a helper function to fix up the state when a

terraform/eval_diff.go

@@ -368,7 +368,7 @@ func (n *EvalDiff) Eval(ctx EvalContext) (interface{}, error) {
 		}
 		plannedNewVal = resp.PlannedState
 		plannedPrivate = resp.PlannedPrivate
-		for _, err := range schema.ImpliedType().TestConformance(plannedNewVal.Type()) {
+		for _, err := range plannedNewVal.Type().TestConformance(schema.ImpliedType()) {
 			diags = diags.Append(tfdiags.Sourceless(
 				tfdiags.Error,
 				"Provider produced invalid plan",

terraform/eval_read_data.go

@@ -291,125 +291,6 @@ func (n *EvalReadData) Eval(ctx EvalContext) (interface{}, error) {
 	return nil, diags.ErrWithWarnings()
 }
 
-// EvalReadDataDiff is an EvalNode implementation that executes a data
-// resource's ReadDataDiff method to discover what attributes it exports.
-type EvalReadDataDiff struct {
-	Addr           addrs.ResourceInstance
-	Config         *configs.Resource
-	ProviderAddr   addrs.AbsProviderConfig
-	ProviderSchema **ProviderSchema
-
-	Output            **plans.ResourceInstanceChange
-	OutputValue       *cty.Value
-	OutputConfigValue *cty.Value
-	OutputState       **states.ResourceInstanceObject
-
-	// Set Previous when re-evaluating diff during apply, to ensure that
-	// the "Destroy" flag is preserved.
-	Previous **plans.ResourceInstanceChange
-}
-
-func (n *EvalReadDataDiff) Eval(ctx EvalContext) (interface{}, error) {
-	absAddr := n.Addr.Absolute(ctx.Path())
-
-	if n.ProviderSchema == nil || *n.ProviderSchema == nil {
-		return nil, fmt.Errorf("provider schema not available for %s", n.Addr)
-	}
-
-	var diags tfdiags.Diagnostics
-	var change *plans.ResourceInstanceChange
-	var configVal cty.Value
-
-	if n.Previous != nil && *n.Previous != nil && (*n.Previous).Action == plans.Delete {
-		// If we're re-diffing for a diff that was already planning to
-		// destroy, then we'll just continue with that plan.
-
-		nullVal := cty.NullVal(cty.DynamicPseudoType)
-		err := ctx.Hook(func(h Hook) (HookAction, error) {
-			return h.PreDiff(absAddr, states.CurrentGen, nullVal, nullVal)
-		})
-		if err != nil {
-			return nil, err
-		}
-
-		change = &plans.ResourceInstanceChange{
-			Addr:         absAddr,
-			ProviderAddr: n.ProviderAddr,
-			Change: plans.Change{
-				Action: plans.Delete,
-				Before: nullVal,
-				After:  nullVal,
-			},
-		}
-	} else {
-		config := *n.Config
-		providerSchema := *n.ProviderSchema
-		schema, _ := providerSchema.SchemaForResourceAddr(n.Addr.ContainingResource())
-		if schema == nil {
-			// Should be caught during validation, so we don't bother with a pretty error here
-			return nil, fmt.Errorf("provider does not support data source %q", n.Addr.Resource.Type)
-		}
-
-		objTy := schema.ImpliedType()
-		priorVal := cty.NullVal(objTy) // for data resources, prior is always null because we start fresh every time
-
-		keyData := EvalDataForInstanceKey(n.Addr.Key)
-
-		var configDiags tfdiags.Diagnostics
-		configVal, _, configDiags = ctx.EvaluateBlock(config.Config, schema, nil, keyData)
-		diags = diags.Append(configDiags)
-		if configDiags.HasErrors() {
-			return nil, diags.Err()
-		}
-
-		proposedNewVal := objchange.ProposedNewObject(schema, priorVal, configVal)
-
-		err := ctx.Hook(func(h Hook) (HookAction, error) {
-			return h.PreDiff(absAddr, states.CurrentGen, priorVal, proposedNewVal)
-		})
-		if err != nil {
-			return nil, err
-		}
-
-		change = &plans.ResourceInstanceChange{
-			Addr:         absAddr,
-			ProviderAddr: n.ProviderAddr,
-			Change: plans.Change{
-				Action: plans.Read,
-				Before: priorVal,
-				After:  proposedNewVal,
-			},
-		}
-	}
-
-	err := ctx.Hook(func(h Hook) (HookAction, error) {
-		return h.PostDiff(absAddr, states.CurrentGen, change.Action, change.Before, change.After)
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	if n.Output != nil {
-		*n.Output = change
-	}
-	if n.OutputValue != nil {
-		*n.OutputValue = change.After
-	}
-	if n.OutputConfigValue != nil {
-		*n.OutputConfigValue = configVal
-	}
-
-	if n.OutputState != nil {
-		state := &states.ResourceInstanceObject{
-			Value:  change.After,
-			Status: states.ObjectReady,
-		}
-		*n.OutputState = state
-	}
-
-	return nil, diags.ErrWithWarnings()
-}
-
 // EvalReadDataApply is an EvalNode implementation that executes a data
 // resource's ReadDataApply method to read data from the data source.
 type EvalReadDataApply struct {

terraform/eval_state_upgrade.go

@@ -19,26 +19,13 @@ import (
 // If any errors occur during upgrade, error diagnostics are returned. In that
 // case it is not safe to proceed with using the original state object.
 func UpgradeResourceState(addr addrs.AbsResourceInstance, provider providers.Interface, src *states.ResourceInstanceObjectSrc, currentSchema *configschema.Block, currentVersion uint64) (*states.ResourceInstanceObjectSrc, tfdiags.Diagnostics) {
-	currentTy := currentSchema.ImpliedType()
-
-	// If the state is currently in flatmap format and the current schema
-	// contains DynamicPseudoType attributes then we won't be able to convert
-	// it to JSON without the provider's help even if the schema version matches,
-	// since only the provider knows how to interpret the dynamic attribute
-	// value in flatmap format to convert it to JSON.
-	schemaHasDynamic := currentTy.HasDynamicTypes()
-	stateIsFlatmap := len(src.AttrsJSON) == 0
-	forceProviderUpgrade := schemaHasDynamic && stateIsFlatmap
-
-	if src.SchemaVersion == currentVersion && !forceProviderUpgrade {
-		// No upgrading required, then.
-		return src, nil
-	}
 	if addr.Resource.Resource.Mode != addrs.ManagedResourceMode {
 		// We only do state upgrading for managed resources.
 		return src, nil
 	}
 
+	stateIsFlatmap := len(src.AttrsJSON) == 0
+
 	providerType := addr.Resource.Resource.DefaultProviderConfig().Type
 	if src.SchemaVersion > currentVersion {
 		log.Printf("[TRACE] UpgradeResourceState: can't downgrade state for %s from version %d to %d", addr, src.SchemaVersion, currentVersion)
@@ -60,7 +47,11 @@ func UpgradeResourceState(addr addrs.AbsResourceInstance, provider providers.Int
 	// v0.12, this also includes translating from legacy flatmap to new-style
 	// representation, since only the provider has enough information to
 	// understand a flatmap built against an older schema.
-	log.Printf("[TRACE] UpgradeResourceState: upgrading state for %s from version %d to %d using provider %q", addr, src.SchemaVersion, currentVersion, providerType)
+	if src.SchemaVersion != currentVersion {
+		log.Printf("[TRACE] UpgradeResourceState: upgrading state for %s from version %d to %d using provider %q", addr, src.SchemaVersion, currentVersion, providerType)
+	} else {
+		log.Printf("[TRACE] UpgradeResourceState: schema version of %s is still %d; calling provider %q for any other minor fixups", addr, currentVersion, providerType)
+	}
 
 	req := providers.UpgradeResourceStateRequest{
 		TypeName: addr.Resource.Resource.Type,

terraform/graphtype_string.go

@@ -4,6 +4,20 @@ package terraform
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[GraphTypeInvalid-0]
+	_ = x[GraphTypeLegacy-1]
+	_ = x[GraphTypeRefresh-2]
+	_ = x[GraphTypePlan-3]
+	_ = x[GraphTypePlanDestroy-4]
+	_ = x[GraphTypeApply-5]
+	_ = x[GraphTypeValidate-6]
+	_ = x[GraphTypeEval-7]
+}
+
 const _GraphType_name = "GraphTypeInvalidGraphTypeLegacyGraphTypeRefreshGraphTypePlanGraphTypePlanDestroyGraphTypeApplyGraphTypeValidateGraphTypeEval"
 
 var _GraphType_index = [...]uint8{0, 16, 31, 47, 60, 80, 94, 111, 124}

terraform/instancetype_string.go

@@ -4,6 +4,16 @@ package terraform
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[TypeInvalid-0]
+	_ = x[TypePrimary-1]
+	_ = x[TypeTainted-2]
+	_ = x[TypeDeposed-3]
+}
+
 const _InstanceType_name = "TypeInvalidTypePrimaryTypeTaintedTypeDeposed"
 
 var _InstanceType_index = [...]uint8{0, 11, 22, 33, 44}

terraform/node_data_refresh.go

@@ -27,11 +27,16 @@ var (
 func (n *NodeRefreshableDataResource) DynamicExpand(ctx EvalContext) (*Graph, error) {
 	var diags tfdiags.Diagnostics
 
-	count, countDiags := evaluateResourceCountExpression(n.Config.Count, ctx)
+	count, countKnown, countDiags := evaluateResourceCountExpressionKnown(n.Config.Count, ctx)
 	diags = diags.Append(countDiags)
 	if countDiags.HasErrors() {
 		return nil, diags.Err()
 	}
+	if !countKnown {
+		// If the count isn't known yet, we'll skip refreshing and try expansion
+		// again during the plan walk.
+		return nil, nil
+	}
 
 	// Next we need to potentially rename an instance address in the state
 	// if we're transitioning whether "count" is set at all.

terraform/node_resource_apply_instance.go

@@ -10,6 +10,7 @@ import (
 	"github.com/hashicorp/terraform/plans"
 	"github.com/hashicorp/terraform/providers"
 	"github.com/hashicorp/terraform/states"
+	"github.com/hashicorp/terraform/tfdiags"
 )
 
 // NodeApplyableResourceInstance represents a resource instance that is
@@ -107,6 +108,27 @@ func (n *NodeApplyableResourceInstance) EvalTree() EvalNode {
 	// Determine the dependencies for the state.
 	stateDeps := n.StateReferences()
 
+	if n.Config == nil {
+		// This should not be possible, but we've got here in at least one
+		// case as discussed in the following issue:
+		//    https://github.com/hashicorp/terraform/issues/21258
+		// To avoid an outright crash here, we'll instead return an explicit
+		// error.
+		var diags tfdiags.Diagnostics
+		diags = diags.Append(tfdiags.Sourceless(
+			tfdiags.Error,
+			"Resource node has no configuration attached",
+			fmt.Sprintf(
+				"The graph node for %s has no configuration attached to it. This suggests a bug in Terraform's apply graph builder; please report it!",
+				addr,
+			),
+		))
+		err := diags.Err()
+		return &EvalReturnError{
+			Error: &err,
+		}
+	}
+
 	// Eval info is different depending on what kind of resource this is
 	switch n.Config.Mode {
 	case addrs.ManagedResourceMode:

terraform/node_resource_plan_instance.go

@@ -81,8 +81,31 @@ func (n *NodePlannableResourceInstance) evalTreeDataResource(addr addrs.AbsResou
 			// here.
 			&EvalIf{
 				If: func(ctx EvalContext) (bool, error) {
-					if state != nil && state.Status != states.ObjectPlanned {
-						return true, EvalEarlyExitError{}
+					depChanges := false
+
+					// Check and see if any of our dependencies have changes.
+					changes := ctx.Changes()
+					for _, d := range n.StateReferences() {
+						ri, ok := d.(addrs.ResourceInstance)
+						if !ok {
+							continue
+						}
+						change := changes.GetResourceInstanceChange(ri.Absolute(ctx.Path()), states.CurrentGen)
+						if change != nil && change.Action != plans.NoOp {
+							depChanges = true
+							break
+						}
+					}
+
+					refreshed := state != nil && state.Status != states.ObjectPlanned
+
+					// If there are no dependency changes, and it's not a forced
+					// read because we there was no Refresh, then we don't need
+					// to re-read. If any dependencies have changes, it means
+					// our config may also have changes and we need to Read the
+					// data source again.
+					if !depChanges && refreshed {
+						return false, EvalEarlyExitError{}
 					}
 					return true, nil
 				},

terraform/provider_mock.go

@@ -6,6 +6,7 @@ import (
 	"sync"
 
 	"github.com/zclconf/go-cty/cty"
+	ctyjson "github.com/zclconf/go-cty/cty/json"
 
 	"github.com/hashicorp/terraform/config"
 	"github.com/hashicorp/terraform/config/hcl2shim"
@@ -191,6 +192,10 @@ func (p *MockProvider) UpgradeResourceState(r providers.UpgradeResourceStateRequ
 	p.Lock()
 	defer p.Unlock()
 
+	schemas := p.getSchema()
+	schema := schemas.ResourceTypes[r.TypeName]
+	schemaType := schema.Block.ImpliedType()
+
 	p.UpgradeResourceStateCalled = true
 	p.UpgradeResourceStateRequest = r
 
@@ -198,7 +203,28 @@ func (p *MockProvider) UpgradeResourceState(r providers.UpgradeResourceStateRequ
 		return p.UpgradeResourceStateFn(r)
 	}
 
-	return p.UpgradeResourceStateResponse
+	resp := p.UpgradeResourceStateResponse
+
+	if resp.UpgradedState == cty.NilVal {
+		switch {
+		case r.RawStateFlatmap != nil:
+			v, err := hcl2shim.HCL2ValueFromFlatmap(r.RawStateFlatmap, schemaType)
+			if err != nil {
+				resp.Diagnostics = resp.Diagnostics.Append(err)
+				return resp
+			}
+			resp.UpgradedState = v
+		case len(r.RawStateJSON) > 0:
+			v, err := ctyjson.Unmarshal(r.RawStateJSON, schemaType)
+
+			if err != nil {
+				resp.Diagnostics = resp.Diagnostics.Append(err)
+				return resp
+			}
+			resp.UpgradedState = v
+		}
+	}
+	return resp
 }
 
 func (p *MockProvider) Configure(r providers.ConfigureRequest) providers.ConfigureResponse {

terraform/resource.go

@@ -2,7 +2,6 @@ package terraform
 
 import (
 	"fmt"
-	"log"
 	"reflect"
 	"sort"
 	"strconv"
@@ -236,7 +235,7 @@ func NewResourceConfigShimmed(val cty.Value, schema *configschema.Block) *Resour
 		// schema here so that we can preserve the expected invariant
 		// that an attribute is always either wholly known or wholly unknown, while
 		// a child block can be partially unknown.
-		ret.ComputedKeys = newResourceConfigShimmedComputedKeys(val, schema, "")
+		ret.ComputedKeys = newResourceConfigShimmedComputedKeys(val, "")
 	} else {
 		ret.Config = make(map[string]interface{})
 	}
@@ -245,72 +244,45 @@ func NewResourceConfigShimmed(val cty.Value, schema *configschema.Block) *Resour
 	return ret
 }
 
-// newResourceConfigShimmedComputedKeys finds all of the unknown values in the
-// given object, which must conform to the given schema, returning them in
-// the format that's expected for ResourceConfig.ComputedKeys.
-func newResourceConfigShimmedComputedKeys(obj cty.Value, schema *configschema.Block, prefix string) []string {
+// Record the any config values in ComputedKeys. This field had been unused in
+// helper/schema, but in the new protocol we're using this so that the SDK can
+// now handle having an unknown collection. The legacy diff code doesn't
+// properly handle the unknown, because it can't be expressed in the same way
+// between the config and diff.
+func newResourceConfigShimmedComputedKeys(val cty.Value, path string) []string {
 	var ret []string
-	ty := obj.Type()
+	ty := val.Type()
 
-	if schema == nil {
-		log.Printf("[WARN] NewResourceConfigShimmed: can't identify computed keys because no schema is available")
-		return nil
+	if val.IsNull() {
+		return ret
 	}
 
-	for attrName := range schema.Attributes {
-		if !ty.HasAttribute(attrName) {
-			// Should never happen, but we'll tolerate it anyway
-			continue
-		}
-
-		attrVal := obj.GetAttr(attrName)
-		if !attrVal.IsWhollyKnown() {
-			ret = append(ret, prefix+attrName)
+	if !val.IsKnown() {
+		// we shouldn't have an entirely unknown resource, but prevent empty
+		// strings just in case
+		if len(path) > 0 {
+			ret = append(ret, path)
 		}
+		return ret
 	}
 
-	for typeName, blockS := range schema.BlockTypes {
-		if !ty.HasAttribute(typeName) {
-			// Should never happen, but we'll tolerate it anyway
-			continue
-		}
-
-		blockVal := obj.GetAttr(typeName)
-		if blockVal.IsNull() || !blockVal.IsKnown() {
-			continue
-		}
-
-		switch blockS.Nesting {
-		case configschema.NestingSingle, configschema.NestingGroup:
-			keys := newResourceConfigShimmedComputedKeys(blockVal, &blockS.Block, fmt.Sprintf("%s%s.", prefix, typeName))
+	if path != "" {
+		path += "."
+	}
+	switch {
+	case ty.IsListType(), ty.IsTupleType(), ty.IsSetType():
+		i := 0
+		for it := val.ElementIterator(); it.Next(); i++ {
+			_, subVal := it.Element()
+			keys := newResourceConfigShimmedComputedKeys(subVal, fmt.Sprintf("%s%d", path, i))
+			ret = append(ret, keys...)
+		}
+
+	case ty.IsMapType(), ty.IsObjectType():
+		for it := val.ElementIterator(); it.Next(); {
+			subK, subVal := it.Element()
+			keys := newResourceConfigShimmedComputedKeys(subVal, fmt.Sprintf("%s%s", path, subK.AsString()))
 			ret = append(ret, keys...)
-		case configschema.NestingList, configschema.NestingSet:
-			// Producing computed keys items for sets is not really useful
-			// since they are not usefully addressable anyway, but we'll treat
-			// them like lists just so that ret.ComputedKeys accounts for them
-			// all. Our legacy system didn't support sets here anyway, so
-			// treating them as lists is the most accurate translation. Although
-			// set traversal isn't in any particular order, it is _stable_ as
-			// long as the list isn't mutated, and so we know we'll see the
-			// same order here as hcl2shim.ConfigValueFromHCL2 would've seen
-			// inside NewResourceConfigShimmed above.
-			i := 0
-			for it := blockVal.ElementIterator(); it.Next(); i++ {
-				_, subVal := it.Element()
-				subPrefix := fmt.Sprintf("%s.%s%d.", typeName, prefix, i)
-				keys := newResourceConfigShimmedComputedKeys(subVal, &blockS.Block, subPrefix)
-				ret = append(ret, keys...)
-			}
-		case configschema.NestingMap:
-			for it := blockVal.ElementIterator(); it.Next(); {
-				subK, subVal := it.Element()
-				subPrefix := fmt.Sprintf("%s.%s%s.", typeName, prefix, subK.AsString())
-				keys := newResourceConfigShimmedComputedKeys(subVal, &blockS.Block, subPrefix)
-				ret = append(ret, keys...)
-			}
-		default:
-			// Should never happen, since the above is exhaustive.
-			panic(fmt.Errorf("unsupported block nesting type %s", blockS.Nesting))
 		}
 	}
 

terraform/resource_test.go

@@ -919,6 +919,7 @@ func TestNewResourceConfigShimmed(t *testing.T) {
 				},
 			},
 			Expected: &ResourceConfig{
+				ComputedKeys: []string{"bar", "baz"},
 				Raw: map[string]interface{}{
 					"bar": config.UnknownVariableValue,
 					"baz": config.UnknownVariableValue,
@@ -929,6 +930,167 @@ func TestNewResourceConfigShimmed(t *testing.T) {
 				},
 			},
 		},
+		{
+			Name: "unknown in nested blocks",
+			Val: cty.ObjectVal(map[string]cty.Value{
+				"bar": cty.ListVal([]cty.Value{
+					cty.ObjectVal(map[string]cty.Value{
+						"baz": cty.ListVal([]cty.Value{
+							cty.ObjectVal(map[string]cty.Value{
+								"list": cty.UnknownVal(cty.List(cty.String)),
+							}),
+						}),
+					}),
+				}),
+			}),
+			Schema: &configschema.Block{
+				BlockTypes: map[string]*configschema.NestedBlock{
+					"bar": {
+						Block: configschema.Block{
+							BlockTypes: map[string]*configschema.NestedBlock{
+								"baz": {
+									Block: configschema.Block{
+										Attributes: map[string]*configschema.Attribute{
+											"list": {Type: cty.List(cty.String),
+												Optional: true,
+											},
+										},
+									},
+									Nesting: configschema.NestingList,
+								},
+							},
+						},
+						Nesting: configschema.NestingList,
+					},
+				},
+			},
+			Expected: &ResourceConfig{
+				ComputedKeys: []string{"bar.0.baz.0.list"},
+				Raw: map[string]interface{}{
+					"bar": []interface{}{map[string]interface{}{
+						"baz": []interface{}{map[string]interface{}{
+							"list": "74D93920-ED26-11E3-AC10-0800200C9A66",
+						}},
+					}},
+				},
+				Config: map[string]interface{}{
+					"bar": []interface{}{map[string]interface{}{
+						"baz": []interface{}{map[string]interface{}{
+							"list": "74D93920-ED26-11E3-AC10-0800200C9A66",
+						}},
+					}},
+				},
+			},
+		},
+		{
+			Name: "unknown in set",
+			Val: cty.ObjectVal(map[string]cty.Value{
+				"bar": cty.SetVal([]cty.Value{
+					cty.ObjectVal(map[string]cty.Value{
+						"val": cty.UnknownVal(cty.String),
+					}),
+				}),
+			}),
+			Schema: &configschema.Block{
+				BlockTypes: map[string]*configschema.NestedBlock{
+					"bar": {
+						Block: configschema.Block{
+							Attributes: map[string]*configschema.Attribute{
+								"val": {
+									Type:     cty.String,
+									Optional: true,
+								},
+							},
+						},
+						Nesting: configschema.NestingSet,
+					},
+				},
+			},
+			Expected: &ResourceConfig{
+				ComputedKeys: []string{"bar.0.val"},
+				Raw: map[string]interface{}{
+					"bar": []interface{}{map[string]interface{}{
+						"val": "74D93920-ED26-11E3-AC10-0800200C9A66",
+					}},
+				},
+				Config: map[string]interface{}{
+					"bar": []interface{}{map[string]interface{}{
+						"val": "74D93920-ED26-11E3-AC10-0800200C9A66",
+					}},
+				},
+			},
+		},
+		{
+			Name: "unknown in attribute sets",
+			Val: cty.ObjectVal(map[string]cty.Value{
+				"bar": cty.SetVal([]cty.Value{
+					cty.ObjectVal(map[string]cty.Value{
+						"val": cty.UnknownVal(cty.String),
+					}),
+				}),
+				"baz": cty.SetVal([]cty.Value{
+					cty.ObjectVal(map[string]cty.Value{
+						"obj": cty.UnknownVal(cty.Object(map[string]cty.Type{
+							"attr": cty.List(cty.String),
+						})),
+					}),
+					cty.ObjectVal(map[string]cty.Value{
+						"obj": cty.ObjectVal(map[string]cty.Value{
+							"attr": cty.UnknownVal(cty.List(cty.String)),
+						}),
+					}),
+				}),
+			}),
+			Schema: &configschema.Block{
+				Attributes: map[string]*configschema.Attribute{
+					"bar": &configschema.Attribute{
+						Type: cty.Set(cty.Object(map[string]cty.Type{
+							"val": cty.String,
+						})),
+					},
+					"baz": &configschema.Attribute{
+						Type: cty.Set(cty.Object(map[string]cty.Type{
+							"obj": cty.Object(map[string]cty.Type{
+								"attr": cty.List(cty.String),
+							}),
+						})),
+					},
+				},
+			},
+			Expected: &ResourceConfig{
+				ComputedKeys: []string{"bar.0.val", "baz.0.obj.attr", "baz.1.obj"},
+				Raw: map[string]interface{}{
+					"bar": []interface{}{map[string]interface{}{
+						"val": "74D93920-ED26-11E3-AC10-0800200C9A66",
+					}},
+					"baz": []interface{}{
+						map[string]interface{}{
+							"obj": map[string]interface{}{
+								"attr": "74D93920-ED26-11E3-AC10-0800200C9A66",
+							},
+						},
+						map[string]interface{}{
+							"obj": "74D93920-ED26-11E3-AC10-0800200C9A66",
+						},
+					},
+				},
+				Config: map[string]interface{}{
+					"bar": []interface{}{map[string]interface{}{
+						"val": "74D93920-ED26-11E3-AC10-0800200C9A66",
+					}},
+					"baz": []interface{}{
+						map[string]interface{}{
+							"obj": map[string]interface{}{
+								"attr": "74D93920-ED26-11E3-AC10-0800200C9A66",
+							},
+						},
+						map[string]interface{}{
+							"obj": "74D93920-ED26-11E3-AC10-0800200C9A66",
+						},
+					},
+				},
+			},
+		},
 		{
 			Name: "null blocks",
 			Val: cty.ObjectVal(map[string]cty.Value{

terraform/test-fixtures/refresh-data-count/refresh-data-count.tf

@@ -0,0 +1,7 @@
+resource "test" "foo" {
+  things = ["foo"]
+}
+
+data "test" "foo" {
+  count = length(test.foo.things)
+}

terraform/valuesourcetype_string.go

@@ -4,6 +4,21 @@ package terraform
 
 import "strconv"
 
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[ValueFromUnknown-0]
+	_ = x[ValueFromConfig-67]
+	_ = x[ValueFromAutoFile-70]
+	_ = x[ValueFromNamedFile-78]
+	_ = x[ValueFromCLIArg-65]
+	_ = x[ValueFromEnvVar-69]
+	_ = x[ValueFromInput-73]
+	_ = x[ValueFromPlan-80]
+	_ = x[ValueFromCaller-83]
+}
+
 const (
 	_ValueSourceType_name_0 = "ValueFromUnknown"
 	_ValueSourceType_name_1 = "ValueFromCLIArg"