core: Allow dynamic attributes in helper/schema

The helper/schema framework for building providers previously validated
in all cases that each field being set in state was in the schema.
However, in order to support remote state in a usable fashion, the need
has arisen for the top level attributes of the resource to be created
dynamically. In order to still be able to use helper/schema, this commit
adds the capability to assign additional fields.

Though I do not forsee this being used by providers other than remote
state (and that eventually may move into Terraform Core rather than
being a provider), the usage and semantics are:

To opt into dynamic attributes, add a schema attribute named
"__has_dynamic_attributes", and make it an optional string with no
default value, in order that it does not appear in diffs:

        "__has_dynamic_attributes": {
            Type: schema.TypeString
            Optional: true
        }

In the read callback, use the d.UnsafeSetFieldRaw(key, value) function
to set the dynamic attributes.

Note that other fields in the schema _are_ copied into state, and that
the names of the schema fields cannot currently be used as dynamic
attribute names, as we check to ensure a value is not already set for a
given key.

helper/schema/field_writer_map.go

@@ -29,6 +29,16 @@ func (w *MapFieldWriter) Map() map[string]string {
 	return w.result
 }
 
+func (w *MapFieldWriter) unsafeWriteField(addr string, value string) {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+	if w.result == nil {
+		w.result = make(map[string]string)
+	}
+
+	w.result[addr] = value
+}
+
 func (w *MapFieldWriter) WriteField(addr []string, value interface{}) error {
 	w.lock.Lock()
 	defer w.lock.Unlock()

helper/schema/resource_data.go

@@ -1,6 +1,7 @@
 package schema
 
 import (
+	"log"
 	"reflect"
 	"strings"
 	"sync"
@@ -44,7 +45,14 @@ type getResult struct {
 	Schema         *Schema
 }
 
-var getResultEmpty getResult
+// UnsafeSetFieldRaw allows setting arbitrary values in state to arbitrary
+// values, bypassing schema. This MUST NOT be used in normal circumstances -
+// it exists only to support the remote_state data source.
+func (d *ResourceData) UnsafeSetFieldRaw(key string, value string) {
+	d.once.Do(d.init)
+
+	d.setWriter.unsafeWriteField(key, value)
+}
 
 // Get returns the data for the given key, or nil if the key doesn't exist
 // in the schema.
@@ -242,6 +250,17 @@ func (d *ResourceData) State() *terraform.InstanceState {
 		return nil
 	}
 
+	// Look for a magic key in the schema that determines we skip the
+	// integrity check of fields existing in the schema, allowing dynamic
+	// keys to be created.
+	hasDynamicAttributes := false
+	for k, _ := range d.schema {
+		if k == "__has_dynamic_attributes" {
+			hasDynamicAttributes = true
+			log.Printf("[INFO] Resource %s has dynamic attributes", result.ID)
+		}
+	}
+
 	// In order to build the final state attributes, we read the full
 	// attribute set as a map[string]interface{}, write it to a MapFieldWriter,
 	// and then use that map.
@@ -263,12 +282,27 @@ func (d *ResourceData) State() *terraform.InstanceState {
 			}
 		}
 	}
+
 	mapW := &MapFieldWriter{Schema: d.schema}
 	if err := mapW.WriteField(nil, rawMap); err != nil {
 		return nil
 	}
 
 	result.Attributes = mapW.Map()
+
+	if hasDynamicAttributes {
+		// If we have dynamic attributes, just copy the attributes map
+		// one for one into the result attributes.
+		for k, v := range d.setWriter.Map() {
+			// Don't clobber schema values. This limits usage of dynamic
+			// attributes to names which _do not_ conflict with schema
+			// keys!
+			if _, ok := result.Attributes[k]; !ok {
+				result.Attributes[k] = v
+			}
+		}
+	}
+
 	if d.newState != nil {
 		result.Ephemeral = d.newState.Ephemeral
 	}

helper/schema/resource_data_test.go

@@ -1755,7 +1755,87 @@ func TestResourceDataSet(t *testing.T) {
 	}
 }
 
-func TestResourceDataState(t *testing.T) {
+func TestResourceDataState_dynamicAttributes(t *testing.T) {
+	cases := []struct {
+		Schema    map[string]*Schema
+		State     *terraform.InstanceState
+		Diff      *terraform.InstanceDiff
+		Set       map[string]interface{}
+		UnsafeSet map[string]string
+		Result    *terraform.InstanceState
+	}{
+		{
+			Schema: map[string]*Schema{
+				"__has_dynamic_attributes": {
+					Type:     TypeString,
+					Optional: true,
+				},
+
+				"schema_field": {
+					Type:     TypeString,
+					Required: true,
+				},
+			},
+
+			State: nil,
+
+			Diff: nil,
+
+			Set: map[string]interface{}{
+				"schema_field": "present",
+			},
+
+			UnsafeSet: map[string]string{
+				"test1": "value",
+				"test2": "value",
+			},
+
+			Result: &terraform.InstanceState{
+				Attributes: map[string]string{
+					"schema_field": "present",
+					"test1":        "value",
+					"test2":        "value",
+				},
+			},
+		},
+	}
+
+	for i, tc := range cases {
+		d, err := schemaMap(tc.Schema).Data(tc.State, tc.Diff)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+
+		for k, v := range tc.Set {
+			d.Set(k, v)
+		}
+
+		for k, v := range tc.UnsafeSet {
+			d.UnsafeSetFieldRaw(k, v)
+		}
+
+		// Set an ID so that the state returned is not nil
+		idSet := false
+		if d.Id() == "" {
+			idSet = true
+			d.SetId("foo")
+		}
+
+		actual := d.State()
+
+		// If we set an ID, then undo what we did so the comparison works
+		if actual != nil && idSet {
+			actual.ID = ""
+			delete(actual.Attributes, "id")
+		}
+
+		if !reflect.DeepEqual(actual, tc.Result) {
+			t.Fatalf("Bad: %d\n\n%#v\n\nExpected:\n\n%#v", i, actual, tc.Result)
+		}
+	}
+}
+
+func TestResourceDataState_schema(t *testing.T) {
 	cases := []struct {
 		Schema  map[string]*Schema
 		State   *terraform.InstanceState