From ccc1b6990f383194cd02bd0c1cda8d4cb5a99f12 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 2 May 2018 17:47:58 -0700 Subject: [PATCH] config/hcl2shim: shims from flatmap to cty and vice-versa These particular shims will have a pretty limited lifetime in mainline Terraform code (primarily to stub out the new expression evaluator against the old state structs until the new format is implemented) but will live on for some time in state migration and provider plugin compatibility shims. --- config/hcl2shim/flatmap.go | 317 ++++++++++++++++++++++ config/hcl2shim/flatmap_test.go | 451 ++++++++++++++++++++++++++++++++ 2 files changed, 768 insertions(+) create mode 100644 config/hcl2shim/flatmap.go create mode 100644 config/hcl2shim/flatmap_test.go diff --git a/config/hcl2shim/flatmap.go b/config/hcl2shim/flatmap.go new file mode 100644 index 000000000..f013bf887 --- /dev/null +++ b/config/hcl2shim/flatmap.go @@ -0,0 +1,317 @@ +package hcl2shim + +import ( + "fmt" + "strconv" + "strings" + + "github.com/zclconf/go-cty/cty/convert" + + "github.com/zclconf/go-cty/cty" +) + +// FlatmapValueFromHCL2 converts a value from HCL2 (really, from the cty dynamic +// types library that HCL2 uses) to a map compatible with what would be +// produced by the "flatmap" package. +// +// The type of the given value informs the structure of the resulting map. +// The value must be of an object type or this function will panic. +// +// Flatmap values can only represent maps when they are of primitive types, +// so the given value must not have any maps of complex types or the result +// is undefined. +func FlatmapValueFromHCL2(v cty.Value) map[string]string { + if !v.Type().IsObjectType() { + panic(fmt.Sprintf("HCL2ValueFromFlatmap called on %#v", v.Type())) + } + + m := make(map[string]string) + flatmapValueFromHCL2Map(m, "", v) + return m +} + +func flatmapValueFromHCL2Value(m map[string]string, key string, val cty.Value) { + ty := val.Type() + switch { + case ty.IsPrimitiveType(): + flatmapValueFromHCL2Primitive(m, key, val) + case ty.IsObjectType() || ty.IsMapType(): + flatmapValueFromHCL2Map(m, key+".", val) + case ty.IsTupleType() || ty.IsListType() || ty.IsSetType(): + flatmapValueFromHCL2Seq(m, key+".", val) + default: + panic(fmt.Sprintf("cannot encode %s to flatmap", ty.FriendlyName())) + } +} + +func flatmapValueFromHCL2Primitive(m map[string]string, key string, val cty.Value) { + if !val.IsKnown() { + m[key] = UnknownVariableValue + return + } + if val.IsNull() { + // Omit entirely + return + } + + var err error + val, err = convert.Convert(val, cty.String) + if err != nil { + // Should not be possible, since all primitive types can convert to string. + panic(fmt.Sprintf("invalid primitive encoding to flatmap: %s", err)) + } + m[key] = val.AsString() +} + +func flatmapValueFromHCL2Map(m map[string]string, prefix string, val cty.Value) { + len := 0 + for it := val.ElementIterator(); it.Next(); { + ak, av := it.Element() + name := ak.AsString() + flatmapValueFromHCL2Value(m, prefix+name, av) + len++ + } + if !val.Type().IsObjectType() { // objects don't have an explicit count included, since their attribute count is fixed + m[prefix+"%"] = strconv.Itoa(len) + } +} + +func flatmapValueFromHCL2Seq(m map[string]string, prefix string, val cty.Value) { + // For sets this won't actually generate exactly what helper/schema would've + // generated, because we don't have access to the set key function it + // would've used. However, in practice it doesn't actually matter what the + // keys are as long as they are unique, so we'll just generate sequential + // indexes for them as if it were a list. + // + // An important implication of this, however, is that the set ordering will + // not be consistent across mutations and so different keys may be assigned + // to the same value when round-tripping. Since this shim is intended to + // be short-lived and not used for round-tripping, we accept this. + i := 0 + for it := val.ElementIterator(); it.Next(); { + _, av := it.Element() + key := prefix + strconv.Itoa(i) + flatmapValueFromHCL2Value(m, key, av) + i++ + } + m[prefix+"#"] = strconv.Itoa(i) +} + +// HCL2ValueFromFlatmap converts a map compatible with what would be produced +// by the "flatmap" package to a HCL2 (really, the cty dynamic types library +// that HCL2 uses) object type. +// +// The intended result type must be provided in order to guide how the +// map contents are decoded. This must be an object type or this function +// will panic. +// +// Flatmap values can only represent maps when they are of primitive types, +// so the given type must not have any maps of complex types or the result +// is undefined. +// +// The result may contain null values if the given map does not contain keys +// for all of the different key paths implied by the given type. +func HCL2ValueFromFlatmap(m map[string]string, ty cty.Type) (cty.Value, error) { + if !ty.IsObjectType() { + panic(fmt.Sprintf("HCL2ValueFromFlatmap called on %#v", ty)) + } + + return hcl2ValueFromFlatmapObject(m, "", ty.AttributeTypes()) +} + +func hcl2ValueFromFlatmapValue(m map[string]string, key string, ty cty.Type) (cty.Value, error) { + var val cty.Value + var err error + switch { + case ty.IsPrimitiveType(): + val, err = hcl2ValueFromFlatmapPrimitive(m, key, ty) + case ty.IsObjectType(): + val, err = hcl2ValueFromFlatmapObject(m, key+".", ty.AttributeTypes()) + case ty.IsTupleType(): + val, err = hcl2ValueFromFlatmapTuple(m, key+".", ty.TupleElementTypes()) + case ty.IsMapType(): + val, err = hcl2ValueFromFlatmapMap(m, key+".", ty) + case ty.IsListType(): + val, err = hcl2ValueFromFlatmapList(m, key+".", ty) + case ty.IsSetType(): + val, err = hcl2ValueFromFlatmapSet(m, key+".", ty) + default: + err = fmt.Errorf("cannot decode %s from flatmap", ty.FriendlyName()) + } + + if err != nil { + return cty.DynamicVal, err + } + return val, nil +} + +func hcl2ValueFromFlatmapPrimitive(m map[string]string, key string, ty cty.Type) (cty.Value, error) { + rawVal, exists := m[key] + if !exists { + return cty.NullVal(ty), nil + } + + var err error + val := cty.StringVal(rawVal) + val, err = convert.Convert(val, ty) + if err != nil { + // This should never happen for _valid_ input, but flatmap data might + // be tampered with by the user and become invalid. + return cty.DynamicVal, fmt.Errorf("invalid value for %q in state: %s", key, err) + } + + return val, nil +} + +func hcl2ValueFromFlatmapObject(m map[string]string, prefix string, atys map[string]cty.Type) (cty.Value, error) { + vals := make(map[string]cty.Value) + for name, aty := range atys { + val, err := hcl2ValueFromFlatmapValue(m, prefix+name, aty) + if err != nil { + return cty.DynamicVal, err + } + vals[name] = val + } + return cty.ObjectVal(vals), nil +} + +func hcl2ValueFromFlatmapTuple(m map[string]string, prefix string, etys []cty.Type) (cty.Value, error) { + var vals []cty.Value + + countStr, exists := m[prefix+"#"] + if !exists { + return cty.NullVal(cty.Tuple(etys)), nil + } + count, err := strconv.Atoi(countStr) + if err != nil { + return cty.DynamicVal, fmt.Errorf("invalid count value for %q in state: %s", prefix, err) + } + if count != len(etys) { + return cty.DynamicVal, fmt.Errorf("wrong number of values for %q in state: got %d, but need %d", prefix, count, len(etys)) + } + + vals = make([]cty.Value, len(etys)) + for i, ety := range etys { + key := prefix + strconv.Itoa(i) + val, err := hcl2ValueFromFlatmapValue(m, key, ety) + if err != nil { + return cty.DynamicVal, err + } + vals[i] = val + } + return cty.TupleVal(vals), nil +} + +func hcl2ValueFromFlatmapMap(m map[string]string, prefix string, ty cty.Type) (cty.Value, error) { + vals := make(map[string]cty.Value) + ety := ty.ElementType() + + // We actually don't really care about the "count" of a map for our + // purposes here, but we do need to check if it _exists_ in order to + // recognize the difference between null (not set at all) and empty. + if _, exists := m[prefix+"%"]; !exists { + return cty.NullVal(ty), nil + } + + for fullKey := range m { + if !strings.HasPrefix(fullKey, prefix) { + continue + } + + // The flatmap format doesn't allow us to distinguish between keys + // that contain periods and nested objects, so by convention a + // map is only ever of primitive type in flatmap, and we just assume + // that the remainder of the raw key (dots and all) is the key we + // want in the result value. + key := fullKey[len(prefix):] + if key == "%" { + // Ignore the "count" key + continue + } + + val, err := hcl2ValueFromFlatmapValue(m, fullKey, ety) + if err != nil { + return cty.DynamicVal, err + } + vals[key] = val + } + + if len(vals) == 0 { + return cty.MapValEmpty(ety), nil + } + return cty.MapVal(vals), nil +} + +func hcl2ValueFromFlatmapList(m map[string]string, prefix string, ty cty.Type) (cty.Value, error) { + var vals []cty.Value + + countStr, exists := m[prefix+"#"] + if !exists { + return cty.NullVal(ty), nil + } + count, err := strconv.Atoi(countStr) + if err != nil { + return cty.DynamicVal, fmt.Errorf("invalid count value for %q in state: %s", prefix, err) + } + + ety := ty.ElementType() + if count == 0 { + return cty.ListValEmpty(ety), nil + } + + vals = make([]cty.Value, count) + for i := 0; i < count; i++ { + key := prefix + strconv.Itoa(i) + val, err := hcl2ValueFromFlatmapValue(m, key, ety) + if err != nil { + return cty.DynamicVal, err + } + vals[i] = val + } + + return cty.ListVal(vals), nil +} + +func hcl2ValueFromFlatmapSet(m map[string]string, prefix string, ty cty.Type) (cty.Value, error) { + var vals []cty.Value + ety := ty.ElementType() + + // We actually don't really care about the "count" of a set for our + // purposes here, but we do need to check if it _exists_ in order to + // recognize the difference between null (not set at all) and empty. + if _, exists := m[prefix+"#"]; !exists { + return cty.NullVal(ty), nil + } + + for fullKey := range m { + if !strings.HasPrefix(fullKey, prefix) { + continue + } + subKey := fullKey[len(prefix):] + if subKey == "#" { + // Ignore the "count" key + continue + } + key := fullKey + if dot := strings.IndexByte(subKey, '.'); dot != -1 { + key = fullKey[:dot+len(prefix)] + } + + // The flatmap format doesn't allow us to distinguish between keys + // that contain periods and nested objects, so by convention a + // map is only ever of primitive type in flatmap, and we just assume + // that the remainder of the raw key (dots and all) is the key we + // want in the result value. + + val, err := hcl2ValueFromFlatmapValue(m, key, ety) + if err != nil { + return cty.DynamicVal, err + } + vals = append(vals, val) + } + + if len(vals) == 0 { + return cty.SetValEmpty(ety), nil + } + return cty.SetVal(vals), nil +} diff --git a/config/hcl2shim/flatmap_test.go b/config/hcl2shim/flatmap_test.go new file mode 100644 index 000000000..ee0a5d6f8 --- /dev/null +++ b/config/hcl2shim/flatmap_test.go @@ -0,0 +1,451 @@ +package hcl2shim + +import ( + "fmt" + "testing" + + "github.com/go-test/deep" + + "github.com/zclconf/go-cty/cty" +) + +func TestFlatmapValueFromHCL2(t *testing.T) { + tests := []struct { + Value cty.Value + Want map[string]string + }{ + { + cty.EmptyObjectVal, + map[string]string{}, + }, + { + cty.ObjectVal(map[string]cty.Value{ + "foo": cty.StringVal("hello"), + }), + map[string]string{ + "foo": "hello", + }, + }, + { + cty.ObjectVal(map[string]cty.Value{ + "foo": cty.NumberIntVal(12), + }), + map[string]string{ + "foo": "12", + }, + }, + { + cty.ObjectVal(map[string]cty.Value{ + "foo": cty.True, + "bar": cty.False, + }), + map[string]string{ + "foo": "true", + "bar": "false", + }, + }, + { + cty.ObjectVal(map[string]cty.Value{ + "foo": cty.StringVal("hello"), + "bar": cty.StringVal("world"), + "baz": cty.StringVal("whelp"), + }), + map[string]string{ + "foo": "hello", + "bar": "world", + "baz": "whelp", + }, + }, + { + cty.ObjectVal(map[string]cty.Value{ + "foo": cty.ListValEmpty(cty.String), + }), + map[string]string{ + "foo.#": "0", + }, + }, + { + cty.ObjectVal(map[string]cty.Value{ + "foo": cty.ListVal([]cty.Value{ + cty.StringVal("hello"), + }), + }), + map[string]string{ + "foo.#": "1", + "foo.0": "hello", + }, + }, + { + cty.ObjectVal(map[string]cty.Value{ + "foo": cty.ListVal([]cty.Value{ + cty.StringVal("hello"), + cty.StringVal("world"), + }), + }), + map[string]string{ + "foo.#": "2", + "foo.0": "hello", + "foo.1": "world", + }, + }, + { + cty.ObjectVal(map[string]cty.Value{ + "foo": cty.MapVal(map[string]cty.Value{ + "hello": cty.NumberIntVal(12), + "hello.world": cty.NumberIntVal(10), + }), + }), + map[string]string{ + "foo.%": "2", + "foo.hello": "12", + "foo.hello.world": "10", + }, + }, + { + cty.ObjectVal(map[string]cty.Value{ + "foo": cty.MapVal(map[string]cty.Value{ + "hello": cty.NumberIntVal(12), + "hello.world": cty.NumberIntVal(10), + }), + }), + map[string]string{ + "foo.%": "2", + "foo.hello": "12", + "foo.hello.world": "10", + }, + }, + { + cty.ObjectVal(map[string]cty.Value{ + "foo": cty.SetVal([]cty.Value{ + cty.StringVal("hello"), + cty.StringVal("world"), + }), + }), + map[string]string{ + "foo.#": "2", + "foo.0": "hello", + "foo.1": "world", + }, + }, + { + cty.ObjectVal(map[string]cty.Value{ + "foo": cty.ListVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + "bar": cty.StringVal("hello"), + "baz": cty.StringVal("world"), + }), + cty.ObjectVal(map[string]cty.Value{ + "bar": cty.StringVal("bloo"), + "baz": cty.StringVal("blaa"), + }), + }), + }), + map[string]string{ + "foo.#": "2", + "foo.0.bar": "hello", + "foo.0.baz": "world", + "foo.1.bar": "bloo", + "foo.1.baz": "blaa", + }, + }, + { + cty.ObjectVal(map[string]cty.Value{ + "foo": cty.ListVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + "bar": cty.StringVal("hello"), + "baz": cty.ListVal([]cty.Value{ + cty.True, + cty.True, + }), + }), + cty.ObjectVal(map[string]cty.Value{ + "bar": cty.StringVal("bloo"), + "baz": cty.ListVal([]cty.Value{ + cty.False, + cty.True, + }), + }), + }), + }), + map[string]string{ + "foo.#": "2", + "foo.0.bar": "hello", + "foo.0.baz.#": "2", + "foo.0.baz.0": "true", + "foo.0.baz.1": "true", + "foo.1.bar": "bloo", + "foo.1.baz.#": "2", + "foo.1.baz.0": "false", + "foo.1.baz.1": "true", + }, + }, + } + + for _, test := range tests { + t.Run(test.Value.GoString(), func(t *testing.T) { + got := FlatmapValueFromHCL2(test.Value) + + for _, problem := range deep.Equal(got, test.Want) { + t.Error(problem) + } + }) + } +} + +func TestHCL2ValueFromFlatmap(t *testing.T) { + tests := []struct { + Flatmap map[string]string + Type cty.Type + Want cty.Value + WantErr string + }{ + { + Flatmap: map[string]string{}, + Type: cty.EmptyObject, + Want: cty.EmptyObjectVal, + }, + { + Flatmap: map[string]string{ + "ignored": "foo", + }, + Type: cty.EmptyObject, + Want: cty.EmptyObjectVal, + }, + { + Flatmap: map[string]string{ + "foo": "blah", + "bar": "true", + "baz": "12.5", + }, + Type: cty.Object(map[string]cty.Type{ + "foo": cty.String, + "bar": cty.Bool, + "baz": cty.Number, + }), + Want: cty.ObjectVal(map[string]cty.Value{ + "foo": cty.StringVal("blah"), + "bar": cty.True, + "baz": cty.NumberFloatVal(12.5), + }), + }, + { + Flatmap: map[string]string{ + "foo.#": "0", + }, + Type: cty.Object(map[string]cty.Type{ + "foo": cty.List(cty.String), + }), + Want: cty.ObjectVal(map[string]cty.Value{ + "foo": cty.ListValEmpty(cty.String), + }), + }, + { + Flatmap: map[string]string{ + "foo.#": "1", + "foo.0": "hello", + }, + Type: cty.Object(map[string]cty.Type{ + "foo": cty.List(cty.String), + }), + Want: cty.ObjectVal(map[string]cty.Value{ + "foo": cty.ListVal([]cty.Value{ + cty.StringVal("hello"), + }), + }), + }, + { + Flatmap: map[string]string{ + "foo.#": "2", + "foo.0": "true", + "foo.1": "false", + "foo.2": "ignored", // (because the count is 2, so this is out of range) + }, + Type: cty.Object(map[string]cty.Type{ + "foo": cty.List(cty.Bool), + }), + Want: cty.ObjectVal(map[string]cty.Value{ + "foo": cty.ListVal([]cty.Value{ + cty.True, + cty.False, + }), + }), + }, + { + Flatmap: map[string]string{ + "foo.#": "2", + "foo.0": "hello", + }, + Type: cty.Object(map[string]cty.Type{ + "foo": cty.Tuple([]cty.Type{ + cty.String, + cty.Bool, + }), + }), + Want: cty.ObjectVal(map[string]cty.Value{ + "foo": cty.TupleVal([]cty.Value{ + cty.StringVal("hello"), + cty.NullVal(cty.Bool), + }), + }), + }, + { + Flatmap: map[string]string{ + "foo.#": "0", + }, + Type: cty.Object(map[string]cty.Type{ + "foo": cty.Set(cty.String), + }), + Want: cty.ObjectVal(map[string]cty.Value{ + "foo": cty.SetValEmpty(cty.String), + }), + }, + { + Flatmap: map[string]string{ + "foo.#": "1", + "foo.24534534": "hello", + }, + Type: cty.Object(map[string]cty.Type{ + "foo": cty.Set(cty.String), + }), + Want: cty.ObjectVal(map[string]cty.Value{ + "foo": cty.SetVal([]cty.Value{ + cty.StringVal("hello"), + }), + }), + }, + { + Flatmap: map[string]string{ + "foo.#": "1", + "foo.24534534": "true", + "foo.95645644": "true", + "foo.34533452": "false", + }, + Type: cty.Object(map[string]cty.Type{ + "foo": cty.Set(cty.Bool), + }), + Want: cty.ObjectVal(map[string]cty.Value{ + "foo": cty.SetVal([]cty.Value{ + cty.True, + cty.False, + }), + }), + }, + { + Flatmap: map[string]string{ + "foo.%": "0", + }, + Type: cty.Object(map[string]cty.Type{ + "foo": cty.Map(cty.String), + }), + Want: cty.ObjectVal(map[string]cty.Value{ + "foo": cty.MapValEmpty(cty.String), + }), + }, + { + Flatmap: map[string]string{ + "foo.%": "2", + "foo.baz": "true", + "foo.bar.baz": "false", + }, + Type: cty.Object(map[string]cty.Type{ + "foo": cty.Map(cty.Bool), + }), + Want: cty.ObjectVal(map[string]cty.Value{ + "foo": cty.MapVal(map[string]cty.Value{ + "baz": cty.True, + "bar.baz": cty.False, + }), + }), + }, + { + Flatmap: map[string]string{ + "foo.#": "2", + "foo.0.bar": "hello", + "foo.0.baz": "1", + "foo.1.bar": "world", + "foo.1.baz": "false", + }, + Type: cty.Object(map[string]cty.Type{ + "foo": cty.List(cty.Object(map[string]cty.Type{ + "bar": cty.String, + "baz": cty.Bool, + })), + }), + Want: cty.ObjectVal(map[string]cty.Value{ + "foo": cty.ListVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + "bar": cty.StringVal("hello"), + "baz": cty.True, + }), + cty.ObjectVal(map[string]cty.Value{ + "bar": cty.StringVal("world"), + "baz": cty.False, + }), + }), + }), + }, + { + Flatmap: map[string]string{ + "foo.#": "2", + "foo.34534534.bar": "hello", + "foo.34534534.baz": "1", + "foo.93453345.bar": "world", + "foo.93453345.baz": "false", + }, + Type: cty.Object(map[string]cty.Type{ + "foo": cty.Set(cty.Object(map[string]cty.Type{ + "bar": cty.String, + "baz": cty.Bool, + })), + }), + Want: cty.ObjectVal(map[string]cty.Value{ + "foo": cty.SetVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + "bar": cty.StringVal("hello"), + "baz": cty.True, + }), + cty.ObjectVal(map[string]cty.Value{ + "bar": cty.StringVal("world"), + "baz": cty.False, + }), + }), + }), + }, + { + Flatmap: map[string]string{ + "foo.#": "not-valid", + }, + Type: cty.Object(map[string]cty.Type{ + "foo": cty.List(cty.String), + }), + WantErr: `invalid count value for "foo." in state: strconv.Atoi: parsing "not-valid": invalid syntax`, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%#v as %#v", test.Flatmap, test.Type), func(t *testing.T) { + got, err := HCL2ValueFromFlatmap(test.Flatmap, test.Type) + + if test.WantErr != "" { + if err == nil { + t.Fatalf("succeeded; want error: %s", test.WantErr) + } + if got, want := err.Error(), test.WantErr; got != want { + t.Fatalf("wrong error\ngot: %s\nwant: %s", got, want) + } + if got == cty.NilVal { + t.Fatalf("result is cty.NilVal; want valid placeholder value") + } + return + } else { + if err != nil { + t.Fatalf("unexpected error: %s", err.Error()) + } + } + + if !got.RawEquals(test.Want) { + t.Errorf("wrong result\ngot: %#v\nwant: %#v", got, test.Want) + } + }) + } +}