diff --git a/.gitignore b/.gitignore index 03a5621e4..e7669ef01 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ example.tf terraform.tfplan terraform.tfstate bin/ +config/y.go +config/y.output vendor/ diff --git a/Makefile b/Makefile index 40de98ffe..ad2da4cf6 100644 --- a/Makefile +++ b/Makefile @@ -16,27 +16,31 @@ export CGO_CFLAGS CGO_LDFLAGS PATH default: test -dev: libucl +dev: config/y.go libucl @sh -c "$(CURDIR)/scripts/build.sh" libucl: vendor/libucl/$(LIBUCL_NAME) -test: libucl +test: config/y.go libucl TF_ACC= go test $(TEST) $(TESTARGS) -timeout=10s -testacc: libucl +testacc: config/y.go libucl @if [ "$(TEST)" = "./..." ]; then \ echo "ERROR: Set TEST to a specific package"; \ exit 1; \ fi TF_ACC=1 go test $(TEST) -v $(TESTARGS) -testrace: libucl +testrace: config/y.go libucl TF_ACC= go test -race $(TEST) $(TESTARGS) updatedeps: go get -u -v ./... +config/y.go: config/expr.y + cd config/ && \ + go tool yacc -p "expr" expr.y + vendor/libucl/libucl.a: vendor/libucl cd vendor/libucl && \ cmake cmake/ && \ diff --git a/config/expr.y b/config/expr.y new file mode 100644 index 000000000..27903f714 --- /dev/null +++ b/config/expr.y @@ -0,0 +1,91 @@ +// This is the yacc input for creating the parser for interpolation +// expressions in Go. + +// To build it: +// +// go tool yacc -p "expr" expr.y (produces y.go) +// + +%{ +package config + +import ( + "fmt" +) + +%} + +%union { + expr Interpolation + str string + variable InterpolatedVariable + args []Interpolation +} + +%type args +%type expr +%type string +%type variable + +%token STRING IDENTIFIER +%token COMMA LEFTPAREN RIGHTPAREN + +%% + +top: + expr + { + exprResult = $1 + } + +expr: + string + { + $$ = &LiteralInterpolation{Literal: $1} + } +| variable + { + $$ = &VariableInterpolation{Variable: $1} + } +| IDENTIFIER LEFTPAREN args RIGHTPAREN + { + f, ok := Funcs[$1] + if !ok { + exprErrors = append(exprErrors, fmt.Errorf( + "Unknown function: %s", $1)) + } + + $$ = &FunctionInterpolation{Func: f, Args: $3} + } + +args: + { + $$ = nil + } +| expr COMMA expr + { + $$ = append($$, $1, $3) + } +| expr + { + $$ = append($$, $1) + } + +string: + STRING + { + $$ = $1 + } + +variable: + IDENTIFIER + { + var err error + $$, err = NewInterpolatedVariable($1) + if err != nil { + exprErrors = append(exprErrors, fmt.Errorf( + "Error parsing variable '%s': %s", $1, err)) + } + } + +%% diff --git a/config/expr_lex.go b/config/expr_lex.go new file mode 100644 index 000000000..b54a14af8 --- /dev/null +++ b/config/expr_lex.go @@ -0,0 +1,131 @@ +package config + +import ( + "bytes" + "log" + "unicode" + "unicode/utf8" +) + +// The parser expects the lexer to return 0 on EOF. +const lexEOF = 0 + +// The parser uses the type Lex as a lexer. It must provide +// the methods Lex(*SymType) int and Error(string). +type exprLex struct { + input string + pos int + width int +} + +// The parser calls this method to get each new token. +func (x *exprLex) Lex(yylval *exprSymType) int { + for { + c := x.next() + if c == lexEOF { + return lexEOF + } + + // Ignore all whitespace + if unicode.IsSpace(c) { + continue + } + + switch c { + case '"': + return x.lexString(yylval) + case ',': + return COMMA + case '(': + return LEFTPAREN + case ')': + return RIGHTPAREN + default: + x.backup() + return x.lexId(yylval) + } + } +} + +func (x *exprLex) lexId(yylval *exprSymType) int { + var b bytes.Buffer + for { + c := x.next() + if c == lexEOF { + break + } + + // If this isn't a character we want in an ID, return out. + // One day we should make this a regexp. + if c != '_' && + c != '-' && + c != '.' && + c != '*' && + !unicode.IsLetter(c) && + !unicode.IsNumber(c) { + x.backup() + break + } + + if _, err := b.WriteRune(c); err != nil { + log.Printf("ERR: %s", err) + return lexEOF + } + } + + yylval.str = b.String() + return IDENTIFIER +} + +func (x *exprLex) lexString(yylval *exprSymType) int { + var b bytes.Buffer + for { + c := x.next() + if c == lexEOF { + break + } + + // String end + if c == '"' { + break + } + + if _, err := b.WriteRune(c); err != nil { + log.Printf("ERR: %s", err) + return lexEOF + } + } + + yylval.str = b.String() + return STRING +} + +// Return the next rune for the lexer. +func (x *exprLex) next() rune { + if int(x.pos) >= len(x.input) { + x.width = 0 + return lexEOF + } + + r, w := utf8.DecodeRuneInString(x.input[x.pos:]) + x.width = w + x.pos += x.width + return r +} + +// peek returns but does not consume the next rune in the input +func (x *exprLex) peek() rune { + r := x.next() + x.backup() + return r +} + +// backup steps back one rune. Can only be called once per next. +func (x *exprLex) backup() { + x.pos -= x.width +} + +// The parser calls this method on a parse error. +func (x *exprLex) Error(s string) { + log.Printf("parse error: %s", s) +} diff --git a/config/expr_parse.go b/config/expr_parse.go new file mode 100644 index 000000000..0ffb0ba14 --- /dev/null +++ b/config/expr_parse.go @@ -0,0 +1,34 @@ +package config + +import ( + "sync" + + "github.com/hashicorp/terraform/helper/multierror" +) + +// exprErrors are the errors built up from parsing. These should not +// be accessed directly. +var exprErrors []error +var exprLock sync.Mutex +var exprResult Interpolation + +// ExprParse parses the given expression and returns an executable +// Interpolation. +func ExprParse(v string) (Interpolation, error) { + exprLock.Lock() + defer exprLock.Unlock() + exprErrors = nil + exprResult = nil + + // Parse + exprParse(&exprLex{input: v}) + + // Build up the errors + var err error + if len(exprErrors) > 0 { + err = &multierror.Error{Errors: exprErrors} + exprResult = nil + } + + return exprResult, err +} diff --git a/config/expr_parse_test.go b/config/expr_parse_test.go new file mode 100644 index 000000000..da6ab0e43 --- /dev/null +++ b/config/expr_parse_test.go @@ -0,0 +1,123 @@ +package config + +import ( + "reflect" + "testing" +) + +func TestExprParse(t *testing.T) { + cases := []struct { + Input string + Result Interpolation + Error bool + }{ + { + "foo", + nil, + true, + }, + + { + `"foo"`, + &LiteralInterpolation{Literal: "foo"}, + false, + }, + + { + "var.foo", + &VariableInterpolation{ + Variable: &UserVariable{ + Name: "foo", + key: "var.foo", + }, + }, + false, + }, + + { + "lookup(var.foo, var.bar)", + &FunctionInterpolation{ + Func: nil, // Funcs["lookup"] + Args: []Interpolation{ + &VariableInterpolation{ + Variable: &UserVariable{ + Name: "foo", + key: "var.foo", + }, + }, + &VariableInterpolation{ + Variable: &UserVariable{ + Name: "bar", + key: "var.bar", + }, + }, + }, + }, + false, + }, + + { + "lookup(var.foo, lookup(var.baz, var.bar))", + &FunctionInterpolation{ + Func: nil, // Funcs["lookup"] + Args: []Interpolation{ + &VariableInterpolation{ + Variable: &UserVariable{ + Name: "foo", + key: "var.foo", + }, + }, + &FunctionInterpolation{ + Func: nil, // Funcs["lookup"] + Args: []Interpolation{ + &VariableInterpolation{ + Variable: &UserVariable{ + Name: "baz", + key: "var.baz", + }, + }, + &VariableInterpolation{ + Variable: &UserVariable{ + Name: "bar", + key: "var.bar", + }, + }, + }, + }, + }, + }, + false, + }, + } + + for i, tc := range cases { + actual, err := ExprParse(tc.Input) + if (err != nil) != tc.Error { + t.Fatalf("%d. Error: %s", i, err) + } + + // This is jank, but reflect.DeepEqual never has functions + // being the same. + f, ok := actual.(*FunctionInterpolation) + if ok { + fs := make([]*FunctionInterpolation, 1) + fs[0] = f + for len(fs) > 0 { + f := fs[0] + fs = fs[1:] + + f.Func = nil + for _, a := range f.Args { + f, ok := a.(*FunctionInterpolation) + if ok { + fs = append(fs, f) + } + } + } + } + + if !reflect.DeepEqual(actual, tc.Result) { + t.Fatalf("%d bad: %#v", i, actual) + } + } +} diff --git a/config/interpolate.go b/config/interpolate.go index 59f14a83b..cffd58b17 100644 --- a/config/interpolate.go +++ b/config/interpolate.go @@ -17,7 +17,6 @@ var funcRegexp *regexp.Regexp = regexp.MustCompile( // Interpolations might be simple variable references, or it might be // function calls, or even nested function calls. type Interpolation interface { - FullString() string Interpolate(map[string]string) (string, error) Variables() map[string]InterpolatedVariable } @@ -38,17 +37,19 @@ type InterpolatedVariable interface { // with some variable number of arguments to generate a value. type FunctionInterpolation struct { Func InterpolationFunc - Args []InterpolatedVariable + Args []Interpolation +} - key string +// LiteralInterpolation implements Interpolation for literals. Ex: +// ${"foo"} will equal "foo". +type LiteralInterpolation struct { + Literal string } // VariableInterpolation implements Interpolation for simple variable // interpolation. Ex: "${var.foo}" or "${aws_instance.foo.bar}" type VariableInterpolation struct { Variable InterpolatedVariable - - key string } // A ResourceVariable is a variable that is referencing the field @@ -74,61 +75,6 @@ type UserVariable struct { key string } -// NewInterpolation takes some string and returns the valid -// Interpolation associated with it, or error if a valid -// interpolation could not be found or the interpolation itself -// is invalid. -func NewInterpolation(v string) (Interpolation, error) { - match := funcRegexp.FindStringSubmatch(v) - if match != nil { - fn, ok := Funcs[match[1]] - if !ok { - return nil, fmt.Errorf( - "%s: Unknown function '%s'", - v, match[1]) - } - - args := make([]InterpolatedVariable, 0, len(match)-2) - for i := 2; i < len(match); i++ { - // This can be empty if we have a single argument - // due to the format of the regexp. - if match[i] == "" { - continue - } - - v, err := NewInterpolatedVariable(match[i]) - if err != nil { - return nil, err - } - - args = append(args, v) - } - - return &FunctionInterpolation{ - Func: fn, - Args: args, - - key: v, - }, nil - } - - if idx := strings.Index(v, "."); idx >= 0 { - v, err := NewInterpolatedVariable(v) - if err != nil { - return nil, err - } - - return &VariableInterpolation{ - Variable: v, - key: v.FullKey(), - }, nil - } - - return nil, fmt.Errorf( - "Interpolation '%s' is not a valid interpolation. " + - "Please check your syntax and try again.") -} - func NewInterpolatedVariable(v string) (InterpolatedVariable, error) { if !strings.HasPrefix(v, "var.") { return NewResourceVariable(v) @@ -137,21 +83,13 @@ func NewInterpolatedVariable(v string) (InterpolatedVariable, error) { return NewUserVariable(v) } -func (i *FunctionInterpolation) FullString() string { - return i.key -} - func (i *FunctionInterpolation) Interpolate( vs map[string]string) (string, error) { args := make([]string, len(i.Args)) for idx, a := range i.Args { - k := a.FullKey() - v, ok := vs[k] - if !ok { - return "", fmt.Errorf( - "%s: variable argument value unknown: %s", - i.FullString(), - k) + v, err := a.Interpolate(vs) + if err != nil { + return "", err } args[idx] = v @@ -160,38 +98,48 @@ func (i *FunctionInterpolation) Interpolate( return i.Func(vs, args...) } +func (i *FunctionInterpolation) GoString() string { + return fmt.Sprintf("*%#v", *i) +} + func (i *FunctionInterpolation) Variables() map[string]InterpolatedVariable { result := make(map[string]InterpolatedVariable) for _, a := range i.Args { - k := a.FullKey() - if _, ok := result[k]; ok { - continue + for k, v := range a.Variables() { + result[k] = v } - - result[k] = a } return result } -func (i *VariableInterpolation) FullString() string { - return i.key +func (i *LiteralInterpolation) Interpolate( + map[string]string) (string, error) { + return i.Literal, nil +} + +func (i *LiteralInterpolation) Variables() map[string]InterpolatedVariable { + return nil } func (i *VariableInterpolation) Interpolate( vs map[string]string) (string, error) { - v, ok := vs[i.key] + v, ok := vs[i.Variable.FullKey()] if !ok { return "", fmt.Errorf( "%s: value for variable not found", - i.key) + i.Variable.FullKey()) } return v, nil } +func (i *VariableInterpolation) GoString() string { + return fmt.Sprintf("*%#v", *i) +} + func (i *VariableInterpolation) Variables() map[string]InterpolatedVariable { - return map[string]InterpolatedVariable{i.key: i.Variable} + return map[string]InterpolatedVariable{i.Variable.FullKey(): i.Variable} } func NewResourceVariable(key string) (*ResourceVariable, error) { diff --git a/config/interpolate_test.go b/config/interpolate_test.go index 3b01c6cd7..92bb0f15c 100644 --- a/config/interpolate_test.go +++ b/config/interpolate_test.go @@ -6,68 +6,6 @@ import ( "testing" ) -func TestNewInterpolation(t *testing.T) { - cases := []struct { - Input string - Result Interpolation - Error bool - }{ - { - "foo", - nil, - true, - }, - - { - "var.foo", - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "foo", - key: "var.foo", - }, - key: "var.foo", - }, - false, - }, - - { - "lookup(var.foo, var.bar)", - &FunctionInterpolation{ - Func: nil, // Funcs["lookup"] - Args: []InterpolatedVariable{ - &UserVariable{ - Name: "foo", - key: "var.foo", - }, - &UserVariable{ - Name: "bar", - key: "var.bar", - }, - }, - key: "lookup(var.foo, var.bar)", - }, - false, - }, - } - - for i, tc := range cases { - actual, err := NewInterpolation(tc.Input) - if (err != nil) != tc.Error { - t.Fatalf("%d. Error: %s", i, err) - } - - // This is jank, but reflect.DeepEqual never has functions - // being the same. - if f, ok := actual.(*FunctionInterpolation); ok { - f.Func = nil - } - - if !reflect.DeepEqual(actual, tc.Result) { - t.Fatalf("%d bad: %#v", i, actual) - } - } -} - func TestNewInterpolatedVariable(t *testing.T) { cases := []struct { Input string @@ -171,11 +109,10 @@ func TestFunctionInterpolation(t *testing.T) { i := &FunctionInterpolation{ Func: fn, - Args: []InterpolatedVariable{v1, v2}, - key: "foo", - } - if i.FullString() != "foo" { - t.Fatalf("err: %#v", i) + Args: []Interpolation{ + &VariableInterpolation{Variable: v1}, + &VariableInterpolation{Variable: v2}, + }, } expected := map[string]InterpolatedVariable{ @@ -199,6 +136,29 @@ func TestFunctionInterpolation(t *testing.T) { } } +func TestLiteralInterpolation_impl(t *testing.T) { + var _ Interpolation = new(LiteralInterpolation) +} + +func TestLiteralInterpolation(t *testing.T) { + i := &LiteralInterpolation{ + Literal: "bar", + } + + if i.Variables() != nil { + t.Fatalf("bad: %#v", i.Variables()) + } + + actual, err := i.Interpolate(nil) + if err != nil { + t.Fatalf("err: %s", err) + } + + if actual != "bar" { + t.Fatalf("bad: %#v", actual) + } +} + func TestResourceVariable_impl(t *testing.T) { var _ InterpolatedVariable = new(ResourceVariable) } @@ -265,10 +225,7 @@ func TestVariableInterpolation(t *testing.T) { t.Fatalf("err: %s", err) } - i := &VariableInterpolation{Variable: uv, key: "var.foo"} - if i.FullString() != "var.foo" { - t.Fatalf("err: %#v", i) - } + i := &VariableInterpolation{Variable: uv} expected := map[string]InterpolatedVariable{"var.foo": uv} if !reflect.DeepEqual(i.Variables(), expected) { @@ -293,7 +250,7 @@ func TestVariableInterpolation_missing(t *testing.T) { t.Fatalf("err: %s", err) } - i := &VariableInterpolation{Variable: uv, key: "var.foo"} + i := &VariableInterpolation{Variable: uv} _, err = i.Interpolate(map[string]string{ "var.bar": "bar", }) diff --git a/config/interpolate_walk.go b/config/interpolate_walk.go index fd8a70833..83fa6c222 100644 --- a/config/interpolate_walk.go +++ b/config/interpolate_walk.go @@ -95,7 +95,7 @@ func (w *interpolationWalker) Primitive(v reflect.Value) error { // Interpolation found, instantiate it key := match[2] - i, err := NewInterpolation(key) + i, err := ExprParse(key) if err != nil { return err } diff --git a/config/interpolate_walk_test.go b/config/interpolate_walk_test.go index a731f8615..c27770e59 100644 --- a/config/interpolate_walk_test.go +++ b/config/interpolate_walk_test.go @@ -29,7 +29,6 @@ func TestInterpolationWalker_detect(t *testing.T) { Name: "foo", key: "var.foo", }, - key: "var.foo", }, }, }, @@ -41,13 +40,14 @@ func TestInterpolationWalker_detect(t *testing.T) { Result: []Interpolation{ &FunctionInterpolation{ Func: nil, - Args: []InterpolatedVariable{ - &UserVariable{ - Name: "foo", - key: "var.foo", + Args: []Interpolation{ + &VariableInterpolation{ + Variable: &UserVariable{ + Name: "foo", + key: "var.foo", + }, }, }, - key: "lookup(var.foo)", }, }, },