diff --git a/.gitignore b/.gitignore index 72a053db9..e852cc3b5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,6 @@ example.tf terraform.tfplan terraform.tfstate bin/ -config/y.go -config/y.output modules-dev/ pkg/ vendor/ diff --git a/Makefile b/Makefile index cfc16cb07..44922e283 100644 --- a/Makefile +++ b/Makefile @@ -2,26 +2,26 @@ TEST?=./... default: test -bin: config/y.go generate +bin: generate @sh -c "'$(CURDIR)/scripts/build.sh'" -dev: config/y.go generate +dev: generate @TF_DEV=1 sh -c "'$(CURDIR)/scripts/build.sh'" -test: config/y.go generate +test: generate TF_ACC= go test $(TEST) $(TESTARGS) -timeout=10s -parallel=4 -testacc: config/y.go generate +testacc: generate @if [ "$(TEST)" = "./..." ]; then \ echo "ERROR: Set TEST to a specific package"; \ exit 1; \ fi TF_ACC=1 go test $(TEST) -v $(TESTARGS) -timeout 45m -testrace: config/y.go generate +testrace: generate TF_ACC= go test -race $(TEST) $(TESTARGS) -updatedeps: config/y.go +updatedeps: go get -u golang.org/x/tools/cmd/stringer # Go 1.4 changed the format of `go get` a bit by requiring the # canonical full path. We work around this and just force. @@ -31,14 +31,7 @@ updatedeps: config/y.go go get -f -u -v ./...; \ fi -config/y.go: config/expr.y - cd config/ && \ - go tool yacc -p "expr" expr.y - -clean: - rm config/y.go - generate: go generate ./... -.PHONY: bin clean default generate test updatedeps +.PHONY: bin default generate test updatedeps diff --git a/config/config.go b/config/config.go index abe60db6e..7163e2829 100644 --- a/config/config.go +++ b/config/config.go @@ -8,6 +8,8 @@ import ( "strconv" "strings" + "github.com/hashicorp/terraform/config/lang" + "github.com/hashicorp/terraform/config/lang/ast" "github.com/hashicorp/terraform/flatmap" "github.com/hashicorp/terraform/helper/multierror" "github.com/mitchellh/mapstructure" @@ -169,7 +171,7 @@ func (c *Config) Validate() error { } interp := false - fn := func(i Interpolation) (string, error) { + fn := func(ast.Node) (string, error) { interp = true return "", nil } @@ -353,9 +355,18 @@ func (c *Config) Validate() error { } } - // Interpolate with a fixed number to verify that its a number - r.RawCount.interpolate(func(Interpolation) (string, error) { - return "5", nil + // Interpolate with a fixed number to verify that its a number. + r.RawCount.interpolate(func(root ast.Node) (string, error) { + // Execute the node but transform the AST so that it returns + // a fixed value of "5" for all interpolations. + var engine lang.Engine + out, _, err := engine.Execute(lang.FixedValueTransform( + root, &ast.LiteralNode{Value: "5", Type: ast.TypeString})) + if err != nil { + return "", err + } + + return out.(string), nil }) _, err := strconv.ParseInt(r.RawCount.Value().(string), 0, 0) if err != nil { @@ -465,20 +476,29 @@ func (c *Config) rawConfigs() map[string]*RawConfig { func (c *Config) validateVarContextFn( source string, errs *[]error) interpolationWalkerContextFunc { - return func(loc reflectwalk.Location, i Interpolation) { - vi, ok := i.(*VariableInterpolation) - if !ok { + return func(loc reflectwalk.Location, node ast.Node) { + if loc == reflectwalk.SliceElem { return } - rv, ok := vi.Variable.(*ResourceVariable) - if !ok { + vars, err := DetectVariables(node) + if err != nil { + // Ignore it since this will be caught during parse. This + // actually probably should never happen by the time this + // is called, but its okay. return } - if rv.Multi && rv.Index == -1 && loc != reflectwalk.SliceElem { - *errs = append(*errs, fmt.Errorf( - "%s: multi-variable must be in a slice", source)) + for _, v := range vars { + rv, ok := v.(*ResourceVariable) + if !ok { + return + } + + if rv.Multi && rv.Index == -1 { + *errs = append(*errs, fmt.Errorf( + "%s: multi-variable must be in a slice", source)) + } } } } diff --git a/config/expr.y b/config/expr.y deleted file mode 100644 index 1c295cce6..000000000 --- a/config/expr.y +++ /dev/null @@ -1,91 +0,0 @@ -// This is the yacc input for creating the parser for interpolation -// expressions in Go. - -// To build it: -// -// go tool yacc -p "expr" expr.y (produces y.go) -// - -%{ -package config - -import ( - "fmt" -) - -%} - -%union { - expr Interpolation - str string - variable InterpolatedVariable - args []Interpolation -} - -%type args -%type expr -%type string -%type variable - -%token STRING IDENTIFIER -%token COMMA LEFTPAREN RIGHTPAREN - -%% - -top: - expr - { - exprResult = $1 - } - -expr: - string - { - $$ = &LiteralInterpolation{Literal: $1} - } -| variable - { - $$ = &VariableInterpolation{Variable: $1} - } -| IDENTIFIER LEFTPAREN args RIGHTPAREN - { - f, ok := Funcs[$1] - if !ok { - exprErrors = append(exprErrors, fmt.Errorf( - "Unknown function: %s", $1)) - } - - $$ = &FunctionInterpolation{Func: f, Args: $3} - } - -args: - { - $$ = nil - } -| args COMMA expr - { - $$ = append($1, $3) - } -| expr - { - $$ = append($$, $1) - } - -string: - STRING - { - $$ = $1 - } - -variable: - IDENTIFIER - { - var err error - $$, err = NewInterpolatedVariable($1) - if err != nil { - exprErrors = append(exprErrors, fmt.Errorf( - "Error parsing variable '%s': %s", $1, err)) - } - } - -%% diff --git a/config/expr_lex.go b/config/expr_lex.go deleted file mode 100644 index f94e947ba..000000000 --- a/config/expr_lex.go +++ /dev/null @@ -1,134 +0,0 @@ -package config - -import ( - "bytes" - "fmt" - "log" - "unicode" - "unicode/utf8" -) - -// The parser expects the lexer to return 0 on EOF. -const lexEOF = 0 - -// The parser uses the type Lex as a lexer. It must provide -// the methods Lex(*SymType) int and Error(string). -type exprLex struct { - Err error - Input string - - pos int - width int -} - -// The parser calls this method to get each new token. -func (x *exprLex) Lex(yylval *exprSymType) int { - for { - c := x.next() - if c == lexEOF { - return lexEOF - } - - // Ignore all whitespace - if unicode.IsSpace(c) { - continue - } - - switch c { - case '"': - return x.lexString(yylval) - case ',': - return COMMA - case '(': - return LEFTPAREN - case ')': - return RIGHTPAREN - default: - x.backup() - return x.lexId(yylval) - } - } -} - -func (x *exprLex) lexId(yylval *exprSymType) int { - var b bytes.Buffer - for { - c := x.next() - if c == lexEOF { - break - } - - // If this isn't a character we want in an ID, return out. - // One day we should make this a regexp. - if c != '_' && - c != '-' && - c != '.' && - c != '*' && - !unicode.IsLetter(c) && - !unicode.IsNumber(c) { - x.backup() - break - } - - if _, err := b.WriteRune(c); err != nil { - log.Printf("ERR: %s", err) - return lexEOF - } - } - - yylval.str = b.String() - return IDENTIFIER -} - -func (x *exprLex) lexString(yylval *exprSymType) int { - var b bytes.Buffer - for { - c := x.next() - if c == lexEOF { - break - } - - // String end - if c == '"' { - break - } - - if _, err := b.WriteRune(c); err != nil { - log.Printf("ERR: %s", err) - return lexEOF - } - } - - yylval.str = b.String() - return STRING -} - -// Return the next rune for the lexer. -func (x *exprLex) next() rune { - if int(x.pos) >= len(x.Input) { - x.width = 0 - return lexEOF - } - - r, w := utf8.DecodeRuneInString(x.Input[x.pos:]) - x.width = w - x.pos += x.width - return r -} - -// peek returns but does not consume the next rune in the input -func (x *exprLex) peek() rune { - r := x.next() - x.backup() - return r -} - -// backup steps back one rune. Can only be called once per next. -func (x *exprLex) backup() { - x.pos -= x.width -} - -// The parser calls this method on a parse error. -func (x *exprLex) Error(s string) { - x.Err = fmt.Errorf("parse error: %s", s) -} diff --git a/config/expr_lex_test.go b/config/expr_lex_test.go deleted file mode 100644 index 8e94d86f3..000000000 --- a/config/expr_lex_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package config - -import ( - "io/ioutil" - "path/filepath" - "reflect" - "testing" -) - -func TestLex(t *testing.T) { - cases := []struct { - Input string - Output []int - }{ - { - "concat.hcl", - []int{IDENTIFIER, LEFTPAREN, - STRING, COMMA, STRING, COMMA, STRING, - RIGHTPAREN, lexEOF}, - }, - } - - for _, tc := range cases { - d, err := ioutil.ReadFile(filepath.Join( - fixtureDir, "interpolations", tc.Input)) - if err != nil { - t.Fatalf("err: %s", err) - } - - l := &exprLex{Input: string(d)} - var actual []int - for { - token := l.Lex(new(exprSymType)) - actual = append(actual, token) - - if token == lexEOF { - break - } - - if len(actual) > 500 { - t.Fatalf("Input:%s\n\nExausted.", tc.Input) - } - } - - if !reflect.DeepEqual(actual, tc.Output) { - t.Fatalf( - "Input: %s\n\nBad: %#v\n\nExpected: %#v", - tc.Input, actual, tc.Output) - } - } -} diff --git a/config/expr_parse.go b/config/expr_parse.go deleted file mode 100644 index c6fb39f87..000000000 --- a/config/expr_parse.go +++ /dev/null @@ -1,40 +0,0 @@ -package config - -import ( - "sync" - - "github.com/hashicorp/terraform/helper/multierror" -) - -// exprErrors are the errors built up from parsing. These should not -// be accessed directly. -var exprErrors []error -var exprLock sync.Mutex -var exprResult Interpolation - -// ExprParse parses the given expression and returns an executable -// Interpolation. -func ExprParse(v string) (Interpolation, error) { - exprLock.Lock() - defer exprLock.Unlock() - exprErrors = nil - exprResult = nil - - // Parse - lex := &exprLex{Input: v} - exprParse(lex) - - // Build up the errors - var err error - if lex.Err != nil { - err = multierror.ErrorAppend(err, lex.Err) - } - if len(exprErrors) > 0 { - err = multierror.ErrorAppend(err, exprErrors...) - } - if err != nil { - exprResult = nil - } - - return exprResult, err -} diff --git a/config/expr_parse_test.go b/config/expr_parse_test.go deleted file mode 100644 index da77c7bda..000000000 --- a/config/expr_parse_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package config - -import ( - "reflect" - "testing" -) - -func TestExprParse(t *testing.T) { - cases := []struct { - Input string - Result Interpolation - Error bool - }{ - { - "foo", - nil, - true, - }, - - { - `"foo"`, - &LiteralInterpolation{Literal: "foo"}, - false, - }, - - { - "var.foo", - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "foo", - key: "var.foo", - }, - }, - false, - }, - - { - "module.foo.bar", - &VariableInterpolation{ - Variable: &ModuleVariable{ - Name: "foo", - Field: "bar", - key: "module.foo.bar", - }, - }, - false, - }, - - { - "lookup(var.foo, var.bar)", - &FunctionInterpolation{ - Func: nil, // Funcs["lookup"] - Args: []Interpolation{ - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "foo", - key: "var.foo", - }, - }, - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "bar", - key: "var.bar", - }, - }, - }, - }, - false, - }, - - { - "lookup(var.foo, lookup(var.baz, var.bar))", - &FunctionInterpolation{ - Func: nil, // Funcs["lookup"] - Args: []Interpolation{ - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "foo", - key: "var.foo", - }, - }, - &FunctionInterpolation{ - Func: nil, // Funcs["lookup"] - Args: []Interpolation{ - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "baz", - key: "var.baz", - }, - }, - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "bar", - key: "var.bar", - }, - }, - }, - }, - }, - }, - false, - }, - - { - `concat("foo","-","0.0/16")`, - &FunctionInterpolation{ - Func: nil, // Funcs["lookup"] - Args: []Interpolation{ - &LiteralInterpolation{Literal: "foo"}, - &LiteralInterpolation{Literal: "-"}, - &LiteralInterpolation{Literal: "0.0/16"}, - }, - }, - false, - }, - } - - for i, tc := range cases { - actual, err := ExprParse(tc.Input) - if (err != nil) != tc.Error { - t.Fatalf("%d. Error: %s", i, err) - } - - // This is jank, but reflect.DeepEqual never has functions - // being the same. - f, ok := actual.(*FunctionInterpolation) - if ok { - fs := make([]*FunctionInterpolation, 1) - fs[0] = f - for len(fs) > 0 { - f := fs[0] - fs = fs[1:] - - f.Func = nil - for _, a := range f.Args { - f, ok := a.(*FunctionInterpolation) - if ok { - fs = append(fs, f) - } - } - } - } - - if !reflect.DeepEqual(actual, tc.Result) { - t.Fatalf("%d bad: %#v", i, actual) - } - } -} diff --git a/config/interpolate.go b/config/interpolate.go index e96b6b76c..e6a709d94 100644 --- a/config/interpolate.go +++ b/config/interpolate.go @@ -2,29 +2,12 @@ package config import ( "fmt" - "regexp" "strconv" "strings" + + "github.com/hashicorp/terraform/config/lang/ast" ) -// We really need to replace this with a real parser. -var funcRegexp *regexp.Regexp = regexp.MustCompile( - `(?i)([a-z0-9_]+)\(\s*(?:([.a-z0-9_]+)\s*,\s*)*([.a-z0-9_]+)\s*\)`) - -// Interpolation is something that can be contained in a "${}" in a -// configuration value. -// -// Interpolations might be simple variable references, or it might be -// function calls, or even nested function calls. -type Interpolation interface { - Interpolate(map[string]string) (string, error) - Variables() map[string]InterpolatedVariable -} - -// InterpolationFunc is the function signature for implementing -// callable functions in Terraform configurations. -type InterpolationFunc func(map[string]string, ...string) (string, error) - // An InterpolatedVariable is a variable reference within an interpolation. // // Implementations of this interface represents various sources where @@ -33,25 +16,6 @@ type InterpolatedVariable interface { FullKey() string } -// FunctionInterpolation is an Interpolation that executes a function -// with some variable number of arguments to generate a value. -type FunctionInterpolation struct { - Func InterpolationFunc - Args []Interpolation -} - -// LiteralInterpolation implements Interpolation for literals. Ex: -// ${"foo"} will equal "foo". -type LiteralInterpolation struct { - Literal string -} - -// VariableInterpolation implements Interpolation for simple variable -// interpolation. Ex: "${var.foo}" or "${aws_instance.foo.bar}" -type VariableInterpolation struct { - Variable InterpolatedVariable -} - // CountVariable is a variable for referencing information about // the count. type CountVariable struct { @@ -128,65 +92,6 @@ func NewInterpolatedVariable(v string) (InterpolatedVariable, error) { } } -func (i *FunctionInterpolation) Interpolate( - vs map[string]string) (string, error) { - args := make([]string, len(i.Args)) - for idx, a := range i.Args { - v, err := a.Interpolate(vs) - if err != nil { - return "", err - } - - args[idx] = v - } - - return i.Func(vs, args...) -} - -func (i *FunctionInterpolation) GoString() string { - return fmt.Sprintf("*%#v", *i) -} - -func (i *FunctionInterpolation) Variables() map[string]InterpolatedVariable { - result := make(map[string]InterpolatedVariable) - for _, a := range i.Args { - for k, v := range a.Variables() { - result[k] = v - } - } - - return result -} - -func (i *LiteralInterpolation) Interpolate( - map[string]string) (string, error) { - return i.Literal, nil -} - -func (i *LiteralInterpolation) Variables() map[string]InterpolatedVariable { - return nil -} - -func (i *VariableInterpolation) Interpolate( - vs map[string]string) (string, error) { - v, ok := vs[i.Variable.FullKey()] - if !ok { - return "", fmt.Errorf( - "%s: value for variable not found", - i.Variable.FullKey()) - } - - return v, nil -} - -func (i *VariableInterpolation) GoString() string { - return fmt.Sprintf("*%#v", *i) -} - -func (i *VariableInterpolation) Variables() map[string]InterpolatedVariable { - return map[string]InterpolatedVariable{i.Variable.FullKey(): i.Variable} -} - func NewCountVariable(key string) (*CountVariable, error) { var fieldType CountValueType parts := strings.SplitN(key, ".", 2) @@ -317,3 +222,39 @@ func (v *UserVariable) FullKey() string { func (v *UserVariable) GoString() string { return fmt.Sprintf("*%#v", *v) } + +// DetectVariables takes an AST root and returns all the interpolated +// variables that are detected in the AST tree. +func DetectVariables(root ast.Node) ([]InterpolatedVariable, error) { + var result []InterpolatedVariable + var resultErr error + + // Visitor callback + fn := func(n ast.Node) { + if resultErr != nil { + return + } + + vn, ok := n.(*ast.VariableAccess) + if !ok { + return + } + + v, err := NewInterpolatedVariable(vn.Name) + if err != nil { + resultErr = err + return + } + + result = append(result, v) + } + + // Visitor pattern + root.Accept(fn) + + if resultErr != nil { + return nil, resultErr + } + + return result, nil +} diff --git a/config/interpolate_funcs.go b/config/interpolate_funcs.go index cbba62d0d..65cebbfa1 100644 --- a/config/interpolate_funcs.go +++ b/config/interpolate_funcs.go @@ -6,109 +6,117 @@ import ( "io/ioutil" "strconv" "strings" + + "github.com/hashicorp/terraform/config/lang" + "github.com/hashicorp/terraform/config/lang/ast" ) // Funcs is the mapping of built-in functions for configuration. -var Funcs map[string]InterpolationFunc +var Funcs map[string]lang.Function func init() { - Funcs = map[string]InterpolationFunc{ - "concat": interpolationFuncConcat, - "file": interpolationFuncFile, - "join": interpolationFuncJoin, - "lookup": interpolationFuncLookup, - "element": interpolationFuncElement, + Funcs = map[string]lang.Function{ + "concat": interpolationFuncConcat(), + "file": interpolationFuncFile(), + "join": interpolationFuncJoin(), + "element": interpolationFuncElement(), } } -// interpolationFuncConcat implements the "concat" function that allows -// strings to be joined together. -func interpolationFuncConcat( - vs map[string]string, args ...string) (string, error) { - var buf bytes.Buffer +// interpolationFuncConcat implements the "concat" function that +// concatenates multiple strings. This isn't actually necessary anymore +// since our language supports string concat natively, but for backwards +// compat we do this. +func interpolationFuncConcat() lang.Function { + return lang.Function{ + ArgTypes: []ast.Type{ast.TypeString}, + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeString, + Callback: func(args []interface{}) (interface{}, error) { + var b bytes.Buffer + for _, v := range args { + b.WriteString(v.(string)) + } - for _, a := range args { - if _, err := buf.WriteString(a); err != nil { - return "", err - } + return b.String(), nil + }, } - - return buf.String(), nil } // interpolationFuncFile implements the "file" function that allows // loading contents from a file. -func interpolationFuncFile( - vs map[string]string, args ...string) (string, error) { - if len(args) != 1 { - return "", fmt.Errorf( - "file expects 1 arguments, got %d", len(args)) - } +func interpolationFuncFile() lang.Function { + return lang.Function{ + ArgTypes: []ast.Type{ast.TypeString}, + ReturnType: ast.TypeString, + Callback: func(args []interface{}) (interface{}, error) { + data, err := ioutil.ReadFile(args[0].(string)) + if err != nil { + return "", err + } - data, err := ioutil.ReadFile(args[0]) - if err != nil { - return "", err + return string(data), nil + }, } - - return string(data), nil } // interpolationFuncJoin implements the "join" function that allows // multi-variable values to be joined by some character. -func interpolationFuncJoin( - vs map[string]string, args ...string) (string, error) { - if len(args) < 2 { - return "", fmt.Errorf("join expects 2 arguments") - } +func interpolationFuncJoin() lang.Function { + return lang.Function{ + ArgTypes: []ast.Type{ast.TypeString, ast.TypeString}, + ReturnType: ast.TypeString, + Callback: func(args []interface{}) (interface{}, error) { + var list []string + for _, arg := range args[1:] { + parts := strings.Split(arg.(string), InterpSplitDelim) + list = append(list, parts...) + } - var list []string - for _, arg := range args[1:] { - parts := strings.Split(arg, InterpSplitDelim) - list = append(list, parts...) + return strings.Join(list, args[0].(string)), nil + }, } - - return strings.Join(list, args[0]), nil } // interpolationFuncLookup implements the "lookup" function that allows // dynamic lookups of map types within a Terraform configuration. -func interpolationFuncLookup( - vs map[string]string, args ...string) (string, error) { - if len(args) != 2 { - return "", fmt.Errorf( - "lookup expects 2 arguments, got %d", len(args)) - } +func interpolationFuncLookup(vs map[string]string) lang.Function { + return lang.Function{ + ArgTypes: []ast.Type{ast.TypeString, ast.TypeString}, + ReturnType: ast.TypeString, + Callback: func(args []interface{}) (interface{}, error) { + k := fmt.Sprintf("var.%s.%s", args[0].(string), args[1].(string)) + v, ok := vs[k] + if !ok { + return "", fmt.Errorf( + "lookup in '%s' failed to find '%s'", + args[0].(string), args[1].(string)) + } - k := fmt.Sprintf("var.%s", strings.Join(args, ".")) - v, ok := vs[k] - if !ok { - return "", fmt.Errorf( - "lookup in '%s' failed to find '%s'", - args[0], args[1]) + return v, nil + }, } - - return v, nil } // interpolationFuncElement implements the "element" function that allows // a specific index to be looked up in a multi-variable value. Note that this will // wrap if the index is larger than the number of elements in the multi-variable value. -func interpolationFuncElement( - vs map[string]string, args ...string) (string, error) { - if len(args) != 2 { - return "", fmt.Errorf( - "element expects 2 arguments, got %d", len(args)) +func interpolationFuncElement() lang.Function { + return lang.Function{ + ArgTypes: []ast.Type{ast.TypeString, ast.TypeString}, + ReturnType: ast.TypeString, + Callback: func(args []interface{}) (interface{}, error) { + list := strings.Split(args[0].(string), InterpSplitDelim) + + index, err := strconv.Atoi(args[1].(string)) + if err != nil { + return "", fmt.Errorf( + "invalid number for index, got %s", args[1]) + } + + v := list[index%len(list)] + return v, nil + }, } - - list := strings.Split(args[0], InterpSplitDelim) - - index, err := strconv.Atoi(args[1]) - if err != nil { - return "", fmt.Errorf( - "invalid number for index, got %s", args[1]) - } - - v := list[index % len(list)] - - return v, nil } diff --git a/config/interpolate_funcs_test.go b/config/interpolate_funcs_test.go index 332b9af4b..757f8aa5b 100644 --- a/config/interpolate_funcs_test.go +++ b/config/interpolate_funcs_test.go @@ -4,44 +4,34 @@ import ( "fmt" "io/ioutil" "os" + "reflect" "testing" + + "github.com/hashicorp/terraform/config/lang" ) func TestInterpolateFuncConcat(t *testing.T) { - cases := []struct { - Args []string - Result string - Error bool - }{ - { - []string{"foo", "bar", "baz"}, - "foobarbaz", - false, + testFunction(t, testFunctionConfig{ + Cases: []testFunctionCase{ + { + `${concat("foo", "bar")}`, + "foobar", + false, + }, + + { + `${concat("foo")}`, + "foo", + false, + }, + + { + `${concat()}`, + nil, + true, + }, }, - - { - []string{"foo", "bar"}, - "foobar", - false, - }, - - { - []string{"foo"}, - "foo", - false, - }, - } - - for i, tc := range cases { - actual, err := interpolationFuncConcat(nil, tc.Args...) - if (err != nil) != tc.Error { - t.Fatalf("%d: err: %s", i, err) - } - - if actual != tc.Result { - t.Fatalf("%d: bad: %#v", i, actual) - } - } + }) } func TestInterpolateFuncFile(t *testing.T) { @@ -54,183 +44,156 @@ func TestInterpolateFuncFile(t *testing.T) { tf.Close() defer os.Remove(path) - cases := []struct { - Args []string - Result string - Error bool - }{ - { - []string{path}, - "foo", - false, + testFunction(t, testFunctionConfig{ + Cases: []testFunctionCase{ + { + fmt.Sprintf(`${file("%s")}`, path), + "foo", + false, + }, + + // Invalid path + { + `${file("/i/dont/exist")}`, + nil, + true, + }, + + // Too many args + { + `${file("foo", "bar")}`, + nil, + true, + }, }, - - // Invalid path - { - []string{"/i/dont/exist"}, - "", - true, - }, - - // Too many args - { - []string{"foo", "bar"}, - "", - true, - }, - } - - for i, tc := range cases { - actual, err := interpolationFuncFile(nil, tc.Args...) - if (err != nil) != tc.Error { - t.Fatalf("%d: err: %s", i, err) - } - - if actual != tc.Result { - t.Fatalf("%d: bad: %#v", i, actual) - } - } + }) } func TestInterpolateFuncJoin(t *testing.T) { - cases := []struct { - Args []string - Result string - Error bool - }{ - { - []string{","}, - "", - true, - }, - - { - []string{",", "foo"}, - "foo", - false, - }, - - { - []string{",", "foo", "bar"}, - "foo,bar", - false, - }, - - { - []string{ - ".", - fmt.Sprintf( - "foo%sbar%sbaz", - InterpSplitDelim, - InterpSplitDelim), + testFunction(t, testFunctionConfig{ + Cases: []testFunctionCase{ + { + `${join(",")}`, + nil, + true, + }, + + { + `${join(",", "foo")}`, + "foo", + false, + }, + + /* + TODO + { + `${join(",", "foo", "bar")}`, + "foo,bar", + false, + }, + */ + + { + fmt.Sprintf(`${join(".", "%s")}`, + fmt.Sprintf( + "foo%sbar%sbaz", + InterpSplitDelim, + InterpSplitDelim)), + "foo.bar.baz", + false, }, - "foo.bar.baz", - false, }, - } - - for i, tc := range cases { - actual, err := interpolationFuncJoin(nil, tc.Args...) - if (err != nil) != tc.Error { - t.Fatalf("%d: err: %s", i, err) - } - - if actual != tc.Result { - t.Fatalf("%d: bad: %#v", i, actual) - } - } + }) } func TestInterpolateFuncLookup(t *testing.T) { - cases := []struct { - M map[string]string - Args []string - Result string - Error bool - }{ - { - map[string]string{ - "var.foo.bar": "baz", + testFunction(t, testFunctionConfig{ + Vars: map[string]string{"var.foo.bar": "baz"}, + Cases: []testFunctionCase{ + { + `${lookup("foo", "bar")}`, + "baz", + false, }, - []string{"foo", "bar"}, - "baz", - false, - }, - // Invalid key - { - map[string]string{ - "var.foo.bar": "baz", + // Invalid key + { + `${lookup("foo", "baz")}`, + nil, + true, }, - []string{"foo", "baz"}, - "", - true, - }, - // Too many args - { - map[string]string{ - "var.foo.bar": "baz", + // Too many args + { + `${lookup("foo", "bar", "baz")}`, + nil, + true, }, - []string{"foo", "bar", "baz"}, - "", - true, }, - } - - for i, tc := range cases { - actual, err := interpolationFuncLookup(tc.M, tc.Args...) - if (err != nil) != tc.Error { - t.Fatalf("%d: err: %s", i, err) - } - - if actual != tc.Result { - t.Fatalf("%d: bad: %#v", i, actual) - } - } + }) } func TestInterpolateFuncElement(t *testing.T) { - cases := []struct { - Args []string - Result string - Error bool - }{ - { - []string{"foo" + InterpSplitDelim + "baz", "1"}, - "baz", - false, - }, + testFunction(t, testFunctionConfig{ + Cases: []testFunctionCase{ + { + fmt.Sprintf(`${element("%s", "1")}`, + "foo"+InterpSplitDelim+"baz"), + "baz", + false, + }, - { - []string{"foo", "0"}, - "foo", - false, - }, + { + `${element("foo", "0")}`, + "foo", + false, + }, - // Invalid index should wrap vs. out-of-bounds - { - []string{"foo" + InterpSplitDelim + "baz", "2"}, - "foo", - false, - }, + // Invalid index should wrap vs. out-of-bounds + { + fmt.Sprintf(`${element("%s", "2")}`, + "foo"+InterpSplitDelim+"baz"), + "foo", + false, + }, - // Too many args - { - []string{"foo" + InterpSplitDelim + "baz", "0", "1"}, - "", - true, + // Too many args + { + fmt.Sprintf(`${element("%s", "0", "2")}`, + "foo"+InterpSplitDelim+"baz"), + nil, + true, + }, }, - } + }) +} - for i, tc := range cases { - actual, err := interpolationFuncElement(nil, tc.Args...) +type testFunctionConfig struct { + Cases []testFunctionCase + Vars map[string]string +} + +type testFunctionCase struct { + Input string + Result interface{} + Error bool +} + +func testFunction(t *testing.T, config testFunctionConfig) { + for i, tc := range config.Cases { + ast, err := lang.Parse(tc.Input) + if err != nil { + t.Fatalf("%d: err: %s", i, err) + } + + engine := langEngine(config.Vars) + out, _, err := engine.Execute(ast) if (err != nil) != tc.Error { t.Fatalf("%d: err: %s", i, err) } - if actual != tc.Result { - t.Fatalf("%d: bad: %#v", i, actual) + if !reflect.DeepEqual(out, tc.Result) { + t.Fatalf("%d: bad: %#v", i, out) } } } diff --git a/config/interpolate_test.go b/config/interpolate_test.go index 9ecbb9a71..46945ee81 100644 --- a/config/interpolate_test.go +++ b/config/interpolate_test.go @@ -2,8 +2,9 @@ package config import ( "reflect" - "strings" "testing" + + "github.com/hashicorp/terraform/config/lang" ) func TestNewInterpolatedVariable(t *testing.T) { @@ -121,77 +122,6 @@ func TestNewUserVariable_map(t *testing.T) { } } -func TestFunctionInterpolation_impl(t *testing.T) { - var _ Interpolation = new(FunctionInterpolation) -} - -func TestFunctionInterpolation(t *testing.T) { - v1, err := NewInterpolatedVariable("var.foo") - if err != nil { - t.Fatalf("err: %s", err) - } - - v2, err := NewInterpolatedVariable("var.bar") - if err != nil { - t.Fatalf("err: %s", err) - } - - fn := func(vs map[string]string, args ...string) (string, error) { - return strings.Join(args, " "), nil - } - - i := &FunctionInterpolation{ - Func: fn, - Args: []Interpolation{ - &VariableInterpolation{Variable: v1}, - &VariableInterpolation{Variable: v2}, - }, - } - - expected := map[string]InterpolatedVariable{ - "var.foo": v1, - "var.bar": v2, - } - if !reflect.DeepEqual(i.Variables(), expected) { - t.Fatalf("bad: %#v", i.Variables()) - } - - actual, err := i.Interpolate(map[string]string{ - "var.foo": "bar", - "var.bar": "baz", - }) - if err != nil { - t.Fatalf("err: %s", err) - } - - if actual != "bar baz" { - t.Fatalf("bad: %#v", actual) - } -} - -func TestLiteralInterpolation_impl(t *testing.T) { - var _ Interpolation = new(LiteralInterpolation) -} - -func TestLiteralInterpolation(t *testing.T) { - i := &LiteralInterpolation{ - Literal: "bar", - } - - if i.Variables() != nil { - t.Fatalf("bad: %#v", i.Variables()) - } - - actual, err := i.Interpolate(nil) - if err != nil { - t.Fatalf("err: %s", err) - } - - if actual != "bar" { - t.Fatalf("bad: %#v", actual) - } -} - func TestResourceVariable_impl(t *testing.T) { var _ InterpolatedVariable = new(ResourceVariable) } @@ -248,46 +178,53 @@ func TestUserVariable_impl(t *testing.T) { var _ InterpolatedVariable = new(UserVariable) } -func TestVariableInterpolation_impl(t *testing.T) { - var _ Interpolation = new(VariableInterpolation) -} +func TestDetectVariables(t *testing.T) { + cases := []struct { + Input string + Result []InterpolatedVariable + }{ + { + "foo $${var.foo}", + nil, + }, -func TestVariableInterpolation(t *testing.T) { - uv, err := NewUserVariable("var.foo") - if err != nil { - t.Fatalf("err: %s", err) + { + "foo ${var.foo}", + []InterpolatedVariable{ + &UserVariable{ + Name: "foo", + key: "var.foo", + }, + }, + }, + + { + "foo ${var.foo} ${var.bar}", + []InterpolatedVariable{ + &UserVariable{ + Name: "foo", + key: "var.foo", + }, + &UserVariable{ + Name: "bar", + key: "var.bar", + }, + }, + }, } - i := &VariableInterpolation{Variable: uv} + for _, tc := range cases { + ast, err := lang.Parse(tc.Input) + if err != nil { + t.Fatalf("%s\n\nInput: %s", err, tc.Input) + } - expected := map[string]InterpolatedVariable{"var.foo": uv} - if !reflect.DeepEqual(i.Variables(), expected) { - t.Fatalf("bad: %#v", i.Variables()) - } - - actual, err := i.Interpolate(map[string]string{ - "var.foo": "bar", - }) - if err != nil { - t.Fatalf("err: %s", err) - } - - if actual != "bar" { - t.Fatalf("bad: %#v", actual) - } -} - -func TestVariableInterpolation_missing(t *testing.T) { - uv, err := NewUserVariable("var.foo") - if err != nil { - t.Fatalf("err: %s", err) - } - - i := &VariableInterpolation{Variable: uv} - _, err = i.Interpolate(map[string]string{ - "var.bar": "bar", - }) - if err == nil { - t.Fatal("should error") + actual, err := DetectVariables(ast) + if err != nil { + t.Fatalf("err: %s", err) + } + if !reflect.DeepEqual(actual, tc.Result) { + t.Fatalf("bad: %#v\n\nInput: %s", actual, tc.Input) + } } } diff --git a/config/interpolate_walk.go b/config/interpolate_walk.go index 7802537dc..17329e5a8 100644 --- a/config/interpolate_walk.go +++ b/config/interpolate_walk.go @@ -3,9 +3,10 @@ package config import ( "fmt" "reflect" - "regexp" "strings" + "github.com/hashicorp/terraform/config/lang" + "github.com/hashicorp/terraform/config/lang/ast" "github.com/mitchellh/reflectwalk" ) @@ -14,10 +15,6 @@ import ( // a value that a user is very unlikely to use (such as UUID). const InterpSplitDelim = `B780FFEC-B661-4EB8-9236-A01737AD98B6` -// interpRegexp is a regexp that matches interpolations such as ${foo.bar} -var interpRegexp *regexp.Regexp = regexp.MustCompile( - `(?i)(\$+)\{([\s*-.,\\/\(\):a-z0-9_"]+)\}`) - // interpolationWalker implements interfaces for the reflectwalk package // (github.com/mitchellh/reflectwalk) that can be used to automatically // execute a callback for an interpolation. @@ -50,7 +47,7 @@ type interpolationWalker struct { // // If Replace is set to false in interpolationWalker, then the replace // value can be anything as it will have no effect. -type interpolationWalkerFunc func(Interpolation) (string, error) +type interpolationWalkerFunc func(ast.Node) (string, error) // interpolationWalkerContextFunc is called by interpolationWalk if // ContextF is set. This receives both the interpolation and the location @@ -58,7 +55,7 @@ type interpolationWalkerFunc func(Interpolation) (string, error) // // This callback can be used to validate the location of the interpolation // within the configuration. -type interpolationWalkerContextFunc func(reflectwalk.Location, Interpolation) +type interpolationWalkerContextFunc func(reflectwalk.Location, ast.Node) func (w *interpolationWalker) Enter(loc reflectwalk.Location) error { w.loc = loc @@ -121,76 +118,54 @@ func (w *interpolationWalker) Primitive(v reflect.Value) error { return nil } - // XXX: This can be a lot more efficient if we used a real - // parser. A regexp is a hammer though that will get this working. + astRoot, err := lang.Parse(v.String()) + if err != nil { + return err + } - matches := interpRegexp.FindAllStringSubmatch(v.String(), -1) - if len(matches) == 0 { + // If the AST we got is just a literal string value, then we ignore it + if _, ok := astRoot.(*ast.LiteralNode); ok { return nil } - result := v.String() - for _, match := range matches { - dollars := len(match[1]) + if w.ContextF != nil { + w.ContextF(w.loc, astRoot) + } - // If there are even amounts of dollar signs, then it is escaped - if dollars%2 == 0 { - continue - } + if w.F == nil { + return nil + } - // Interpolation found, instantiate it - key := match[2] - - i, err := ExprParse(key) - if err != nil { - return err - } - - if w.ContextF != nil { - w.ContextF(w.loc, i) - } - - if w.F == nil { - continue - } - - replaceVal, err := w.F(i) - if err != nil { - return fmt.Errorf( - "%s: %s", - key, - err) - } - - if w.Replace { - // We need to determine if we need to remove this element - // if the result contains any "UnknownVariableValue" which is - // set if it is computed. This behavior is different if we're - // splitting (in a SliceElem) or not. - remove := false - if w.loc == reflectwalk.SliceElem { - parts := strings.Split(replaceVal, InterpSplitDelim) - for _, p := range parts { - if p == UnknownVariableValue { - remove = true - break - } - } - } else if replaceVal == UnknownVariableValue { - remove = true - } - if remove { - w.removeCurrent() - return nil - } - - // Replace in our interpolation and continue on. - result = strings.Replace(result, match[0], replaceVal, -1) - } + replaceVal, err := w.F(astRoot) + if err != nil { + return fmt.Errorf( + "%s in:\n\n%s", + err, v.String()) } if w.Replace { - resultVal := reflect.ValueOf(result) + // We need to determine if we need to remove this element + // if the result contains any "UnknownVariableValue" which is + // set if it is computed. This behavior is different if we're + // splitting (in a SliceElem) or not. + remove := false + if w.loc == reflectwalk.SliceElem { + parts := strings.Split(replaceVal, InterpSplitDelim) + for _, p := range parts { + if p == UnknownVariableValue { + remove = true + break + } + } + } else if replaceVal == UnknownVariableValue { + remove = true + } + if remove { + w.removeCurrent() + return nil + } + + resultVal := reflect.ValueOf(replaceVal) switch w.loc { case reflectwalk.MapKey: m := w.cs[len(w.cs)-1] diff --git a/config/interpolate_walk_test.go b/config/interpolate_walk_test.go index 6e8928694..9b2c34133 100644 --- a/config/interpolate_walk_test.go +++ b/config/interpolate_walk_test.go @@ -1,16 +1,18 @@ package config import ( + "fmt" "reflect" "testing" + "github.com/hashicorp/terraform/config/lang/ast" "github.com/mitchellh/reflectwalk" ) func TestInterpolationWalker_detect(t *testing.T) { cases := []struct { Input interface{} - Result []Interpolation + Result []string }{ { Input: map[string]interface{}{ @@ -23,13 +25,8 @@ func TestInterpolationWalker_detect(t *testing.T) { Input: map[string]interface{}{ "foo": "${var.foo}", }, - Result: []Interpolation{ - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "foo", - key: "var.foo", - }, - }, + Result: []string{ + "Variable(var.foo)", }, }, @@ -37,19 +34,8 @@ func TestInterpolationWalker_detect(t *testing.T) { Input: map[string]interface{}{ "foo": "${aws_instance.foo.*.num}", }, - Result: []Interpolation{ - &VariableInterpolation{ - Variable: &ResourceVariable{ - Type: "aws_instance", - Name: "foo", - Field: "num", - - Multi: true, - Index: -1, - - key: "aws_instance.foo.*.num", - }, - }, + Result: []string{ + "Variable(aws_instance.foo.*.num)", }, }, @@ -57,18 +43,8 @@ func TestInterpolationWalker_detect(t *testing.T) { Input: map[string]interface{}{ "foo": "${lookup(var.foo)}", }, - Result: []Interpolation{ - &FunctionInterpolation{ - Func: nil, - Args: []Interpolation{ - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "foo", - key: "var.foo", - }, - }, - }, - }, + Result: []string{ + "Call(lookup, Variable(var.foo))", }, }, @@ -76,15 +52,8 @@ func TestInterpolationWalker_detect(t *testing.T) { Input: map[string]interface{}{ "foo": `${file("test.txt")}`, }, - Result: []Interpolation{ - &FunctionInterpolation{ - Func: nil, - Args: []Interpolation{ - &LiteralInterpolation{ - Literal: "test.txt", - }, - }, - }, + Result: []string{ + "Call(file, Literal(TypeString, test.txt))", }, }, @@ -92,15 +61,8 @@ func TestInterpolationWalker_detect(t *testing.T) { Input: map[string]interface{}{ "foo": `${file("foo/bar.txt")}`, }, - Result: []Interpolation{ - &FunctionInterpolation{ - Func: nil, - Args: []Interpolation{ - &LiteralInterpolation{ - Literal: "foo/bar.txt", - }, - }, - }, + Result: []string{ + "Call(file, Literal(TypeString, foo/bar.txt))", }, }, @@ -108,25 +70,8 @@ func TestInterpolationWalker_detect(t *testing.T) { Input: map[string]interface{}{ "foo": `${join(",", foo.bar.*.id)}`, }, - Result: []Interpolation{ - &FunctionInterpolation{ - Func: nil, - Args: []Interpolation{ - &LiteralInterpolation{ - Literal: ",", - }, - &VariableInterpolation{ - Variable: &ResourceVariable{ - Type: "foo", - Name: "bar", - Field: "id", - Multi: true, - Index: -1, - key: "foo.bar.*.id", - }, - }, - }, - }, + Result: []string{ + "Call(join, Literal(TypeString, ,), Variable(foo.bar.*.id))", }, }, @@ -134,27 +79,16 @@ func TestInterpolationWalker_detect(t *testing.T) { Input: map[string]interface{}{ "foo": `${concat("localhost", ":8080")}`, }, - Result: []Interpolation{ - &FunctionInterpolation{ - Func: nil, - Args: []Interpolation{ - &LiteralInterpolation{ - Literal: "localhost", - }, - &LiteralInterpolation{ - Literal: ":8080", - }, - }, - }, + Result: []string{ + "Call(concat, Literal(TypeString, localhost), Literal(TypeString, :8080))", }, }, } for i, tc := range cases { - var actual []Interpolation - - detectFn := func(i Interpolation) (string, error) { - actual = append(actual, i) + var actual []string + detectFn := func(root ast.Node) (string, error) { + actual = append(actual, fmt.Sprintf("%s", root)) return "", nil } @@ -163,14 +97,6 @@ func TestInterpolationWalker_detect(t *testing.T) { t.Fatalf("err: %s", err) } - for _, a := range actual { - // This is jank, but reflect.DeepEqual never has functions - // being the same. - if f, ok := a.(*FunctionInterpolation); ok { - f.Func = nil - } - } - if !reflect.DeepEqual(actual, tc.Result) { t.Fatalf("%d: bad:\n\n%#v", i, actual) } @@ -198,7 +124,7 @@ func TestInterpolationWalker_replace(t *testing.T) { "foo": "hello, ${var.foo}", }, Output: map[string]interface{}{ - "foo": "hello, bar", + "foo": "bar", }, Value: "bar", }, @@ -247,7 +173,7 @@ func TestInterpolationWalker_replace(t *testing.T) { } for i, tc := range cases { - fn := func(i Interpolation) (string, error) { + fn := func(ast.Node) (string, error) { return tc.Value, nil } diff --git a/config/lang/ast/ast.go b/config/lang/ast/ast.go new file mode 100644 index 000000000..31951621f --- /dev/null +++ b/config/lang/ast/ast.go @@ -0,0 +1,43 @@ +package ast + +import ( + "fmt" +) + +// Node is the interface that all AST nodes must implement. +type Node interface { + // Accept is called to dispatch to the visitors. + Accept(Visitor) + + // Pos returns the position of this node in some source. + Pos() Pos +} + +// Pos is the starting position of an AST node +type Pos struct { + Column, Line int // Column/Line number, starting at 1 +} + +func (p Pos) String() string { + return fmt.Sprintf("%d:%d", p.Line, p.Column) +} + +// Visitors are just implementations of this function. +// +// Note that this isn't a true implementation of the visitor pattern, which +// generally requires proper type dispatch on the function. However, +// implementing this basic visitor pattern style is still very useful even +// if you have to type switch. +type Visitor func(Node) + +//go:generate stringer -type=Type + +// Type is the type of a literal. +type Type uint32 + +const ( + TypeInvalid Type = 0 + TypeString Type = 1 << iota + TypeInt + TypeFloat +) diff --git a/config/lang/ast/call.go b/config/lang/ast/call.go new file mode 100644 index 000000000..40b0773e1 --- /dev/null +++ b/config/lang/ast/call.go @@ -0,0 +1,34 @@ +package ast + +import ( + "fmt" + "strings" +) + +// Call represents a function call. +type Call struct { + Func string + Args []Node + Posx Pos +} + +func (n *Call) Accept(v Visitor) { + for _, a := range n.Args { + a.Accept(v) + } + + v(n) +} + +func (n *Call) Pos() Pos { + return n.Posx +} + +func (n *Call) String() string { + args := make([]string, len(n.Args)) + for i, arg := range n.Args { + args[i] = fmt.Sprintf("%s", arg) + } + + return fmt.Sprintf("Call(%s, %s)", n.Func, strings.Join(args, ", ")) +} diff --git a/config/lang/ast/concat.go b/config/lang/ast/concat.go new file mode 100644 index 000000000..9d3d998ee --- /dev/null +++ b/config/lang/ast/concat.go @@ -0,0 +1,28 @@ +package ast + +import ( + "fmt" +) + +// Concat represents a node where the result of two or more expressions are +// concatenated. The result of all expressions must be a string. +type Concat struct { + Exprs []Node + Posx Pos +} + +func (n *Concat) Accept(v Visitor) { + for _, n := range n.Exprs { + n.Accept(v) + } + + v(n) +} + +func (n *Concat) Pos() Pos { + return n.Posx +} + +func (n *Concat) GoString() string { + return fmt.Sprintf("*%#v", *n) +} diff --git a/config/lang/ast/literal.go b/config/lang/ast/literal.go new file mode 100644 index 000000000..1fd7669ff --- /dev/null +++ b/config/lang/ast/literal.go @@ -0,0 +1,29 @@ +package ast + +import ( + "fmt" +) + +// LiteralNode represents a single literal value, such as "foo" or +// 42 or 3.14159. Based on the Type, the Value can be safely cast. +type LiteralNode struct { + Value interface{} + Type Type + Posx Pos +} + +func (n *LiteralNode) Accept(v Visitor) { + v(n) +} + +func (n *LiteralNode) Pos() Pos { + return n.Posx +} + +func (n *LiteralNode) GoString() string { + return fmt.Sprintf("*%#v", *n) +} + +func (n *LiteralNode) String() string { + return fmt.Sprintf("Literal(%s, %v)", n.Type, n.Value) +} diff --git a/config/lang/ast/type_string.go b/config/lang/ast/type_string.go new file mode 100644 index 000000000..fd0e9e355 --- /dev/null +++ b/config/lang/ast/type_string.go @@ -0,0 +1,34 @@ +// generated by stringer -type=Type; DO NOT EDIT + +package ast + +import "fmt" + +const ( + _Type_name_0 = "TypeInvalid" + _Type_name_1 = "TypeString" + _Type_name_2 = "TypeInt" + _Type_name_3 = "TypeFloat" +) + +var ( + _Type_index_0 = [...]uint8{0, 11} + _Type_index_1 = [...]uint8{0, 10} + _Type_index_2 = [...]uint8{0, 7} + _Type_index_3 = [...]uint8{0, 9} +) + +func (i Type) String() string { + switch { + case i == 0: + return _Type_name_0 + case i == 2: + return _Type_name_1 + case i == 4: + return _Type_name_2 + case i == 8: + return _Type_name_3 + default: + return fmt.Sprintf("Type(%d)", i) + } +} diff --git a/config/lang/ast/variable_access.go b/config/lang/ast/variable_access.go new file mode 100644 index 000000000..1f86a260d --- /dev/null +++ b/config/lang/ast/variable_access.go @@ -0,0 +1,27 @@ +package ast + +import ( + "fmt" +) + +// VariableAccess represents a variable access. +type VariableAccess struct { + Name string + Posx Pos +} + +func (n *VariableAccess) Accept(v Visitor) { + v(n) +} + +func (n *VariableAccess) Pos() Pos { + return n.Posx +} + +func (n *VariableAccess) GoString() string { + return fmt.Sprintf("*%#v", *n) +} + +func (n *VariableAccess) String() string { + return fmt.Sprintf("Variable(%s)", n.Name) +} diff --git a/config/lang/check_identifier.go b/config/lang/check_identifier.go new file mode 100644 index 000000000..2e467c098 --- /dev/null +++ b/config/lang/check_identifier.go @@ -0,0 +1,85 @@ +package lang + +import ( + "fmt" + "sync" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +// IdentifierCheck is a SemanticCheck that checks that all identifiers +// resolve properly and that the right number of arguments are passed +// to functions. +type IdentifierCheck struct { + Scope *Scope + + err error + lock sync.Mutex +} + +func (c *IdentifierCheck) Visit(root ast.Node) error { + c.lock.Lock() + defer c.lock.Unlock() + defer c.reset() + root.Accept(c.visit) + return c.err +} + +func (c *IdentifierCheck) visit(raw ast.Node) { + if c.err != nil { + return + } + + switch n := raw.(type) { + case *ast.Call: + c.visitCall(n) + case *ast.VariableAccess: + c.visitVariableAccess(n) + case *ast.Concat: + // Ignore + case *ast.LiteralNode: + // Ignore + default: + c.createErr(n, fmt.Sprintf("unknown node: %#v", raw)) + } +} + +func (c *IdentifierCheck) visitCall(n *ast.Call) { + // Look up the function in the map + function, ok := c.Scope.LookupFunc(n.Func) + if !ok { + c.createErr(n, fmt.Sprintf("unknown function called: %s", n.Func)) + return + } + + // Break up the args into what is variadic and what is required + args := n.Args + if function.Variadic && len(args) > len(function.ArgTypes) { + args = n.Args[:len(function.ArgTypes)] + } + + // Verify the number of arguments + if len(args) != len(function.ArgTypes) { + c.createErr(n, fmt.Sprintf( + "%s: expected %d arguments, got %d", + n.Func, len(function.ArgTypes), len(n.Args))) + return + } +} + +func (c *IdentifierCheck) visitVariableAccess(n *ast.VariableAccess) { + // Look up the variable in the map + if _, ok := c.Scope.LookupVar(n.Name); !ok { + c.createErr(n, fmt.Sprintf( + "unknown variable accessed: %s", n.Name)) + return + } +} + +func (c *IdentifierCheck) createErr(n ast.Node, str string) { + c.err = fmt.Errorf("%s: %s", n.Pos(), str) +} + +func (c *IdentifierCheck) reset() { + c.err = nil +} diff --git a/config/lang/check_identifier_test.go b/config/lang/check_identifier_test.go new file mode 100644 index 000000000..526128424 --- /dev/null +++ b/config/lang/check_identifier_test.go @@ -0,0 +1,141 @@ +package lang + +import ( + "testing" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +func TestIdentifierCheck(t *testing.T) { + cases := []struct { + Input string + Scope *Scope + Error bool + }{ + { + "foo", + &Scope{}, + false, + }, + + { + "foo ${bar} success", + &Scope{ + VarMap: map[string]Variable{ + "bar": Variable{ + Value: "baz", + Type: ast.TypeString, + }, + }, + }, + false, + }, + + { + "foo ${bar}", + &Scope{}, + true, + }, + + { + "foo ${rand()} success", + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, + + { + "foo ${rand()}", + &Scope{}, + true, + }, + + { + "foo ${rand(42)} ", + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + true, + }, + + { + "foo ${rand()} ", + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeInt, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, + + { + "foo ${rand(42)} ", + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeInt, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, + + { + "foo ${rand(\"foo\", 42)} ", + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ArgTypes: []ast.Type{ast.TypeString}, + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeInt, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, + } + + for _, tc := range cases { + node, err := Parse(tc.Input) + if err != nil { + t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) + } + + visitor := &IdentifierCheck{Scope: tc.Scope} + err = visitor.Visit(node) + if (err != nil) != tc.Error { + t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) + } + } +} diff --git a/config/lang/check_types.go b/config/lang/check_types.go new file mode 100644 index 000000000..5015781e8 --- /dev/null +++ b/config/lang/check_types.go @@ -0,0 +1,142 @@ +package lang + +import ( + "fmt" + "sync" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +// TypeCheck implements ast.Visitor for type checking an AST tree. +// It requires some configuration to look up the type of nodes. +type TypeCheck struct { + Scope *Scope + + stack []ast.Type + err error + lock sync.Mutex +} + +func (v *TypeCheck) Visit(root ast.Node) error { + v.lock.Lock() + defer v.lock.Unlock() + defer v.reset() + root.Accept(v.visit) + return v.err +} + +func (v *TypeCheck) visit(raw ast.Node) { + if v.err != nil { + return + } + + switch n := raw.(type) { + case *ast.Call: + v.visitCall(n) + case *ast.Concat: + v.visitConcat(n) + case *ast.LiteralNode: + v.visitLiteral(n) + case *ast.VariableAccess: + v.visitVariableAccess(n) + default: + v.createErr(n, fmt.Sprintf("unknown node: %#v", raw)) + } +} + +func (v *TypeCheck) visitCall(n *ast.Call) { + // Look up the function in the map + function, ok := v.Scope.LookupFunc(n.Func) + if !ok { + v.createErr(n, fmt.Sprintf("unknown function called: %s", n.Func)) + return + } + + // The arguments are on the stack in reverse order, so pop them off. + args := make([]ast.Type, len(n.Args)) + for i, _ := range n.Args { + args[len(n.Args)-1-i] = v.stackPop() + } + + // Verify the args + for i, expected := range function.ArgTypes { + if args[i] != expected { + v.createErr(n, fmt.Sprintf( + "%s: argument %d should be %s, got %s", + n.Func, i+1, expected, args[i])) + return + } + } + + // If we're variadic, then verify the types there + if function.Variadic { + args = args[len(function.ArgTypes):] + for i, t := range args { + if t != function.VariadicType { + v.createErr(n, fmt.Sprintf( + "%s: argument %d should be %s, got %s", + n.Func, i+len(function.ArgTypes), + function.VariadicType, t)) + return + } + } + } + + // Return type + v.stackPush(function.ReturnType) +} + +func (v *TypeCheck) visitConcat(n *ast.Concat) { + types := make([]ast.Type, len(n.Exprs)) + for i, _ := range n.Exprs { + types[len(n.Exprs)-1-i] = v.stackPop() + } + + // All concat args must be strings, so validate that + for i, t := range types { + if t != ast.TypeString { + v.createErr(n, fmt.Sprintf( + "argument %d must be a sting", n, i+1)) + return + } + } + + // This always results in type string + v.stackPush(ast.TypeString) +} + +func (v *TypeCheck) visitLiteral(n *ast.LiteralNode) { + v.stackPush(n.Type) +} + +func (v *TypeCheck) visitVariableAccess(n *ast.VariableAccess) { + // Look up the variable in the map + variable, ok := v.Scope.LookupVar(n.Name) + if !ok { + v.createErr(n, fmt.Sprintf( + "unknown variable accessed: %s", n.Name)) + return + } + + // Add the type to the stack + v.stackPush(variable.Type) +} + +func (v *TypeCheck) createErr(n ast.Node, str string) { + v.err = fmt.Errorf("%s: %s", n.Pos(), str) +} + +func (v *TypeCheck) reset() { + v.stack = nil + v.err = nil +} + +func (v *TypeCheck) stackPush(t ast.Type) { + v.stack = append(v.stack, t) +} + +func (v *TypeCheck) stackPop() ast.Type { + var x ast.Type + x, v.stack = v.stack[len(v.stack)-1], v.stack[:len(v.stack)-1] + return x +} diff --git a/config/lang/check_types_test.go b/config/lang/check_types_test.go new file mode 100644 index 000000000..f39ff8743 --- /dev/null +++ b/config/lang/check_types_test.go @@ -0,0 +1,176 @@ +package lang + +import ( + "testing" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +func TestTypeCheck(t *testing.T) { + cases := []struct { + Input string + Scope *Scope + Error bool + }{ + { + "foo", + &Scope{}, + false, + }, + + { + "foo ${bar}", + &Scope{ + VarMap: map[string]Variable{ + "bar": Variable{ + Value: "baz", + Type: ast.TypeString, + }, + }, + }, + false, + }, + + { + "foo ${rand()}", + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, + + { + `foo ${rand("42")}`, + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ArgTypes: []ast.Type{ast.TypeString}, + ReturnType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, + + { + `foo ${rand(42)}`, + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ArgTypes: []ast.Type{ast.TypeString}, + ReturnType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + true, + }, + + { + `foo ${rand()}`, + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ArgTypes: nil, + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, + + { + `foo ${rand("42")}`, + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ArgTypes: nil, + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, + + { + `foo ${rand("42", 42)}`, + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ArgTypes: nil, + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + true, + }, + + { + "foo ${bar}", + &Scope{ + VarMap: map[string]Variable{ + "bar": Variable{ + Value: 42, + Type: ast.TypeInt, + }, + }, + }, + true, + }, + + { + "foo ${rand()}", + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeInt, + Callback: func([]interface{}) (interface{}, error) { + return 42, nil + }, + }, + }, + }, + true, + }, + } + + for _, tc := range cases { + node, err := Parse(tc.Input) + if err != nil { + t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) + } + + visitor := &TypeCheck{Scope: tc.Scope} + err = visitor.Visit(node) + if (err != nil) != tc.Error { + t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) + } + } +} diff --git a/config/lang/engine.go b/config/lang/engine.go new file mode 100644 index 000000000..594048ed9 --- /dev/null +++ b/config/lang/engine.go @@ -0,0 +1,259 @@ +package lang + +import ( + "bytes" + "fmt" + "sync" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +// Engine is the execution engine for this language. It should be configured +// prior to running Execute. +type Engine struct { + // GlobalScope is the global scope of execution for this engine. + GlobalScope *Scope + + // SemanticChecks is a list of additional semantic checks that will be run + // on the tree prior to executing it. The type checker, identifier checker, + // etc. will be run before these. + SemanticChecks []SemanticChecker +} + +// SemanticChecker is the type that must be implemented to do a +// semantic check on an AST tree. This will be called with the root node. +type SemanticChecker func(ast.Node) error + +// Execute executes the given ast.Node and returns its final value, its +// type, and an error if one exists. +func (e *Engine) Execute(root ast.Node) (interface{}, ast.Type, error) { + // Build our own semantic checks that we always run + tv := &TypeCheck{Scope: e.GlobalScope} + ic := &IdentifierCheck{Scope: e.GlobalScope} + + // Build up the semantic checks for execution + checks := make( + []SemanticChecker, len(e.SemanticChecks), len(e.SemanticChecks)+2) + copy(checks, e.SemanticChecks) + checks = append(checks, ic.Visit) + checks = append(checks, tv.Visit) + + // Run the semantic checks + for _, check := range checks { + if err := check(root); err != nil { + return nil, ast.TypeInvalid, err + } + } + + // Execute + v := &executeVisitor{Scope: e.GlobalScope} + return v.Visit(root) +} + +// executeVisitor is the visitor used to do the actual execution of +// a program. Note at this point it is assumed that the types check out +// and the identifiers exist. +type executeVisitor struct { + Scope *Scope + + stack EngineStack + err error + lock sync.Mutex +} + +func (v *executeVisitor) Visit(root ast.Node) (interface{}, ast.Type, error) { + v.lock.Lock() + defer v.lock.Unlock() + + // Run the actual visitor pattern + root.Accept(v.visit) + + // Get our result and clear out everything else + var result *ast.LiteralNode + if v.stack.Len() > 0 { + result = v.stack.Pop() + } else { + result = new(ast.LiteralNode) + } + resultErr := v.err + + // Clear everything else so we aren't just dangling + v.stack.Reset() + v.err = nil + + return result.Value, result.Type, resultErr +} + +func (v *executeVisitor) visit(raw ast.Node) { + if v.err != nil { + return + } + + switch n := raw.(type) { + case *ast.Call: + v.visitCall(n) + case *ast.Concat: + v.visitConcat(n) + case *ast.LiteralNode: + v.visitLiteral(n) + case *ast.VariableAccess: + v.visitVariableAccess(n) + default: + v.err = fmt.Errorf("unknown node: %#v", raw) + } +} + +func (v *executeVisitor) visitCall(n *ast.Call) { + // Look up the function in the map + function, ok := v.Scope.LookupFunc(n.Func) + if !ok { + v.err = fmt.Errorf("unknown function called: %s", n.Func) + return + } + + // The arguments are on the stack in reverse order, so pop them off. + args := make([]interface{}, len(n.Args)) + for i, _ := range n.Args { + node := v.stack.Pop() + args[len(n.Args)-1-i] = node.Value + } + + // Call the function + result, err := function.Callback(args) + if err != nil { + v.err = fmt.Errorf("%s: %s", n.Func, err) + return + } + + // Push the result + v.stack.Push(&ast.LiteralNode{ + Value: result, + Type: function.ReturnType, + }) +} + +func (v *executeVisitor) visitConcat(n *ast.Concat) { + // The expressions should all be on the stack in reverse + // order. So pop them off, reverse their order, and concatenate. + nodes := make([]*ast.LiteralNode, 0, len(n.Exprs)) + for range n.Exprs { + nodes = append(nodes, v.stack.Pop()) + } + + var buf bytes.Buffer + for i := len(nodes) - 1; i >= 0; i-- { + buf.WriteString(nodes[i].Value.(string)) + } + + v.stack.Push(&ast.LiteralNode{ + Value: buf.String(), + Type: ast.TypeString, + }) +} + +func (v *executeVisitor) visitLiteral(n *ast.LiteralNode) { + v.stack.Push(n) +} + +func (v *executeVisitor) visitVariableAccess(n *ast.VariableAccess) { + // Look up the variable in the map + variable, ok := v.Scope.LookupVar(n.Name) + if !ok { + v.err = fmt.Errorf("unknown variable accessed: %s", n.Name) + return + } + + v.stack.Push(&ast.LiteralNode{ + Value: variable.Value, + Type: variable.Type, + }) +} + +// EngineStack is a stack of ast.LiteralNodes that the Engine keeps track +// of during execution. This is currently backed by a dumb slice, but can be +// replaced with a better data structure at some point in the future if this +// turns out to require optimization. +type EngineStack struct { + stack []*ast.LiteralNode +} + +func (s *EngineStack) Len() int { + return len(s.stack) +} + +func (s *EngineStack) Push(n *ast.LiteralNode) { + s.stack = append(s.stack, n) +} + +func (s *EngineStack) Pop() *ast.LiteralNode { + x := s.stack[len(s.stack)-1] + s.stack[len(s.stack)-1] = nil + s.stack = s.stack[:len(s.stack)-1] + return x +} + +func (s *EngineStack) Reset() { + s.stack = nil +} + +// Scope represents a lookup scope for execution. +type Scope struct { + // VarMap and FuncMap are the mappings of identifiers to functions + // and variable values. + VarMap map[string]Variable + FuncMap map[string]Function +} + +// Variable is a variable value for execution given as input to the engine. +// It records the value of a variables along with their type. +type Variable struct { + Value interface{} + Type ast.Type +} + +// Function defines a function that can be executed by the engine. +// The type checker will validate that the proper types will be called +// to the callback. +type Function struct { + // ArgTypes is the list of types in argument order. These are the + // required arguments. + // + // ReturnType is the type of the returned value. The Callback MUST + // return this type. + ArgTypes []ast.Type + ReturnType ast.Type + + // Variadic, if true, says that this function is variadic, meaning + // it takes a variable number of arguments. In this case, the + // VariadicType must be set. + Variadic bool + VariadicType ast.Type + + // Callback is the function called for a function. The argument + // types are guaranteed to match the spec above by the type checker. + // The length of the args is strictly == len(ArgTypes) unless Varidiac + // is true, in which case its >= len(ArgTypes). + Callback func([]interface{}) (interface{}, error) +} + +// LookupFunc will look up a variable by name. +// TODO test +func (s *Scope) LookupFunc(n string) (Function, bool) { + if s == nil { + return Function{}, false + } + + v, ok := s.FuncMap[n] + return v, ok +} + +// LookupVar will look up a variable by name. +// TODO test +func (s *Scope) LookupVar(n string) (Variable, bool) { + if s == nil { + return Variable{}, false + } + + v, ok := s.VarMap[n] + return v, ok +} diff --git a/config/lang/engine_test.go b/config/lang/engine_test.go new file mode 100644 index 000000000..e5a840cbc --- /dev/null +++ b/config/lang/engine_test.go @@ -0,0 +1,100 @@ +package lang + +import ( + "reflect" + "testing" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +func TestEngineExecute(t *testing.T) { + cases := []struct { + Input string + Scope *Scope + Error bool + Result interface{} + ResultType ast.Type + }{ + { + "foo", + nil, + false, + "foo", + ast.TypeString, + }, + + { + "foo ${bar}", + &Scope{ + VarMap: map[string]Variable{ + "bar": Variable{ + Value: "baz", + Type: ast.TypeString, + }, + }, + }, + false, + "foo baz", + ast.TypeString, + }, + + { + "foo ${rand()}", + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + "foo 42", + ast.TypeString, + }, + + { + `foo ${rand("foo", "bar")}`, + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeString, + Callback: func(args []interface{}) (interface{}, error) { + var result string + for _, a := range args { + result += a.(string) + } + return result, nil + }, + }, + }, + }, + false, + "foo foobar", + ast.TypeString, + }, + } + + for _, tc := range cases { + node, err := Parse(tc.Input) + if err != nil { + t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) + } + + engine := &Engine{GlobalScope: tc.Scope} + out, outType, err := engine.Execute(node) + if (err != nil) != tc.Error { + t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) + } + if outType != tc.ResultType { + t.Fatalf("Bad: %s\n\nInput: %s", outType, tc.Input) + } + if !reflect.DeepEqual(out, tc.Result) { + t.Fatalf("Bad: %#v\n\nInput: %s", out, tc.Input) + } + } +} diff --git a/config/lang/lang.y b/config/lang/lang.y new file mode 100644 index 000000000..310af6128 --- /dev/null +++ b/config/lang/lang.y @@ -0,0 +1,134 @@ +// This is the yacc input for creating the parser for interpolation +// expressions in Go. To build it, just run `go generate` on this +// package, as the lexer has the go generate pragma within it. + +%{ +package lang + +import ( + "github.com/hashicorp/terraform/config/lang/ast" +) + +%} + +%union { + node ast.Node + nodeList []ast.Node + str string + token *parserToken +} + +%token PROGRAM_BRACKET_LEFT PROGRAM_BRACKET_RIGHT +%token PROGRAM_STRING_START PROGRAM_STRING_END +%token PAREN_LEFT PAREN_RIGHT COMMA + +%token IDENTIFIER INTEGER FLOAT STRING + +%type expr interpolation literal literalModeTop literalModeValue +%type args + +%% + +top: + { + parserResult = &ast.LiteralNode{ + Value: "", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + } + } +| literalModeTop + { + parserResult = $1 + } + +literalModeTop: + literalModeValue + { + $$ = $1 + } +| literalModeTop literalModeValue + { + var result []ast.Node + if c, ok := $1.(*ast.Concat); ok { + result = append(c.Exprs, $2) + } else { + result = []ast.Node{$1, $2} + } + + $$ = &ast.Concat{ + Exprs: result, + Posx: result[0].Pos(), + } + } + +literalModeValue: + literal + { + $$ = $1 + } +| interpolation + { + $$ = $1 + } + +interpolation: + PROGRAM_BRACKET_LEFT expr PROGRAM_BRACKET_RIGHT + { + $$ = $2 + } + +expr: + literalModeTop + { + $$ = $1 + } +| INTEGER + { + $$ = &ast.LiteralNode{ + Value: $1.Value.(int), + Type: ast.TypeInt, + Posx: $1.Pos, + } + } +| FLOAT + { + $$ = &ast.LiteralNode{ + Value: $1.Value.(float64), + Type: ast.TypeFloat, + Posx: $1.Pos, + } + } +| IDENTIFIER + { + $$ = &ast.VariableAccess{Name: $1.Value.(string), Posx: $1.Pos} + } +| IDENTIFIER PAREN_LEFT args PAREN_RIGHT + { + $$ = &ast.Call{Func: $1.Value.(string), Args: $3, Posx: $1.Pos} + } + +args: + { + $$ = nil + } +| args COMMA expr + { + $$ = append($1, $3) + } +| expr + { + $$ = append($$, $1) + } + +literal: + STRING + { + $$ = &ast.LiteralNode{ + Value: $1.Value.(string), + Type: ast.TypeString, + Posx: $1.Pos, + } + } + +%% diff --git a/config/lang/lex.go b/config/lang/lex.go new file mode 100644 index 000000000..ed1f8fb64 --- /dev/null +++ b/config/lang/lex.go @@ -0,0 +1,380 @@ +package lang + +import ( + "bytes" + "fmt" + "strconv" + "unicode" + "unicode/utf8" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +//go:generate go tool yacc -p parser lang.y + +// The parser expects the lexer to return 0 on EOF. +const lexEOF = 0 + +// The parser uses the type Lex as a lexer. It must provide +// the methods Lex(*SymType) int and Error(string). +type parserLex struct { + Err error + Input string + + mode parserMode + interpolationDepth int + pos int + width int + col, line int + lastLine int + astPos *ast.Pos +} + +// parserToken is the token yielded to the parser. The value can be +// determined within the parser type based on the enum value returned +// from Lex. +type parserToken struct { + Value interface{} + Pos ast.Pos +} + +// parserMode keeps track of what mode we're in for the parser. We have +// two modes: literal and interpolation. Literal mode is when strings +// don't have to be quoted, and interpolations are defined as ${foo}. +// Interpolation mode means that strings have to be quoted and unquoted +// things are identifiers, such as foo("bar"). +type parserMode uint8 + +const ( + parserModeInvalid parserMode = 0 + parserModeLiteral = 1 << iota + parserModeInterpolation +) + +// The parser calls this method to get each new token. +func (x *parserLex) Lex(yylval *parserSymType) int { + // We always start in literal mode, since programs don't start + // in an interpolation. ex. "foo ${bar}" vs "bar" (and assuming interp.) + if x.mode == parserModeInvalid { + x.mode = parserModeLiteral + } + + // Defer an update to set the proper column/line we read the next token. + defer func() { + if yylval.token != nil && yylval.token.Pos.Column == 0 { + yylval.token.Pos = *x.astPos + } + }() + + x.astPos = nil + return x.lex(yylval) +} + +func (x *parserLex) lex(yylval *parserSymType) int { + switch x.mode { + case parserModeLiteral: + return x.lexModeLiteral(yylval) + case parserModeInterpolation: + return x.lexModeInterpolation(yylval) + default: + x.Error(fmt.Sprintf("Unknown parse mode: %s", x.mode)) + return lexEOF + } +} + +func (x *parserLex) lexModeLiteral(yylval *parserSymType) int { + for { + c := x.next() + if c == lexEOF { + return lexEOF + } + + // Are we starting an interpolation? + if c == '$' && x.peek() == '{' { + x.next() + x.interpolationDepth++ + x.mode = parserModeInterpolation + return PROGRAM_BRACKET_LEFT + } + + // We're just a normal string that isn't part of any interpolation yet. + x.backup() + result, terminated := x.lexString(yylval, x.interpolationDepth > 0) + + // If the string terminated and we're within an interpolation already + // then that means that we finished a nested string, so pop + // back out to interpolation mode. + if terminated && x.interpolationDepth > 0 { + x.mode = parserModeInterpolation + + // If the string is empty, just skip it. We're still in + // an interpolation so we do this to avoid empty nodes. + if yylval.token.Value.(string) == "" { + return x.lex(yylval) + } + } + + return result + } +} + +func (x *parserLex) lexModeInterpolation(yylval *parserSymType) int { + for { + c := x.next() + if c == lexEOF { + return lexEOF + } + + // Ignore all whitespace + if unicode.IsSpace(c) { + continue + } + + // If we see a double quote then we're lexing a string since + // we're in interpolation mode. + if c == '"' { + result, terminated := x.lexString(yylval, true) + if !terminated { + // The string didn't end, which means that we're in the + // middle of starting another interpolation. + x.mode = parserModeLiteral + + // If the string is empty and we're starting an interpolation, + // then just skip it to avoid empty string AST nodes + if yylval.token.Value.(string) == "" { + return x.lex(yylval) + } + } + + return result + } + + // If we are seeing a number, it is the start of a number. Lex it. + if c >= '0' && c <= '9' { + x.backup() + return x.lexNumber(yylval) + } + + switch c { + case '}': + // '}' means we ended the interpolation. Pop back into + // literal mode and reduce our interpolation depth. + x.interpolationDepth-- + x.mode = parserModeLiteral + return PROGRAM_BRACKET_RIGHT + case '(': + return PAREN_LEFT + case ')': + return PAREN_RIGHT + case ',': + return COMMA + default: + x.backup() + return x.lexId(yylval) + } + } +} + +func (x *parserLex) lexId(yylval *parserSymType) int { + var b bytes.Buffer + for { + c := x.next() + if c == lexEOF { + break + } + + // If this isn't a character we want in an ID, return out. + // One day we should make this a regexp. + if c != '_' && + c != '-' && + c != '.' && + c != '*' && + !unicode.IsLetter(c) && + !unicode.IsNumber(c) { + x.backup() + break + } + + if _, err := b.WriteRune(c); err != nil { + x.Error(err.Error()) + return lexEOF + } + } + + yylval.token = &parserToken{Value: b.String()} + return IDENTIFIER +} + +// lexNumber lexes out a number: an integer or a float. +func (x *parserLex) lexNumber(yylval *parserSymType) int { + var b bytes.Buffer + gotPeriod := false + for { + c := x.next() + if c == lexEOF { + break + } + + // If we see a period, we might be getting a float.. + if c == '.' { + // If we've already seen a period, then ignore it, and + // exit. This will probably result in a syntax error later. + if gotPeriod { + x.backup() + break + } + + gotPeriod = true + } else if c < '0' || c > '9' { + // If we're not seeing a number, then also exit. + x.backup() + break + } + + if _, err := b.WriteRune(c); err != nil { + x.Error(fmt.Sprintf("internal error: %s", err)) + return lexEOF + } + } + + // If we didn't see a period, it is an int + if !gotPeriod { + v, err := strconv.ParseInt(b.String(), 0, 0) + if err != nil { + x.Error(fmt.Sprintf("expected number: %s", err)) + return lexEOF + } + + yylval.token = &parserToken{Value: int(v)} + return INTEGER + } + + // If we did see a period, it is a float + f, err := strconv.ParseFloat(b.String(), 64) + if err != nil { + x.Error(fmt.Sprintf("expected float: %s", err)) + return lexEOF + } + + yylval.token = &parserToken{Value: f} + return FLOAT +} + +func (x *parserLex) lexString(yylval *parserSymType, quoted bool) (int, bool) { + var b bytes.Buffer + terminated := false + for { + c := x.next() + if c == lexEOF { + if quoted { + x.Error("unterminated string") + } + + break + } + + // Behavior is a bit different if we're lexing within a quoted string. + if quoted { + // If its a double quote, we've reached the end of the string + if c == '"' { + terminated = true + break + } + + // Let's check to see if we're escaping anything. + if c == '\\' { + switch n := x.next(); n { + case '\\': + fallthrough + case '"': + c = n + case 'n': + c = '\n' + default: + x.backup() + } + } + } + + // If we hit a dollar sign, then check if we're starting + // another interpolation. If so, then we're done. + if c == '$' { + n := x.peek() + + // If it is '{', then we're starting another interpolation + if n == '{' { + x.backup() + break + } + + // If it is '$', then we're escaping a dollar sign + if n == '$' { + x.next() + } + } + + if _, err := b.WriteRune(c); err != nil { + x.Error(err.Error()) + return lexEOF, false + } + } + + yylval.token = &parserToken{Value: b.String()} + return STRING, terminated +} + +// Return the next rune for the lexer. +func (x *parserLex) next() rune { + if int(x.pos) >= len(x.Input) { + x.width = 0 + return lexEOF + } + + r, w := utf8.DecodeRuneInString(x.Input[x.pos:]) + x.width = w + x.pos += x.width + + if x.line == 0 { + x.line = 1 + x.col = 1 + } else { + x.col += 1 + } + + if r == '\n' { + x.lastLine = x.col + x.line += 1 + x.col = 1 + } + + if x.astPos == nil { + x.astPos = &ast.Pos{Column: x.col, Line: x.line} + } + + return r +} + +// peek returns but does not consume the next rune in the input +func (x *parserLex) peek() rune { + r := x.next() + x.backup() + return r +} + +// backup steps back one rune. Can only be called once per next. +func (x *parserLex) backup() { + x.pos -= x.width + x.col -= 1 + + // If we are at column 0, we're backing up across a line boundary + // so we need to be careful to get the proper value. + if x.col == 0 { + x.col = x.lastLine + x.line -= 1 + } +} + +// The parser calls this method on a parse error. +func (x *parserLex) Error(s string) { + x.Err = fmt.Errorf("parse error: %s", s) +} diff --git a/config/lang/lex_test.go b/config/lang/lex_test.go new file mode 100644 index 000000000..8c5207d78 --- /dev/null +++ b/config/lang/lex_test.go @@ -0,0 +1,131 @@ +package lang + +import ( + "reflect" + "testing" +) + +func TestLex(t *testing.T) { + cases := []struct { + Input string + Output []int + }{ + { + "foo", + []int{STRING, lexEOF}, + }, + + { + "foo$bar", + []int{STRING, lexEOF}, + }, + + { + "foo ${bar}", + []int{STRING, PROGRAM_BRACKET_LEFT, IDENTIFIER, PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + "foo $${bar}", + []int{STRING, lexEOF}, + }, + + { + "foo $$$${bar}", + []int{STRING, lexEOF}, + }, + + { + "foo ${\"bar\"}", + []int{STRING, PROGRAM_BRACKET_LEFT, STRING, PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + "${bar(baz)}", + []int{PROGRAM_BRACKET_LEFT, + IDENTIFIER, PAREN_LEFT, IDENTIFIER, PAREN_RIGHT, + PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + "${bar(baz, foo)}", + []int{PROGRAM_BRACKET_LEFT, + IDENTIFIER, PAREN_LEFT, + IDENTIFIER, COMMA, IDENTIFIER, + PAREN_RIGHT, + PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + "${bar(42)}", + []int{PROGRAM_BRACKET_LEFT, + IDENTIFIER, PAREN_LEFT, INTEGER, PAREN_RIGHT, + PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + "${bar(3.14159)}", + []int{PROGRAM_BRACKET_LEFT, + IDENTIFIER, PAREN_LEFT, FLOAT, PAREN_RIGHT, + PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + "${bar(inner(baz))}", + []int{PROGRAM_BRACKET_LEFT, + IDENTIFIER, PAREN_LEFT, + IDENTIFIER, PAREN_LEFT, + IDENTIFIER, + PAREN_RIGHT, PAREN_RIGHT, + PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + "foo ${foo.bar.baz}", + []int{STRING, PROGRAM_BRACKET_LEFT, IDENTIFIER, PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + "foo ${foo.bar.*.baz}", + []int{STRING, PROGRAM_BRACKET_LEFT, IDENTIFIER, PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + "foo ${foo(\"baz\")}", + []int{STRING, PROGRAM_BRACKET_LEFT, + IDENTIFIER, PAREN_LEFT, STRING, PAREN_RIGHT, + PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + `foo ${"${var.foo}"}`, + []int{STRING, PROGRAM_BRACKET_LEFT, + PROGRAM_BRACKET_LEFT, IDENTIFIER, PROGRAM_BRACKET_RIGHT, + PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + } + + for _, tc := range cases { + l := &parserLex{Input: tc.Input} + var actual []int + for { + token := l.Lex(new(parserSymType)) + actual = append(actual, token) + + if token == lexEOF { + break + } + + // Be careful against what are probably infinite loops + if len(actual) > 100 { + t.Fatalf("Input:%s\n\nExausted.", tc.Input) + } + } + + if !reflect.DeepEqual(actual, tc.Output) { + t.Fatalf( + "Input: %s\n\nBad: %#v\n\nExpected: %#v", + tc.Input, actual, tc.Output) + } + } +} diff --git a/config/lang/parse.go b/config/lang/parse.go new file mode 100644 index 000000000..8ece590af --- /dev/null +++ b/config/lang/parse.go @@ -0,0 +1,32 @@ +package lang + +import ( + "sync" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +var parserErrors []error +var parserLock sync.Mutex +var parserResult ast.Node + +// Parse parses the given program and returns an executable AST tree. +func Parse(v string) (ast.Node, error) { + // Unfortunately due to the way that goyacc generated parsers are + // formatted, we can only do a single parse at a time without a lot + // of extra work. In the future we can remove this limitation. + parserLock.Lock() + defer parserLock.Unlock() + + // Reset our globals + parserErrors = nil + parserResult = nil + + // Create the lexer + lex := &parserLex{Input: v} + + // Parse! + parserParse(lex) + + return parserResult, lex.Err +} diff --git a/config/lang/parse_test.go b/config/lang/parse_test.go new file mode 100644 index 000000000..05b37ac18 --- /dev/null +++ b/config/lang/parse_test.go @@ -0,0 +1,265 @@ +package lang + +import ( + "reflect" + "testing" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +func TestParse(t *testing.T) { + cases := []struct { + Input string + Error bool + Result ast.Node + }{ + { + "", + false, + &ast.LiteralNode{ + Value: "", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + }, + }, + + { + "foo", + false, + &ast.LiteralNode{ + Value: "foo", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + }, + }, + + { + "$${var.foo}", + false, + &ast.LiteralNode{ + Value: "${var.foo}", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + }, + }, + + { + "foo ${var.bar}", + false, + &ast.Concat{ + Posx: ast.Pos{Column: 1, Line: 1}, + Exprs: []ast.Node{ + &ast.LiteralNode{ + Value: "foo ", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + }, + &ast.VariableAccess{ + Name: "var.bar", + Posx: ast.Pos{Column: 7, Line: 1}, + }, + }, + }, + }, + + { + "foo ${var.bar} baz", + false, + &ast.Concat{ + Posx: ast.Pos{Column: 1, Line: 1}, + Exprs: []ast.Node{ + &ast.LiteralNode{ + Value: "foo ", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + }, + &ast.VariableAccess{ + Name: "var.bar", + Posx: ast.Pos{Column: 7, Line: 1}, + }, + &ast.LiteralNode{ + Value: " baz", + Type: ast.TypeString, + Posx: ast.Pos{Column: 15, Line: 1}, + }, + }, + }, + }, + + { + "foo ${\"bar\"}", + false, + &ast.Concat{ + Posx: ast.Pos{Column: 1, Line: 1}, + Exprs: []ast.Node{ + &ast.LiteralNode{ + Value: "foo ", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + }, + &ast.LiteralNode{ + Value: "bar", + Type: ast.TypeString, + Posx: ast.Pos{Column: 7, Line: 1}, + }, + }, + }, + }, + + { + "foo ${42}", + false, + &ast.Concat{ + Posx: ast.Pos{Column: 1, Line: 1}, + Exprs: []ast.Node{ + &ast.LiteralNode{ + Value: "foo ", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + }, + &ast.LiteralNode{ + Value: 42, + Type: ast.TypeInt, + Posx: ast.Pos{Column: 7, Line: 1}, + }, + }, + }, + }, + + { + "foo ${3.14159}", + false, + &ast.Concat{ + Posx: ast.Pos{Column: 1, Line: 1}, + Exprs: []ast.Node{ + &ast.LiteralNode{ + Value: "foo ", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + }, + &ast.LiteralNode{ + Value: 3.14159, + Type: ast.TypeFloat, + Posx: ast.Pos{Column: 7, Line: 1}, + }, + }, + }, + }, + + { + "${foo()}", + false, + &ast.Call{ + Func: "foo", + Args: nil, + Posx: ast.Pos{Column: 3, Line: 1}, + }, + }, + + { + "${foo(bar)}", + false, + &ast.Call{ + Func: "foo", + Posx: ast.Pos{Column: 3, Line: 1}, + Args: []ast.Node{ + &ast.VariableAccess{ + Name: "bar", + Posx: ast.Pos{Column: 7, Line: 1}, + }, + }, + }, + }, + + { + "${foo(bar, baz)}", + false, + &ast.Call{ + Func: "foo", + Posx: ast.Pos{Column: 3, Line: 1}, + Args: []ast.Node{ + &ast.VariableAccess{ + Name: "bar", + Posx: ast.Pos{Column: 7, Line: 1}, + }, + &ast.VariableAccess{ + Name: "baz", + Posx: ast.Pos{Column: 11, Line: 1}, + }, + }, + }, + }, + + { + "${foo(bar(baz))}", + false, + &ast.Call{ + Func: "foo", + Posx: ast.Pos{Column: 3, Line: 1}, + Args: []ast.Node{ + &ast.Call{ + Func: "bar", + Posx: ast.Pos{Column: 7, Line: 1}, + Args: []ast.Node{ + &ast.VariableAccess{ + Name: "baz", + Posx: ast.Pos{Column: 11, Line: 1}, + }, + }, + }, + }, + }, + }, + + { + `foo ${"bar ${baz}"}`, + false, + &ast.Concat{ + Posx: ast.Pos{Column: 1, Line: 1}, + Exprs: []ast.Node{ + &ast.LiteralNode{ + Value: "foo ", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + }, + &ast.Concat{ + Posx: ast.Pos{Column: 7, Line: 1}, + Exprs: []ast.Node{ + &ast.LiteralNode{ + Value: "bar ", + Type: ast.TypeString, + Posx: ast.Pos{Column: 7, Line: 1}, + }, + &ast.VariableAccess{ + Name: "baz", + Posx: ast.Pos{Column: 14, Line: 1}, + }, + }, + }, + }, + }, + }, + + { + `foo ${bar ${baz}}`, + true, + nil, + }, + + { + `foo ${${baz}}`, + true, + nil, + }, + } + + for _, tc := range cases { + actual, err := Parse(tc.Input) + if (err != nil) != tc.Error { + t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) + } + if !reflect.DeepEqual(actual, tc.Result) { + t.Fatalf("Bad: %#v\n\nInput: %s", actual, tc.Input) + } + } +} diff --git a/config/lang/token.go b/config/lang/token.go new file mode 100644 index 000000000..48e87f869 --- /dev/null +++ b/config/lang/token.go @@ -0,0 +1 @@ +package lang diff --git a/config/lang/transform_fixed.go b/config/lang/transform_fixed.go new file mode 100644 index 000000000..c551423df --- /dev/null +++ b/config/lang/transform_fixed.go @@ -0,0 +1,26 @@ +package lang + +import ( + "github.com/hashicorp/terraform/config/lang/ast" +) + +// FixedValueTransform transforms an AST to return a fixed value for +// all interpolations. i.e. you can make "hello ${anything}" always +// turn into "hello foo". +func FixedValueTransform(root ast.Node, Value *ast.LiteralNode) ast.Node { + // We visit the nodes in top-down order + result := root + switch n := result.(type) { + case *ast.Concat: + for i, v := range n.Exprs { + n.Exprs[i] = FixedValueTransform(v, Value) + } + case *ast.LiteralNode: + // We keep it as-is + default: + // Anything else we replace + result = Value + } + + return result +} diff --git a/config/lang/transform_fixed_test.go b/config/lang/transform_fixed_test.go new file mode 100644 index 000000000..c5e5a1ca3 --- /dev/null +++ b/config/lang/transform_fixed_test.go @@ -0,0 +1,48 @@ +package lang + +import ( + "reflect" + "testing" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +func TestFixedValueTransform(t *testing.T) { + cases := []struct { + Input ast.Node + Output ast.Node + }{ + { + &ast.LiteralNode{Value: 42}, + &ast.LiteralNode{Value: 42}, + }, + + { + &ast.VariableAccess{Name: "bar"}, + &ast.LiteralNode{Value: "foo"}, + }, + + { + &ast.Concat{ + Exprs: []ast.Node{ + &ast.VariableAccess{Name: "bar"}, + &ast.LiteralNode{Value: 42}, + }, + }, + &ast.Concat{ + Exprs: []ast.Node{ + &ast.LiteralNode{Value: "foo"}, + &ast.LiteralNode{Value: 42}, + }, + }, + }, + } + + value := &ast.LiteralNode{Value: "foo"} + for _, tc := range cases { + actual := FixedValueTransform(tc.Input, value) + if !reflect.DeepEqual(actual, tc.Output) { + t.Fatalf("bad: %#v\n\nInput: %#v", actual, tc.Input) + } + } +} diff --git a/config/lang/transform_implicit_types_test.go b/config/lang/transform_implicit_types_test.go new file mode 100644 index 000000000..9eb0fd92f --- /dev/null +++ b/config/lang/transform_implicit_types_test.go @@ -0,0 +1,74 @@ +package lang + +import ( + "testing" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +func TestLookupType(t *testing.T) { + cases := []struct { + Input ast.Node + Scope *Scope + Output ast.Type + Error bool + }{ + { + &customUntyped{}, + nil, + ast.TypeInvalid, + true, + }, + + { + &customTyped{}, + nil, + ast.TypeString, + false, + }, + + { + &ast.LiteralNode{ + Value: 42, + Type: ast.TypeInt, + }, + nil, + ast.TypeInt, + false, + }, + + { + &ast.VariableAccess{ + Name: "foo", + }, + &Scope{ + VarMap: map[string]Variable{ + "foo": Variable{Type: ast.TypeInt}, + }, + }, + ast.TypeInt, + false, + }, + } + + for _, tc := range cases { + actual, err := LookupType(tc.Input, tc.Scope) + if (err != nil) != tc.Error { + t.Fatalf("bad: %s\n\nInput: %#v", err, tc.Input) + } + if actual != tc.Output { + t.Fatalf("bad: %s\n\nInput: %#v", actual, tc.Input) + } + } +} + +type customUntyped struct{} + +func (n customUntyped) Accept(ast.Visitor) {} +func (n customUntyped) Pos() (v ast.Pos) { return } + +type customTyped struct{} + +func (n customTyped) Accept(ast.Visitor) {} +func (n customTyped) Pos() (v ast.Pos) { return } +func (n customTyped) Type(*Scope) (ast.Type, error) { return ast.TypeString, nil } diff --git a/config/lang/types.go b/config/lang/types.go new file mode 100644 index 000000000..8adfcd05e --- /dev/null +++ b/config/lang/types.go @@ -0,0 +1,53 @@ +package lang + +import ( + "fmt" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +// LookupType looks up the type of the given node with the given scope. +func LookupType(raw ast.Node, scope *Scope) (ast.Type, error) { + switch n := raw.(type) { + case *ast.LiteralNode: + return typedLiteralNode{n}.Type(scope) + case *ast.VariableAccess: + return typedVariableAccess{n}.Type(scope) + default: + if t, ok := raw.(TypedNode); ok { + return t.Type(scope) + } + + return ast.TypeInvalid, fmt.Errorf( + "unknown node to get type of: %T", raw) + } +} + +// TypedNode is an interface that custom AST nodes should implement +// if they want to work with LookupType. All the builtin AST nodes have +// implementations of this. +type TypedNode interface { + Type(*Scope) (ast.Type, error) +} + +type typedLiteralNode struct { + n *ast.LiteralNode +} + +func (n typedLiteralNode) Type(s *Scope) (ast.Type, error) { + return n.n.Type, nil +} + +type typedVariableAccess struct { + n *ast.VariableAccess +} + +func (n typedVariableAccess) Type(s *Scope) (ast.Type, error) { + v, ok := s.LookupVar(n.n.Name) + if !ok { + return ast.TypeInvalid, fmt.Errorf( + "%s: couldn't find variable %s", n.n.Pos(), n.n.Name) + } + + return v.Type, nil +} diff --git a/config/lang/y.go b/config/lang/y.go new file mode 100644 index 000000000..a6b139520 --- /dev/null +++ b/config/lang/y.go @@ -0,0 +1,452 @@ +//line lang.y:6 +package lang + +import __yyfmt__ "fmt" + +//line lang.y:6 +import ( + "github.com/hashicorp/terraform/config/lang/ast" +) + +//line lang.y:14 +type parserSymType struct { + yys int + node ast.Node + nodeList []ast.Node + str string + token *parserToken +} + +const PROGRAM_BRACKET_LEFT = 57346 +const PROGRAM_BRACKET_RIGHT = 57347 +const PROGRAM_STRING_START = 57348 +const PROGRAM_STRING_END = 57349 +const PAREN_LEFT = 57350 +const PAREN_RIGHT = 57351 +const COMMA = 57352 +const IDENTIFIER = 57353 +const INTEGER = 57354 +const FLOAT = 57355 +const STRING = 57356 + +var parserToknames = []string{ + "PROGRAM_BRACKET_LEFT", + "PROGRAM_BRACKET_RIGHT", + "PROGRAM_STRING_START", + "PROGRAM_STRING_END", + "PAREN_LEFT", + "PAREN_RIGHT", + "COMMA", + "IDENTIFIER", + "INTEGER", + "FLOAT", + "STRING", +} +var parserStatenames = []string{} + +const parserEofCode = 1 +const parserErrCode = 2 +const parserMaxDepth = 200 + +//line lang.y:134 + +//line yacctab:1 +var parserExca = []int{ + -1, 1, + 1, -1, + -2, 0, +} + +const parserNprod = 17 +const parserPrivate = 57344 + +var parserTokenNames []string +var parserStates []string + +const parserLast = 23 + +var parserAct = []int{ + + 9, 7, 7, 3, 18, 19, 8, 15, 13, 11, + 12, 6, 6, 14, 8, 1, 17, 10, 2, 16, + 20, 4, 5, +} +var parserPact = []int{ + + -2, -1000, -2, -1000, -1000, -1000, -1000, -3, -1000, 8, + -2, -1000, -1000, -1, -1000, -3, -5, -1000, -1000, -3, + -1000, +} +var parserPgo = []int{ + + 0, 0, 22, 21, 17, 3, 19, 15, +} +var parserR1 = []int{ + + 0, 7, 7, 4, 4, 5, 5, 2, 1, 1, + 1, 1, 1, 6, 6, 6, 3, +} +var parserR2 = []int{ + + 0, 0, 1, 1, 2, 1, 1, 3, 1, 1, + 1, 1, 4, 0, 3, 1, 1, +} +var parserChk = []int{ + + -1000, -7, -4, -5, -3, -2, 14, 4, -5, -1, + -4, 12, 13, 11, 5, 8, -6, -1, 9, 10, + -1, +} +var parserDef = []int{ + + 1, -2, 2, 3, 5, 6, 16, 0, 4, 0, + 8, 9, 10, 11, 7, 13, 0, 15, 12, 0, + 14, +} +var parserTok1 = []int{ + + 1, +} +var parserTok2 = []int{ + + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, +} +var parserTok3 = []int{ + 0, +} + +//line yaccpar:1 + +/* parser for yacc output */ + +var parserDebug = 0 + +type parserLexer interface { + Lex(lval *parserSymType) int + Error(s string) +} + +const parserFlag = -1000 + +func parserTokname(c int) string { + // 4 is TOKSTART above + if c >= 4 && c-4 < len(parserToknames) { + if parserToknames[c-4] != "" { + return parserToknames[c-4] + } + } + return __yyfmt__.Sprintf("tok-%v", c) +} + +func parserStatname(s int) string { + if s >= 0 && s < len(parserStatenames) { + if parserStatenames[s] != "" { + return parserStatenames[s] + } + } + return __yyfmt__.Sprintf("state-%v", s) +} + +func parserlex1(lex parserLexer, lval *parserSymType) int { + c := 0 + char := lex.Lex(lval) + if char <= 0 { + c = parserTok1[0] + goto out + } + if char < len(parserTok1) { + c = parserTok1[char] + goto out + } + if char >= parserPrivate { + if char < parserPrivate+len(parserTok2) { + c = parserTok2[char-parserPrivate] + goto out + } + } + for i := 0; i < len(parserTok3); i += 2 { + c = parserTok3[i+0] + if c == char { + c = parserTok3[i+1] + goto out + } + } + +out: + if c == 0 { + c = parserTok2[1] /* unknown char */ + } + if parserDebug >= 3 { + __yyfmt__.Printf("lex %s(%d)\n", parserTokname(c), uint(char)) + } + return c +} + +func parserParse(parserlex parserLexer) int { + var parsern int + var parserlval parserSymType + var parserVAL parserSymType + parserS := make([]parserSymType, parserMaxDepth) + + Nerrs := 0 /* number of errors */ + Errflag := 0 /* error recovery flag */ + parserstate := 0 + parserchar := -1 + parserp := -1 + goto parserstack + +ret0: + return 0 + +ret1: + return 1 + +parserstack: + /* put a state and value onto the stack */ + if parserDebug >= 4 { + __yyfmt__.Printf("char %v in %v\n", parserTokname(parserchar), parserStatname(parserstate)) + } + + parserp++ + if parserp >= len(parserS) { + nyys := make([]parserSymType, len(parserS)*2) + copy(nyys, parserS) + parserS = nyys + } + parserS[parserp] = parserVAL + parserS[parserp].yys = parserstate + +parsernewstate: + parsern = parserPact[parserstate] + if parsern <= parserFlag { + goto parserdefault /* simple state */ + } + if parserchar < 0 { + parserchar = parserlex1(parserlex, &parserlval) + } + parsern += parserchar + if parsern < 0 || parsern >= parserLast { + goto parserdefault + } + parsern = parserAct[parsern] + if parserChk[parsern] == parserchar { /* valid shift */ + parserchar = -1 + parserVAL = parserlval + parserstate = parsern + if Errflag > 0 { + Errflag-- + } + goto parserstack + } + +parserdefault: + /* default state action */ + parsern = parserDef[parserstate] + if parsern == -2 { + if parserchar < 0 { + parserchar = parserlex1(parserlex, &parserlval) + } + + /* look through exception table */ + xi := 0 + for { + if parserExca[xi+0] == -1 && parserExca[xi+1] == parserstate { + break + } + xi += 2 + } + for xi += 2; ; xi += 2 { + parsern = parserExca[xi+0] + if parsern < 0 || parsern == parserchar { + break + } + } + parsern = parserExca[xi+1] + if parsern < 0 { + goto ret0 + } + } + if parsern == 0 { + /* error ... attempt to resume parsing */ + switch Errflag { + case 0: /* brand new error */ + parserlex.Error("syntax error") + Nerrs++ + if parserDebug >= 1 { + __yyfmt__.Printf("%s", parserStatname(parserstate)) + __yyfmt__.Printf(" saw %s\n", parserTokname(parserchar)) + } + fallthrough + + case 1, 2: /* incompletely recovered error ... try again */ + Errflag = 3 + + /* find a state where "error" is a legal shift action */ + for parserp >= 0 { + parsern = parserPact[parserS[parserp].yys] + parserErrCode + if parsern >= 0 && parsern < parserLast { + parserstate = parserAct[parsern] /* simulate a shift of "error" */ + if parserChk[parserstate] == parserErrCode { + goto parserstack + } + } + + /* the current p has no shift on "error", pop stack */ + if parserDebug >= 2 { + __yyfmt__.Printf("error recovery pops state %d\n", parserS[parserp].yys) + } + parserp-- + } + /* there is no state on the stack with an error shift ... abort */ + goto ret1 + + case 3: /* no shift yet; clobber input char */ + if parserDebug >= 2 { + __yyfmt__.Printf("error recovery discards %s\n", parserTokname(parserchar)) + } + if parserchar == parserEofCode { + goto ret1 + } + parserchar = -1 + goto parsernewstate /* try again in the same state */ + } + } + + /* reduction by production parsern */ + if parserDebug >= 2 { + __yyfmt__.Printf("reduce %v in:\n\t%v\n", parsern, parserStatname(parserstate)) + } + + parsernt := parsern + parserpt := parserp + _ = parserpt // guard against "declared and not used" + + parserp -= parserR2[parsern] + parserVAL = parserS[parserp+1] + + /* consult goto table to find next state */ + parsern = parserR1[parsern] + parserg := parserPgo[parsern] + parserj := parserg + parserS[parserp].yys + 1 + + if parserj >= parserLast { + parserstate = parserAct[parserg] + } else { + parserstate = parserAct[parserj] + if parserChk[parserstate] != -parsern { + parserstate = parserAct[parserg] + } + } + // dummy call; replaced with literal code + switch parsernt { + + case 1: + //line lang.y:33 + { + parserResult = &ast.LiteralNode{ + Value: "", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + } + } + case 2: + //line lang.y:41 + { + parserResult = parserS[parserpt-0].node + } + case 3: + //line lang.y:47 + { + parserVAL.node = parserS[parserpt-0].node + } + case 4: + //line lang.y:51 + { + var result []ast.Node + if c, ok := parserS[parserpt-1].node.(*ast.Concat); ok { + result = append(c.Exprs, parserS[parserpt-0].node) + } else { + result = []ast.Node{parserS[parserpt-1].node, parserS[parserpt-0].node} + } + + parserVAL.node = &ast.Concat{ + Exprs: result, + Posx: result[0].Pos(), + } + } + case 5: + //line lang.y:67 + { + parserVAL.node = parserS[parserpt-0].node + } + case 6: + //line lang.y:71 + { + parserVAL.node = parserS[parserpt-0].node + } + case 7: + //line lang.y:77 + { + parserVAL.node = parserS[parserpt-1].node + } + case 8: + //line lang.y:83 + { + parserVAL.node = parserS[parserpt-0].node + } + case 9: + //line lang.y:87 + { + parserVAL.node = &ast.LiteralNode{ + Value: parserS[parserpt-0].token.Value.(int), + Type: ast.TypeInt, + Posx: parserS[parserpt-0].token.Pos, + } + } + case 10: + //line lang.y:95 + { + parserVAL.node = &ast.LiteralNode{ + Value: parserS[parserpt-0].token.Value.(float64), + Type: ast.TypeFloat, + Posx: parserS[parserpt-0].token.Pos, + } + } + case 11: + //line lang.y:103 + { + parserVAL.node = &ast.VariableAccess{Name: parserS[parserpt-0].token.Value.(string), Posx: parserS[parserpt-0].token.Pos} + } + case 12: + //line lang.y:107 + { + parserVAL.node = &ast.Call{Func: parserS[parserpt-3].token.Value.(string), Args: parserS[parserpt-1].nodeList, Posx: parserS[parserpt-3].token.Pos} + } + case 13: + //line lang.y:112 + { + parserVAL.nodeList = nil + } + case 14: + //line lang.y:116 + { + parserVAL.nodeList = append(parserS[parserpt-2].nodeList, parserS[parserpt-0].node) + } + case 15: + //line lang.y:120 + { + parserVAL.nodeList = append(parserVAL.nodeList, parserS[parserpt-0].node) + } + case 16: + //line lang.y:126 + { + parserVAL.node = &ast.LiteralNode{ + Value: parserS[parserpt-0].token.Value.(string), + Type: ast.TypeString, + Posx: parserS[parserpt-0].token.Pos, + } + } + } + goto parserstack /* stack new state and value */ +} diff --git a/config/lang/y.output b/config/lang/y.output new file mode 100644 index 000000000..df08ec4ad --- /dev/null +++ b/config/lang/y.output @@ -0,0 +1,198 @@ + +state 0 + $accept: .top $end + top: . (1) + + PROGRAM_BRACKET_LEFT shift 7 + STRING shift 6 + . reduce 1 (src line 32) + + interpolation goto 5 + literal goto 4 + literalModeTop goto 2 + literalModeValue goto 3 + top goto 1 + +state 1 + $accept: top.$end + + $end accept + . error + + +state 2 + top: literalModeTop. (2) + literalModeTop: literalModeTop.literalModeValue + + PROGRAM_BRACKET_LEFT shift 7 + STRING shift 6 + . reduce 2 (src line 40) + + interpolation goto 5 + literal goto 4 + literalModeValue goto 8 + +state 3 + literalModeTop: literalModeValue. (3) + + . reduce 3 (src line 45) + + +state 4 + literalModeValue: literal. (5) + + . reduce 5 (src line 65) + + +state 5 + literalModeValue: interpolation. (6) + + . reduce 6 (src line 70) + + +state 6 + literal: STRING. (16) + + . reduce 16 (src line 124) + + +state 7 + interpolation: PROGRAM_BRACKET_LEFT.expr PROGRAM_BRACKET_RIGHT + + PROGRAM_BRACKET_LEFT shift 7 + IDENTIFIER shift 13 + INTEGER shift 11 + FLOAT shift 12 + STRING shift 6 + . error + + expr goto 9 + interpolation goto 5 + literal goto 4 + literalModeTop goto 10 + literalModeValue goto 3 + +state 8 + literalModeTop: literalModeTop literalModeValue. (4) + + . reduce 4 (src line 50) + + +state 9 + interpolation: PROGRAM_BRACKET_LEFT expr.PROGRAM_BRACKET_RIGHT + + PROGRAM_BRACKET_RIGHT shift 14 + . error + + +state 10 + literalModeTop: literalModeTop.literalModeValue + expr: literalModeTop. (8) + + PROGRAM_BRACKET_LEFT shift 7 + STRING shift 6 + . reduce 8 (src line 81) + + interpolation goto 5 + literal goto 4 + literalModeValue goto 8 + +state 11 + expr: INTEGER. (9) + + . reduce 9 (src line 86) + + +state 12 + expr: FLOAT. (10) + + . reduce 10 (src line 94) + + +state 13 + expr: IDENTIFIER. (11) + expr: IDENTIFIER.PAREN_LEFT args PAREN_RIGHT + + PAREN_LEFT shift 15 + . reduce 11 (src line 102) + + +state 14 + interpolation: PROGRAM_BRACKET_LEFT expr PROGRAM_BRACKET_RIGHT. (7) + + . reduce 7 (src line 75) + + +state 15 + expr: IDENTIFIER PAREN_LEFT.args PAREN_RIGHT + args: . (13) + + PROGRAM_BRACKET_LEFT shift 7 + IDENTIFIER shift 13 + INTEGER shift 11 + FLOAT shift 12 + STRING shift 6 + . reduce 13 (src line 111) + + expr goto 17 + interpolation goto 5 + literal goto 4 + literalModeTop goto 10 + literalModeValue goto 3 + args goto 16 + +state 16 + expr: IDENTIFIER PAREN_LEFT args.PAREN_RIGHT + args: args.COMMA expr + + PAREN_RIGHT shift 18 + COMMA shift 19 + . error + + +state 17 + args: expr. (15) + + . reduce 15 (src line 119) + + +state 18 + expr: IDENTIFIER PAREN_LEFT args PAREN_RIGHT. (12) + + . reduce 12 (src line 106) + + +state 19 + args: args COMMA.expr + + PROGRAM_BRACKET_LEFT shift 7 + IDENTIFIER shift 13 + INTEGER shift 11 + FLOAT shift 12 + STRING shift 6 + . error + + expr goto 20 + interpolation goto 5 + literal goto 4 + literalModeTop goto 10 + literalModeValue goto 3 + +state 20 + args: args COMMA expr. (14) + + . reduce 14 (src line 115) + + +14 terminals, 8 nonterminals +17 grammar rules, 21/2000 states +0 shift/reduce, 0 reduce/reduce conflicts reported +57 working sets used +memory: parser 25/30000 +16 extra closures +25 shift entries, 1 exceptions +12 goto entries +15 entries saved by goto default +Optimizer space used: output 23/30000 +23 table entries, 0 zero +maximum spread: 14, maximum offset: 19 diff --git a/config/raw_config.go b/config/raw_config.go index e1c6c4108..0e1939184 100644 --- a/config/raw_config.go +++ b/config/raw_config.go @@ -4,6 +4,8 @@ import ( "bytes" "encoding/gob" + "github.com/hashicorp/terraform/config/lang" + "github.com/hashicorp/terraform/config/lang/ast" "github.com/mitchellh/copystructure" "github.com/mitchellh/reflectwalk" ) @@ -26,7 +28,7 @@ const UnknownVariableValue = "74D93920-ED26-11E3-AC10-0800200C9A66" type RawConfig struct { Key string Raw map[string]interface{} - Interpolations []Interpolation + Interpolations []ast.Node Variables map[string]InterpolatedVariable config map[string]interface{} @@ -79,8 +81,14 @@ func (r *RawConfig) Config() map[string]interface{} { // // If a variable key is missing, this will panic. func (r *RawConfig) Interpolate(vs map[string]string) error { - return r.interpolate(func(i Interpolation) (string, error) { - return i.Interpolate(vs) + engine := langEngine(vs) + return r.interpolate(func(root ast.Node) (string, error) { + out, _, err := engine.Execute(root) + if err != nil { + return "", err + } + + return out.(string), nil }) } @@ -89,15 +97,19 @@ func (r *RawConfig) init() error { r.Interpolations = nil r.Variables = nil - fn := func(i Interpolation) (string, error) { - r.Interpolations = append(r.Interpolations, i) + fn := func(node ast.Node) (string, error) { + r.Interpolations = append(r.Interpolations, node) + vars, err := DetectVariables(node) + if err != nil { + return "", err + } - for k, v := range i.Variables() { + for _, v := range vars { if r.Variables == nil { r.Variables = make(map[string]InterpolatedVariable) } - r.Variables[k] = v + r.Variables[v.FullKey()] = v } return "", nil @@ -189,3 +201,24 @@ type gobRawConfig struct { Key string Raw map[string]interface{} } + +// langEngine returns the lang.Engine to use for evaluating configurations. +func langEngine(vs map[string]string) *lang.Engine { + varMap := make(map[string]lang.Variable) + for k, v := range vs { + varMap[k] = lang.Variable{Value: v, Type: ast.TypeString} + } + + funcMap := make(map[string]lang.Function) + for k, v := range Funcs { + funcMap[k] = v + } + funcMap["lookup"] = interpolationFuncLookup(vs) + + return &lang.Engine{ + GlobalScope: &lang.Scope{ + VarMap: varMap, + FuncMap: funcMap, + }, + } +}