From fcdcf117f0edb077d1b1ee4f9a267396f2f7774a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 11 Jan 2015 12:38:45 -0800 Subject: [PATCH 01/31] config/lang: initial work --- config/lang/ast.go | 1 + config/lang/ast/ast.go | 12 + config/lang/ast/concat.go | 7 + config/lang/ast/literal.go | 8 + config/lang/ast/variable_access.go | 6 + config/lang/lang.y | 59 +++++ config/lang/lex.go | 176 +++++++++++++++ config/lang/lex_test.go | 65 ++++++ config/lang/parse.go | 32 +++ config/lang/parse_test.go | 68 ++++++ config/lang/token.go | 1 + config/lang/y.go | 351 +++++++++++++++++++++++++++++ config/lang/y.output | 85 +++++++ 13 files changed, 871 insertions(+) create mode 100644 config/lang/ast.go create mode 100644 config/lang/ast/ast.go create mode 100644 config/lang/ast/concat.go create mode 100644 config/lang/ast/literal.go create mode 100644 config/lang/ast/variable_access.go create mode 100644 config/lang/lang.y create mode 100644 config/lang/lex.go create mode 100644 config/lang/lex_test.go create mode 100644 config/lang/parse.go create mode 100644 config/lang/parse_test.go create mode 100644 config/lang/token.go create mode 100644 config/lang/y.go create mode 100644 config/lang/y.output diff --git a/config/lang/ast.go b/config/lang/ast.go new file mode 100644 index 000000000..48e87f869 --- /dev/null +++ b/config/lang/ast.go @@ -0,0 +1 @@ +package lang diff --git a/config/lang/ast/ast.go b/config/lang/ast/ast.go new file mode 100644 index 000000000..3bb748ccc --- /dev/null +++ b/config/lang/ast/ast.go @@ -0,0 +1,12 @@ +package ast + +// Node is the interface that all AST nodes must implement. +type Node interface{} + +// Type is the type of a literal. +type Type uint + +const ( + TypeInvalid Type = 1 << iota + TypeString +) diff --git a/config/lang/ast/concat.go b/config/lang/ast/concat.go new file mode 100644 index 000000000..3a3374ccc --- /dev/null +++ b/config/lang/ast/concat.go @@ -0,0 +1,7 @@ +package ast + +// Concat represents a node where the result of two or more expressions are +// concatenated. The result of all expressions must be a string. +type Concat struct { + Exprs []Node +} diff --git a/config/lang/ast/literal.go b/config/lang/ast/literal.go new file mode 100644 index 000000000..f82ad4f9c --- /dev/null +++ b/config/lang/ast/literal.go @@ -0,0 +1,8 @@ +package ast + +// LiteralNode represents a single literal value, such as "foo" or +// 42 or 3.14159. Based on the Type, the Value can be safely cast. +type LiteralNode struct { + Value interface{} + Type Type +} diff --git a/config/lang/ast/variable_access.go b/config/lang/ast/variable_access.go new file mode 100644 index 000000000..bf22ce356 --- /dev/null +++ b/config/lang/ast/variable_access.go @@ -0,0 +1,6 @@ +package ast + +// VariableAccess represents a variable access. +type VariableAccess struct { + Name string +} diff --git a/config/lang/lang.y b/config/lang/lang.y new file mode 100644 index 000000000..52db6ee34 --- /dev/null +++ b/config/lang/lang.y @@ -0,0 +1,59 @@ +// This is the yacc input for creating the parser for interpolation +// expressions in Go. To build it, just run `go generate` on this +// package, as the lexer has the go generate pragma within it. + +%{ +package lang + +import ( + "github.com/hashicorp/terraform/config/lang/ast" +) + +%} + +%union { + node ast.Node + str string +} + +%token STRING IDENTIFIER PROGRAM_BRACKET_LEFT PROGRAM_BRACKET_RIGHT + +%type expr interpolation literal + +%% + +top: + literal + { + parserResult = $1 + } +| literal interpolation + { + parserResult = &ast.Concat{ + Exprs: []ast.Node{$1, $2}, + } + } + +interpolation: + PROGRAM_BRACKET_LEFT expr PROGRAM_BRACKET_RIGHT + { + $$ = $2 + } + +expr: + IDENTIFIER + { + $$ = &ast.VariableAccess{Name: $1} + } +| literal + { + $$ = $1 + } + +literal: + STRING + { + $$ = &ast.LiteralNode{Value: $1, Type: ast.TypeString} + } + +%% diff --git a/config/lang/lex.go b/config/lang/lex.go new file mode 100644 index 000000000..6f0705c84 --- /dev/null +++ b/config/lang/lex.go @@ -0,0 +1,176 @@ +package lang + +import ( + "bytes" + "fmt" + "unicode" + "unicode/utf8" +) + +//go:generate go tool yacc -p parser lang.y + +// The parser expects the lexer to return 0 on EOF. +const lexEOF = 0 + +// The parser uses the type Lex as a lexer. It must provide +// the methods Lex(*SymType) int and Error(string). +type parserLex struct { + Err error + Input string + + interpolationDepth int + pos int + width int +} + +// The parser calls this method to get each new token. +func (x *parserLex) Lex(yylval *parserSymType) int { + for { + c := x.next() + if c == lexEOF { + return lexEOF + } + + // Are we starting an interpolation? + if c == '$' && x.peek() == '{' { + x.next() + x.interpolationDepth++ + return PROGRAM_BRACKET_LEFT + } + + // If we see a double quote and we're in an interpolation, then + // we are lexing a string. + if c == '"' && x.interpolationDepth > 0 { + return x.lexString(yylval, true) + } + + switch c { + case '}': + x.interpolationDepth-- + return PROGRAM_BRACKET_RIGHT + default: + x.backup() + if x.interpolationDepth > 0 { + // We're within an interpolation. + return x.lexId(yylval) + } else { + // We're just a normal string that isn't part of any + // interpolation yet. + return x.lexString(yylval, false) + } + } + } +} + +func (x *parserLex) lexId(yylval *parserSymType) int { + var b bytes.Buffer + for { + c := x.next() + if c == lexEOF { + break + } + + // If this isn't a character we want in an ID, return out. + // One day we should make this a regexp. + if c != '_' && + c != '-' && + c != '.' && + c != '*' && + !unicode.IsLetter(c) && + !unicode.IsNumber(c) { + x.backup() + break + } + + if _, err := b.WriteRune(c); err != nil { + x.Error(err.Error()) + return lexEOF + } + } + + yylval.str = b.String() + return IDENTIFIER +} + +func (x *parserLex) lexString(yylval *parserSymType, quoted bool) int { + var b bytes.Buffer + for { + c := x.next() + if c == lexEOF { + break + } + + // Behavior is a bit different if we're lexing within a quoted string. + if quoted { + // If its a double quote, we've reached the end of the string + if c == '"' { + break + } + + // Let's check to see if we're escaping anything. + if c == '\\' { + switch n := x.next(); n { + case '\\': + fallthrough + case '"': + c = n + case 'n': + c = '\n' + default: + x.backup() + } + } + } + + // If we hit a '}' and we're in a program, then end it. + if c == '}' && x.interpolationDepth > 0 { + x.backup() + break + } + + // If we hit a dollar sign, then check if we're starting + // another interpolation. If so, then we're done. + if c == '$' && x.peek() == '{' { + x.backup() + break + } + + if _, err := b.WriteRune(c); err != nil { + x.Error(err.Error()) + return lexEOF + } + } + + yylval.str = b.String() + return STRING +} + +// Return the next rune for the lexer. +func (x *parserLex) next() rune { + if int(x.pos) >= len(x.Input) { + x.width = 0 + return lexEOF + } + + r, w := utf8.DecodeRuneInString(x.Input[x.pos:]) + x.width = w + x.pos += x.width + return r +} + +// peek returns but does not consume the next rune in the input +func (x *parserLex) peek() rune { + r := x.next() + x.backup() + return r +} + +// backup steps back one rune. Can only be called once per next. +func (x *parserLex) backup() { + x.pos -= x.width +} + +// The parser calls this method on a parse error. +func (x *parserLex) Error(s string) { + x.Err = fmt.Errorf("parse error: %s", s) +} diff --git a/config/lang/lex_test.go b/config/lang/lex_test.go new file mode 100644 index 000000000..ab5a18b9b --- /dev/null +++ b/config/lang/lex_test.go @@ -0,0 +1,65 @@ +package lang + +import ( + "reflect" + "testing" +) + +func TestLex(t *testing.T) { + cases := []struct { + Input string + Output []int + }{ + { + "foo", + []int{STRING, lexEOF}, + }, + + { + "foo$bar", + []int{STRING, lexEOF}, + }, + + { + "foo ${bar}", + []int{STRING, PROGRAM_BRACKET_LEFT, IDENTIFIER, PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + "foo ${\"bar\"}", + []int{STRING, PROGRAM_BRACKET_LEFT, STRING, PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + } + + for _, tc := range cases { + l := &parserLex{Input: tc.Input} + var actual []int + for { + token := l.Lex(new(parserSymType)) + actual = append(actual, token) + + if token == lexEOF { + break + } + + // Be careful against what are probably infinite loops + if len(actual) > 100 { + t.Fatalf("Input:%s\n\nExausted.", tc.Input) + } + } + + if !reflect.DeepEqual(actual, tc.Output) { + t.Fatalf( + "Input: %s\n\nBad: %#v\n\nExpected: %#v", + tc.Input, actual, tc.Output) + } + } +} + +/* OTHERS: + +foo ${var.foo} +bar ${"hello"} +foo ${concat("foo ${var.bar}", var.baz)} + +*/ diff --git a/config/lang/parse.go b/config/lang/parse.go new file mode 100644 index 000000000..8ece590af --- /dev/null +++ b/config/lang/parse.go @@ -0,0 +1,32 @@ +package lang + +import ( + "sync" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +var parserErrors []error +var parserLock sync.Mutex +var parserResult ast.Node + +// Parse parses the given program and returns an executable AST tree. +func Parse(v string) (ast.Node, error) { + // Unfortunately due to the way that goyacc generated parsers are + // formatted, we can only do a single parse at a time without a lot + // of extra work. In the future we can remove this limitation. + parserLock.Lock() + defer parserLock.Unlock() + + // Reset our globals + parserErrors = nil + parserResult = nil + + // Create the lexer + lex := &parserLex{Input: v} + + // Parse! + parserParse(lex) + + return parserResult, lex.Err +} diff --git a/config/lang/parse_test.go b/config/lang/parse_test.go new file mode 100644 index 000000000..38507582a --- /dev/null +++ b/config/lang/parse_test.go @@ -0,0 +1,68 @@ +package lang + +import ( + "reflect" + "testing" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +func TestParse(t *testing.T) { + cases := []struct { + Input string + Error bool + Result ast.Node + }{ + { + "foo", + false, + &ast.LiteralNode{ + Value: "foo", + Type: ast.TypeString, + }, + }, + + { + "foo ${var.bar}", + false, + &ast.Concat{ + Exprs: []ast.Node{ + &ast.LiteralNode{ + Value: "foo ", + Type: ast.TypeString, + }, + &ast.VariableAccess{ + Name: "var.bar", + }, + }, + }, + }, + + { + "foo ${\"bar\"}", + false, + &ast.Concat{ + Exprs: []ast.Node{ + &ast.LiteralNode{ + Value: "foo ", + Type: ast.TypeString, + }, + &ast.LiteralNode{ + Value: "bar", + Type: ast.TypeString, + }, + }, + }, + }, + } + + for _, tc := range cases { + actual, err := Parse(tc.Input) + if (err != nil) != tc.Error { + t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) + } + if !reflect.DeepEqual(actual, tc.Result) { + t.Fatalf("Bad: %#v\n\nInput: %s", actual, tc.Input) + } + } +} diff --git a/config/lang/token.go b/config/lang/token.go new file mode 100644 index 000000000..48e87f869 --- /dev/null +++ b/config/lang/token.go @@ -0,0 +1 @@ +package lang diff --git a/config/lang/y.go b/config/lang/y.go new file mode 100644 index 000000000..a2b2e81fd --- /dev/null +++ b/config/lang/y.go @@ -0,0 +1,351 @@ +//line lang.y:6 +package lang + +import __yyfmt__ "fmt" + +//line lang.y:6 +import ( + "github.com/hashicorp/terraform/config/lang/ast" +) + +//line lang.y:14 +type parserSymType struct { + yys int + node ast.Node + str string +} + +const STRING = 57346 +const IDENTIFIER = 57347 +const PROGRAM_BRACKET_LEFT = 57348 +const PROGRAM_BRACKET_RIGHT = 57349 + +var parserToknames = []string{ + "STRING", + "IDENTIFIER", + "PROGRAM_BRACKET_LEFT", + "PROGRAM_BRACKET_RIGHT", +} +var parserStatenames = []string{} + +const parserEofCode = 1 +const parserErrCode = 2 +const parserMaxDepth = 200 + +//line lang.y:59 + +//line yacctab:1 +var parserExca = []int{ + -1, 1, + 1, -1, + -2, 0, +} + +const parserNprod = 7 +const parserPrivate = 57344 + +var parserTokenNames []string +var parserStates []string + +const parserLast = 10 + +var parserAct = []int{ + + 9, 3, 7, 2, 5, 3, 1, 4, 6, 8, +} +var parserPact = []int{ + + 1, -1000, -2, -1000, -1000, -3, -7, -1000, -1000, -1000, +} +var parserPgo = []int{ + + 0, 8, 7, 3, 6, +} +var parserR1 = []int{ + + 0, 4, 4, 2, 1, 1, 3, +} +var parserR2 = []int{ + + 0, 1, 2, 3, 1, 1, 1, +} +var parserChk = []int{ + + -1000, -4, -3, 4, -2, 6, -1, 5, -3, 7, +} +var parserDef = []int{ + + 0, -2, 1, 6, 2, 0, 0, 4, 5, 3, +} +var parserTok1 = []int{ + + 1, +} +var parserTok2 = []int{ + + 2, 3, 4, 5, 6, 7, +} +var parserTok3 = []int{ + 0, +} + +//line yaccpar:1 + +/* parser for yacc output */ + +var parserDebug = 0 + +type parserLexer interface { + Lex(lval *parserSymType) int + Error(s string) +} + +const parserFlag = -1000 + +func parserTokname(c int) string { + // 4 is TOKSTART above + if c >= 4 && c-4 < len(parserToknames) { + if parserToknames[c-4] != "" { + return parserToknames[c-4] + } + } + return __yyfmt__.Sprintf("tok-%v", c) +} + +func parserStatname(s int) string { + if s >= 0 && s < len(parserStatenames) { + if parserStatenames[s] != "" { + return parserStatenames[s] + } + } + return __yyfmt__.Sprintf("state-%v", s) +} + +func parserlex1(lex parserLexer, lval *parserSymType) int { + c := 0 + char := lex.Lex(lval) + if char <= 0 { + c = parserTok1[0] + goto out + } + if char < len(parserTok1) { + c = parserTok1[char] + goto out + } + if char >= parserPrivate { + if char < parserPrivate+len(parserTok2) { + c = parserTok2[char-parserPrivate] + goto out + } + } + for i := 0; i < len(parserTok3); i += 2 { + c = parserTok3[i+0] + if c == char { + c = parserTok3[i+1] + goto out + } + } + +out: + if c == 0 { + c = parserTok2[1] /* unknown char */ + } + if parserDebug >= 3 { + __yyfmt__.Printf("lex %s(%d)\n", parserTokname(c), uint(char)) + } + return c +} + +func parserParse(parserlex parserLexer) int { + var parsern int + var parserlval parserSymType + var parserVAL parserSymType + parserS := make([]parserSymType, parserMaxDepth) + + Nerrs := 0 /* number of errors */ + Errflag := 0 /* error recovery flag */ + parserstate := 0 + parserchar := -1 + parserp := -1 + goto parserstack + +ret0: + return 0 + +ret1: + return 1 + +parserstack: + /* put a state and value onto the stack */ + if parserDebug >= 4 { + __yyfmt__.Printf("char %v in %v\n", parserTokname(parserchar), parserStatname(parserstate)) + } + + parserp++ + if parserp >= len(parserS) { + nyys := make([]parserSymType, len(parserS)*2) + copy(nyys, parserS) + parserS = nyys + } + parserS[parserp] = parserVAL + parserS[parserp].yys = parserstate + +parsernewstate: + parsern = parserPact[parserstate] + if parsern <= parserFlag { + goto parserdefault /* simple state */ + } + if parserchar < 0 { + parserchar = parserlex1(parserlex, &parserlval) + } + parsern += parserchar + if parsern < 0 || parsern >= parserLast { + goto parserdefault + } + parsern = parserAct[parsern] + if parserChk[parsern] == parserchar { /* valid shift */ + parserchar = -1 + parserVAL = parserlval + parserstate = parsern + if Errflag > 0 { + Errflag-- + } + goto parserstack + } + +parserdefault: + /* default state action */ + parsern = parserDef[parserstate] + if parsern == -2 { + if parserchar < 0 { + parserchar = parserlex1(parserlex, &parserlval) + } + + /* look through exception table */ + xi := 0 + for { + if parserExca[xi+0] == -1 && parserExca[xi+1] == parserstate { + break + } + xi += 2 + } + for xi += 2; ; xi += 2 { + parsern = parserExca[xi+0] + if parsern < 0 || parsern == parserchar { + break + } + } + parsern = parserExca[xi+1] + if parsern < 0 { + goto ret0 + } + } + if parsern == 0 { + /* error ... attempt to resume parsing */ + switch Errflag { + case 0: /* brand new error */ + parserlex.Error("syntax error") + Nerrs++ + if parserDebug >= 1 { + __yyfmt__.Printf("%s", parserStatname(parserstate)) + __yyfmt__.Printf(" saw %s\n", parserTokname(parserchar)) + } + fallthrough + + case 1, 2: /* incompletely recovered error ... try again */ + Errflag = 3 + + /* find a state where "error" is a legal shift action */ + for parserp >= 0 { + parsern = parserPact[parserS[parserp].yys] + parserErrCode + if parsern >= 0 && parsern < parserLast { + parserstate = parserAct[parsern] /* simulate a shift of "error" */ + if parserChk[parserstate] == parserErrCode { + goto parserstack + } + } + + /* the current p has no shift on "error", pop stack */ + if parserDebug >= 2 { + __yyfmt__.Printf("error recovery pops state %d\n", parserS[parserp].yys) + } + parserp-- + } + /* there is no state on the stack with an error shift ... abort */ + goto ret1 + + case 3: /* no shift yet; clobber input char */ + if parserDebug >= 2 { + __yyfmt__.Printf("error recovery discards %s\n", parserTokname(parserchar)) + } + if parserchar == parserEofCode { + goto ret1 + } + parserchar = -1 + goto parsernewstate /* try again in the same state */ + } + } + + /* reduction by production parsern */ + if parserDebug >= 2 { + __yyfmt__.Printf("reduce %v in:\n\t%v\n", parsern, parserStatname(parserstate)) + } + + parsernt := parsern + parserpt := parserp + _ = parserpt // guard against "declared and not used" + + parserp -= parserR2[parsern] + parserVAL = parserS[parserp+1] + + /* consult goto table to find next state */ + parsern = parserR1[parsern] + parserg := parserPgo[parsern] + parserj := parserg + parserS[parserp].yys + 1 + + if parserj >= parserLast { + parserstate = parserAct[parserg] + } else { + parserstate = parserAct[parserj] + if parserChk[parserstate] != -parsern { + parserstate = parserAct[parserg] + } + } + // dummy call; replaced with literal code + switch parsernt { + + case 1: + //line lang.y:27 + { + parserResult = parserS[parserpt-0].node + } + case 2: + //line lang.y:31 + { + parserResult = &ast.Concat{ + Exprs: []ast.Node{parserS[parserpt-1].node, parserS[parserpt-0].node}, + } + } + case 3: + //line lang.y:39 + { + parserVAL.node = parserS[parserpt-1].node + } + case 4: + //line lang.y:45 + { + parserVAL.node = &ast.VariableAccess{Name: parserS[parserpt-0].str} + } + case 5: + //line lang.y:49 + { + parserVAL.node = parserS[parserpt-0].node + } + case 6: + //line lang.y:55 + { + parserVAL.node = &ast.LiteralNode{Value: parserS[parserpt-0].str, Type: ast.TypeString} + } + } + goto parserstack /* stack new state and value */ +} diff --git a/config/lang/y.output b/config/lang/y.output new file mode 100644 index 000000000..1fe8babd4 --- /dev/null +++ b/config/lang/y.output @@ -0,0 +1,85 @@ + +state 0 + $accept: .top $end + + STRING shift 3 + . error + + literal goto 2 + top goto 1 + +state 1 + $accept: top.$end + + $end accept + . error + + +state 2 + top: literal. (1) + top: literal.interpolation + + PROGRAM_BRACKET_LEFT shift 5 + . reduce 1 (src line 25) + + interpolation goto 4 + +state 3 + literal: STRING. (6) + + . reduce 6 (src line 53) + + +state 4 + top: literal interpolation. (2) + + . reduce 2 (src line 30) + + +state 5 + interpolation: PROGRAM_BRACKET_LEFT.expr PROGRAM_BRACKET_RIGHT + + STRING shift 3 + IDENTIFIER shift 7 + . error + + expr goto 6 + literal goto 8 + +state 6 + interpolation: PROGRAM_BRACKET_LEFT expr.PROGRAM_BRACKET_RIGHT + + PROGRAM_BRACKET_RIGHT shift 9 + . error + + +state 7 + expr: IDENTIFIER. (4) + + . reduce 4 (src line 43) + + +state 8 + expr: literal. (5) + + . reduce 5 (src line 48) + + +state 9 + interpolation: PROGRAM_BRACKET_LEFT expr PROGRAM_BRACKET_RIGHT. (3) + + . reduce 3 (src line 37) + + +7 terminals, 5 nonterminals +7 grammar rules, 10/2000 states +0 shift/reduce, 0 reduce/reduce conflicts reported +54 working sets used +memory: parser 5/30000 +1 extra closures +5 shift entries, 1 exceptions +5 goto entries +0 entries saved by goto default +Optimizer space used: output 10/30000 +10 table entries, 0 zero +maximum spread: 7, maximum offset: 7 From 669bdc61f961e17bda1c8d143d4457913acf1931 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 11 Jan 2015 13:03:37 -0800 Subject: [PATCH 02/31] config/lang: more implemented --- config/lang/ast/call.go | 7 ++ config/lang/ast/variable_access.go | 8 ++ config/lang/lang.y | 38 ++++++++-- config/lang/lex.go | 29 ++++--- config/lang/lex_test.go | 16 ++++ config/lang/parse_test.go | 29 +++++++ config/lang/y.go | 88 ++++++++++++++++------ config/lang/y.output | 117 ++++++++++++++++++++++------- 8 files changed, 262 insertions(+), 70 deletions(-) create mode 100644 config/lang/ast/call.go diff --git a/config/lang/ast/call.go b/config/lang/ast/call.go new file mode 100644 index 000000000..35fc03f50 --- /dev/null +++ b/config/lang/ast/call.go @@ -0,0 +1,7 @@ +package ast + +// Call represents a function call. +type Call struct { + Func string + Args []Node +} diff --git a/config/lang/ast/variable_access.go b/config/lang/ast/variable_access.go index bf22ce356..5d63dcba1 100644 --- a/config/lang/ast/variable_access.go +++ b/config/lang/ast/variable_access.go @@ -1,6 +1,14 @@ package ast +import ( + "fmt" +) + // VariableAccess represents a variable access. type VariableAccess struct { Name string } + +func (n *VariableAccess) GoString() string { + return fmt.Sprintf("*%#v", *n) +} diff --git a/config/lang/lang.y b/config/lang/lang.y index 52db6ee34..f346799fd 100644 --- a/config/lang/lang.y +++ b/config/lang/lang.y @@ -12,13 +12,16 @@ import ( %} %union { - node ast.Node - str string + node ast.Node + nodeList []ast.Node + str string } %token STRING IDENTIFIER PROGRAM_BRACKET_LEFT PROGRAM_BRACKET_RIGHT +%token PAREN_LEFT PAREN_RIGHT COMMA %type expr interpolation literal +%type args %% @@ -27,6 +30,10 @@ top: { parserResult = $1 } +| interpolation + { + parserResult = $1 + } | literal interpolation { parserResult = &ast.Concat{ @@ -41,14 +48,31 @@ interpolation: } expr: - IDENTIFIER - { - $$ = &ast.VariableAccess{Name: $1} - } -| literal + literal { $$ = $1 } +| IDENTIFIER + { + $$ = &ast.VariableAccess{Name: $1} + } +| IDENTIFIER PAREN_LEFT args PAREN_RIGHT + { + $$ = &ast.Call{Func: $1, Args: $3} + } + +args: + { + $$ = nil + } +| args COMMA expr + { + $$ = append($1, $3) + } +| expr + { + $$ = append($$, $1) + } literal: STRING diff --git a/config/lang/lex.go b/config/lang/lex.go index 6f0705c84..365001199 100644 --- a/config/lang/lex.go +++ b/config/lang/lex.go @@ -38,9 +38,21 @@ func (x *parserLex) Lex(yylval *parserSymType) int { return PROGRAM_BRACKET_LEFT } + if x.interpolationDepth == 0 { + // We're just a normal string that isn't part of any + // interpolation yet. + x.backup() + return x.lexString(yylval, false) + } + + // Ignore all whitespace + if unicode.IsSpace(c) { + continue + } + // If we see a double quote and we're in an interpolation, then // we are lexing a string. - if c == '"' && x.interpolationDepth > 0 { + if c == '"' { return x.lexString(yylval, true) } @@ -48,16 +60,15 @@ func (x *parserLex) Lex(yylval *parserSymType) int { case '}': x.interpolationDepth-- return PROGRAM_BRACKET_RIGHT + case '(': + return PAREN_LEFT + case ')': + return PAREN_RIGHT + case ',': + return COMMA default: x.backup() - if x.interpolationDepth > 0 { - // We're within an interpolation. - return x.lexId(yylval) - } else { - // We're just a normal string that isn't part of any - // interpolation yet. - return x.lexString(yylval, false) - } + return x.lexId(yylval) } } } diff --git a/config/lang/lex_test.go b/config/lang/lex_test.go index ab5a18b9b..3546bc885 100644 --- a/config/lang/lex_test.go +++ b/config/lang/lex_test.go @@ -29,6 +29,22 @@ func TestLex(t *testing.T) { "foo ${\"bar\"}", []int{STRING, PROGRAM_BRACKET_LEFT, STRING, PROGRAM_BRACKET_RIGHT, lexEOF}, }, + + { + "${bar(baz)}", + []int{PROGRAM_BRACKET_LEFT, + IDENTIFIER, PAREN_LEFT, IDENTIFIER, PAREN_RIGHT, + PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + "${bar(baz, foo)}", + []int{PROGRAM_BRACKET_LEFT, + IDENTIFIER, PAREN_LEFT, + IDENTIFIER, COMMA, IDENTIFIER, + PAREN_RIGHT, + PROGRAM_BRACKET_RIGHT, lexEOF}, + }, } for _, tc := range cases { diff --git a/config/lang/parse_test.go b/config/lang/parse_test.go index 38507582a..7e7114f57 100644 --- a/config/lang/parse_test.go +++ b/config/lang/parse_test.go @@ -54,6 +54,35 @@ func TestParse(t *testing.T) { }, }, }, + + { + "${foo(bar)}", + false, + &ast.Call{ + Func: "foo", + Args: []ast.Node{ + &ast.VariableAccess{ + Name: "bar", + }, + }, + }, + }, + + { + "${foo(bar, baz)}", + false, + &ast.Call{ + Func: "foo", + Args: []ast.Node{ + &ast.VariableAccess{ + Name: "bar", + }, + &ast.VariableAccess{ + Name: "baz", + }, + }, + }, + }, } for _, tc := range cases { diff --git a/config/lang/y.go b/config/lang/y.go index a2b2e81fd..a0ef3d105 100644 --- a/config/lang/y.go +++ b/config/lang/y.go @@ -10,21 +10,28 @@ import ( //line lang.y:14 type parserSymType struct { - yys int - node ast.Node - str string + yys int + node ast.Node + nodeList []ast.Node + str string } const STRING = 57346 const IDENTIFIER = 57347 const PROGRAM_BRACKET_LEFT = 57348 const PROGRAM_BRACKET_RIGHT = 57349 +const PAREN_LEFT = 57350 +const PAREN_RIGHT = 57351 +const COMMA = 57352 var parserToknames = []string{ "STRING", "IDENTIFIER", "PROGRAM_BRACKET_LEFT", "PROGRAM_BRACKET_RIGHT", + "PAREN_LEFT", + "PAREN_RIGHT", + "COMMA", } var parserStatenames = []string{} @@ -32,7 +39,7 @@ const parserEofCode = 1 const parserErrCode = 2 const parserMaxDepth = 200 -//line lang.y:59 +//line lang.y:83 //line yacctab:1 var parserExca = []int{ @@ -41,41 +48,47 @@ var parserExca = []int{ -2, 0, } -const parserNprod = 7 +const parserNprod = 12 const parserPrivate = 57344 var parserTokenNames []string var parserStates []string -const parserLast = 10 +const parserLast = 18 var parserAct = []int{ - 9, 3, 7, 2, 5, 3, 1, 4, 6, 8, + 7, 14, 15, 11, 10, 4, 5, 5, 4, 9, + 3, 1, 13, 6, 8, 2, 16, 12, } var parserPact = []int{ - 1, -1000, -2, -1000, -1000, -3, -7, -1000, -1000, -1000, + 1, -1000, 0, -1000, -1000, 4, -1000, -3, -1000, -5, + -1000, 4, -8, -1000, -1000, 4, -1000, } var parserPgo = []int{ - 0, 8, 7, 3, 6, + 0, 0, 10, 14, 17, 11, } var parserR1 = []int{ - 0, 4, 4, 2, 1, 1, 3, + 0, 5, 5, 5, 2, 1, 1, 1, 4, 4, + 4, 3, } var parserR2 = []int{ - 0, 1, 2, 3, 1, 1, 1, + 0, 1, 1, 2, 3, 1, 1, 4, 0, 3, + 1, 1, } var parserChk = []int{ - -1000, -4, -3, 4, -2, 6, -1, 5, -3, 7, + -1000, -5, -3, -2, 4, 6, -2, -1, -3, 5, + 7, 8, -4, -1, 9, 10, -1, } var parserDef = []int{ - 0, -2, 1, 6, 2, 0, 0, 4, 5, 3, + 0, -2, 1, 2, 11, 0, 3, 0, 5, 6, + 4, 8, 0, 10, 7, 0, 9, } var parserTok1 = []int{ @@ -83,7 +96,7 @@ var parserTok1 = []int{ } var parserTok2 = []int{ - 2, 3, 4, 5, 6, 7, + 2, 3, 4, 5, 6, 7, 8, 9, 10, } var parserTok3 = []int{ 0, @@ -315,34 +328,59 @@ parserdefault: switch parsernt { case 1: - //line lang.y:27 + //line lang.y:30 { parserResult = parserS[parserpt-0].node } case 2: - //line lang.y:31 + //line lang.y:34 + { + parserResult = parserS[parserpt-0].node + } + case 3: + //line lang.y:38 { parserResult = &ast.Concat{ Exprs: []ast.Node{parserS[parserpt-1].node, parserS[parserpt-0].node}, } } - case 3: - //line lang.y:39 + case 4: + //line lang.y:46 { parserVAL.node = parserS[parserpt-1].node } - case 4: - //line lang.y:45 - { - parserVAL.node = &ast.VariableAccess{Name: parserS[parserpt-0].str} - } case 5: - //line lang.y:49 + //line lang.y:52 { parserVAL.node = parserS[parserpt-0].node } case 6: - //line lang.y:55 + //line lang.y:56 + { + parserVAL.node = &ast.VariableAccess{Name: parserS[parserpt-0].str} + } + case 7: + //line lang.y:60 + { + parserVAL.node = &ast.Call{Func: parserS[parserpt-3].str, Args: parserS[parserpt-1].nodeList} + } + case 8: + //line lang.y:65 + { + parserVAL.nodeList = nil + } + case 9: + //line lang.y:69 + { + parserVAL.nodeList = append(parserS[parserpt-2].nodeList, parserS[parserpt-0].node) + } + case 10: + //line lang.y:73 + { + parserVAL.nodeList = append(parserVAL.nodeList, parserS[parserpt-0].node) + } + case 11: + //line lang.y:79 { parserVAL.node = &ast.LiteralNode{Value: parserS[parserpt-0].str, Type: ast.TypeString} } diff --git a/config/lang/y.output b/config/lang/y.output index 1fe8babd4..f098e493d 100644 --- a/config/lang/y.output +++ b/config/lang/y.output @@ -2,9 +2,11 @@ state 0 $accept: .top $end - STRING shift 3 + STRING shift 4 + PROGRAM_BRACKET_LEFT shift 5 . error + interpolation goto 3 literal goto 2 top goto 1 @@ -20,66 +22,123 @@ state 2 top: literal.interpolation PROGRAM_BRACKET_LEFT shift 5 - . reduce 1 (src line 25) + . reduce 1 (src line 28) - interpolation goto 4 + interpolation goto 6 state 3 - literal: STRING. (6) + top: interpolation. (2) - . reduce 6 (src line 53) + . reduce 2 (src line 33) state 4 - top: literal interpolation. (2) + literal: STRING. (11) - . reduce 2 (src line 30) + . reduce 11 (src line 77) state 5 interpolation: PROGRAM_BRACKET_LEFT.expr PROGRAM_BRACKET_RIGHT - STRING shift 3 - IDENTIFIER shift 7 + STRING shift 4 + IDENTIFIER shift 9 . error - expr goto 6 + expr goto 7 literal goto 8 state 6 - interpolation: PROGRAM_BRACKET_LEFT expr.PROGRAM_BRACKET_RIGHT + top: literal interpolation. (3) - PROGRAM_BRACKET_RIGHT shift 9 - . error + . reduce 3 (src line 37) state 7 - expr: IDENTIFIER. (4) + interpolation: PROGRAM_BRACKET_LEFT expr.PROGRAM_BRACKET_RIGHT - . reduce 4 (src line 43) + PROGRAM_BRACKET_RIGHT shift 10 + . error state 8 expr: literal. (5) - . reduce 5 (src line 48) + . reduce 5 (src line 50) state 9 - interpolation: PROGRAM_BRACKET_LEFT expr PROGRAM_BRACKET_RIGHT. (3) + expr: IDENTIFIER. (6) + expr: IDENTIFIER.PAREN_LEFT args PAREN_RIGHT - . reduce 3 (src line 37) + PAREN_LEFT shift 11 + . reduce 6 (src line 55) -7 terminals, 5 nonterminals -7 grammar rules, 10/2000 states +state 10 + interpolation: PROGRAM_BRACKET_LEFT expr PROGRAM_BRACKET_RIGHT. (4) + + . reduce 4 (src line 44) + + +state 11 + expr: IDENTIFIER PAREN_LEFT.args PAREN_RIGHT + args: . (8) + + STRING shift 4 + IDENTIFIER shift 9 + . reduce 8 (src line 64) + + expr goto 13 + literal goto 8 + args goto 12 + +state 12 + expr: IDENTIFIER PAREN_LEFT args.PAREN_RIGHT + args: args.COMMA expr + + PAREN_RIGHT shift 14 + COMMA shift 15 + . error + + +state 13 + args: expr. (10) + + . reduce 10 (src line 72) + + +state 14 + expr: IDENTIFIER PAREN_LEFT args PAREN_RIGHT. (7) + + . reduce 7 (src line 59) + + +state 15 + args: args COMMA.expr + + STRING shift 4 + IDENTIFIER shift 9 + . error + + expr goto 16 + literal goto 8 + +state 16 + args: args COMMA expr. (9) + + . reduce 9 (src line 68) + + +10 terminals, 6 nonterminals +12 grammar rules, 17/2000 states 0 shift/reduce, 0 reduce/reduce conflicts reported -54 working sets used -memory: parser 5/30000 -1 extra closures -5 shift entries, 1 exceptions -5 goto entries -0 entries saved by goto default -Optimizer space used: output 10/30000 -10 table entries, 0 zero -maximum spread: 7, maximum offset: 7 +55 working sets used +memory: parser 11/30000 +6 extra closures +13 shift entries, 1 exceptions +9 goto entries +2 entries saved by goto default +Optimizer space used: output 18/30000 +18 table entries, 0 zero +maximum spread: 10, maximum offset: 15 From 1ff5a838a48cf9e012b59e40e463752f0ebeddec Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 11 Jan 2015 13:59:24 -0800 Subject: [PATCH 03/31] config/lang: can parse nested interpolations --- config/lang/ast/concat.go | 8 ++ config/lang/ast/literal.go | 8 ++ config/lang/lang.y | 40 ++++++--- config/lang/lex.go | 90 ++++++++++++++++++-- config/lang/lex_test.go | 34 ++++++++ config/lang/parse_test.go | 74 ++++++++++++++++ config/lang/y.go | 103 +++++++++++++--------- config/lang/y.output | 169 ++++++++++++++++++++++--------------- 8 files changed, 400 insertions(+), 126 deletions(-) diff --git a/config/lang/ast/concat.go b/config/lang/ast/concat.go index 3a3374ccc..c88e3de5a 100644 --- a/config/lang/ast/concat.go +++ b/config/lang/ast/concat.go @@ -1,7 +1,15 @@ package ast +import ( + "fmt" +) + // Concat represents a node where the result of two or more expressions are // concatenated. The result of all expressions must be a string. type Concat struct { Exprs []Node } + +func (n *Concat) GoString() string { + return fmt.Sprintf("*%#v", *n) +} diff --git a/config/lang/ast/literal.go b/config/lang/ast/literal.go index f82ad4f9c..3477a5dbd 100644 --- a/config/lang/ast/literal.go +++ b/config/lang/ast/literal.go @@ -1,8 +1,16 @@ package ast +import ( + "fmt" +) + // LiteralNode represents a single literal value, such as "foo" or // 42 or 3.14159. Based on the Type, the Value can be safely cast. type LiteralNode struct { Value interface{} Type Type } + +func (n *LiteralNode) GoString() string { + return fmt.Sprintf("*%#v", *n) +} diff --git a/config/lang/lang.y b/config/lang/lang.y index f346799fd..4ea84f456 100644 --- a/config/lang/lang.y +++ b/config/lang/lang.y @@ -18,27 +18,47 @@ import ( } %token STRING IDENTIFIER PROGRAM_BRACKET_LEFT PROGRAM_BRACKET_RIGHT +%token PROGRAM_STRING_START PROGRAM_STRING_END %token PAREN_LEFT PAREN_RIGHT COMMA -%type expr interpolation literal +%type expr interpolation literal literalModeTop literalModeValue %type args %% top: - literal + literalModeTop { parserResult = $1 } + +literalModeTop: + literalModeValue + { + $$ = $1 + } +| literalModeTop literalModeValue + { + var result []ast.Node + if c, ok := $1.(*ast.Concat); ok { + result = append(c.Exprs, $2) + } else { + result = []ast.Node{$1, $2} + } + + $$ = &ast.Concat{ + Exprs: result, + } + } + +literalModeValue: + literal + { + $$ = $1 + } | interpolation { - parserResult = $1 - } -| literal interpolation - { - parserResult = &ast.Concat{ - Exprs: []ast.Node{$1, $2}, - } + $$ = $1 } interpolation: @@ -48,7 +68,7 @@ interpolation: } expr: - literal + literalModeTop { $$ = $1 } diff --git a/config/lang/lex.go b/config/lang/lex.go index 365001199..ff35d8ca5 100644 --- a/config/lang/lex.go +++ b/config/lang/lex.go @@ -18,13 +18,43 @@ type parserLex struct { Err error Input string + mode parserMode interpolationDepth int pos int width int } +// parserMode keeps track of what mode we're in for the parser. We have +// two modes: literal and interpolation. Literal mode is when strings +// don't have to be quoted, and interpolations are defined as ${foo}. +// Interpolation mode means that strings have to be quoted and unquoted +// things are identifiers, such as foo("bar"). +type parserMode uint8 + +const ( + parserModeInvalid parserMode = 0 + parserModeLiteral = 1 << iota + parserModeInterpolation +) + // The parser calls this method to get each new token. func (x *parserLex) Lex(yylval *parserSymType) int { + if x.mode == parserModeInvalid { + x.mode = parserModeLiteral + } + + switch x.mode { + case parserModeLiteral: + return x.lexModeLiteral(yylval) + case parserModeInterpolation: + return x.lexModeInterpolation(yylval) + default: + x.Error(fmt.Sprintf("Unknown parse mode: %s", x.mode)) + return lexEOF + } +} + +func (x *parserLex) lexModeLiteral(yylval *parserSymType) int { for { c := x.next() if c == lexEOF { @@ -35,14 +65,36 @@ func (x *parserLex) Lex(yylval *parserSymType) int { if c == '$' && x.peek() == '{' { x.next() x.interpolationDepth++ + x.mode = parserModeInterpolation return PROGRAM_BRACKET_LEFT } - if x.interpolationDepth == 0 { - // We're just a normal string that isn't part of any - // interpolation yet. - x.backup() - return x.lexString(yylval, false) + // We're just a normal string that isn't part of any interpolation yet. + x.backup() + result, terminated := x.lexString(yylval, x.interpolationDepth > 0) + + // If the string terminated and we're within an interpolation already + // then that means that we finished a nested string, so pop + // back out to interpolation mode. + if terminated && x.interpolationDepth > 0 { + x.mode = parserModeInterpolation + + // If the string is empty, just skip it. We're still in + // an interpolation so we do this to avoid empty nodes. + if yylval.str == "" { + return x.Lex(yylval) + } + } + + return result + } +} + +func (x *parserLex) lexModeInterpolation(yylval *parserSymType) int { + for { + c := x.next() + if c == lexEOF { + return lexEOF } // Ignore all whitespace @@ -53,12 +105,26 @@ func (x *parserLex) Lex(yylval *parserSymType) int { // If we see a double quote and we're in an interpolation, then // we are lexing a string. if c == '"' { - return x.lexString(yylval, true) + result, terminated := x.lexString(yylval, true) + if !terminated { + // The string didn't end, which means that we're in the + // middle of starting another interpolation. + x.mode = parserModeLiteral + + // If the string is empty and we're starting an interpolation, + // then just skip it to avoid empty string AST nodes + if yylval.str == "" { + return x.Lex(yylval) + } + } + + return result } switch c { case '}': x.interpolationDepth-- + x.mode = parserModeLiteral return PROGRAM_BRACKET_RIGHT case '(': return PAREN_LEFT @@ -103,11 +169,16 @@ func (x *parserLex) lexId(yylval *parserSymType) int { return IDENTIFIER } -func (x *parserLex) lexString(yylval *parserSymType, quoted bool) int { +func (x *parserLex) lexString(yylval *parserSymType, quoted bool) (int, bool) { var b bytes.Buffer + terminated := false for { c := x.next() if c == lexEOF { + if quoted { + x.Error("unterminated string") + } + break } @@ -115,6 +186,7 @@ func (x *parserLex) lexString(yylval *parserSymType, quoted bool) int { if quoted { // If its a double quote, we've reached the end of the string if c == '"' { + terminated = true break } @@ -148,12 +220,12 @@ func (x *parserLex) lexString(yylval *parserSymType, quoted bool) int { if _, err := b.WriteRune(c); err != nil { x.Error(err.Error()) - return lexEOF + return lexEOF, false } } yylval.str = b.String() - return STRING + return STRING, terminated } // Return the next rune for the lexer. diff --git a/config/lang/lex_test.go b/config/lang/lex_test.go index 3546bc885..ce6a36960 100644 --- a/config/lang/lex_test.go +++ b/config/lang/lex_test.go @@ -45,6 +45,40 @@ func TestLex(t *testing.T) { PAREN_RIGHT, PROGRAM_BRACKET_RIGHT, lexEOF}, }, + + { + "${bar(inner(baz))}", + []int{PROGRAM_BRACKET_LEFT, + IDENTIFIER, PAREN_LEFT, + IDENTIFIER, PAREN_LEFT, + IDENTIFIER, + PAREN_RIGHT, PAREN_RIGHT, + PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + "foo ${foo.bar.baz}", + []int{STRING, PROGRAM_BRACKET_LEFT, IDENTIFIER, PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + "foo ${foo.bar.*.baz}", + []int{STRING, PROGRAM_BRACKET_LEFT, IDENTIFIER, PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + "foo ${foo(\"baz\")}", + []int{STRING, PROGRAM_BRACKET_LEFT, + IDENTIFIER, PAREN_LEFT, STRING, PAREN_RIGHT, + PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + `foo ${"${var.foo}"}`, + []int{STRING, PROGRAM_BRACKET_LEFT, + PROGRAM_BRACKET_LEFT, IDENTIFIER, PROGRAM_BRACKET_RIGHT, + PROGRAM_BRACKET_RIGHT, lexEOF}, + }, } for _, tc := range cases { diff --git a/config/lang/parse_test.go b/config/lang/parse_test.go index 7e7114f57..a635be858 100644 --- a/config/lang/parse_test.go +++ b/config/lang/parse_test.go @@ -38,6 +38,26 @@ func TestParse(t *testing.T) { }, }, + { + "foo ${var.bar} baz", + false, + &ast.Concat{ + Exprs: []ast.Node{ + &ast.LiteralNode{ + Value: "foo ", + Type: ast.TypeString, + }, + &ast.VariableAccess{ + Name: "var.bar", + }, + &ast.LiteralNode{ + Value: " baz", + Type: ast.TypeString, + }, + }, + }, + }, + { "foo ${\"bar\"}", false, @@ -83,6 +103,60 @@ func TestParse(t *testing.T) { }, }, }, + + { + "${foo(bar(baz))}", + false, + &ast.Call{ + Func: "foo", + Args: []ast.Node{ + &ast.Call{ + Func: "bar", + Args: []ast.Node{ + &ast.VariableAccess{ + Name: "baz", + }, + }, + }, + }, + }, + }, + + { + `foo ${"bar ${baz}"}`, + false, + &ast.Concat{ + Exprs: []ast.Node{ + &ast.LiteralNode{ + Value: "foo ", + Type: ast.TypeString, + }, + &ast.Concat{ + Exprs: []ast.Node{ + &ast.LiteralNode{ + Value: "bar ", + Type: ast.TypeString, + }, + &ast.VariableAccess{ + Name: "baz", + }, + }, + }, + }, + }, + }, + + { + `foo ${bar ${baz}}`, + true, + nil, + }, + + { + `foo ${${baz}}`, + true, + nil, + }, } for _, tc := range cases { diff --git a/config/lang/y.go b/config/lang/y.go index a0ef3d105..173c1208c 100644 --- a/config/lang/y.go +++ b/config/lang/y.go @@ -20,15 +20,19 @@ const STRING = 57346 const IDENTIFIER = 57347 const PROGRAM_BRACKET_LEFT = 57348 const PROGRAM_BRACKET_RIGHT = 57349 -const PAREN_LEFT = 57350 -const PAREN_RIGHT = 57351 -const COMMA = 57352 +const PROGRAM_STRING_START = 57350 +const PROGRAM_STRING_END = 57351 +const PAREN_LEFT = 57352 +const PAREN_RIGHT = 57353 +const COMMA = 57354 var parserToknames = []string{ "STRING", "IDENTIFIER", "PROGRAM_BRACKET_LEFT", "PROGRAM_BRACKET_RIGHT", + "PROGRAM_STRING_START", + "PROGRAM_STRING_END", "PAREN_LEFT", "PAREN_RIGHT", "COMMA", @@ -39,7 +43,7 @@ const parserEofCode = 1 const parserErrCode = 2 const parserMaxDepth = 200 -//line lang.y:83 +//line lang.y:103 //line yacctab:1 var parserExca = []int{ @@ -48,47 +52,48 @@ var parserExca = []int{ -2, 0, } -const parserNprod = 12 +const parserNprod = 14 const parserPrivate = 57344 var parserTokenNames []string var parserStates []string -const parserLast = 18 +const parserLast = 21 var parserAct = []int{ - 7, 14, 15, 11, 10, 4, 5, 5, 4, 9, - 3, 1, 13, 6, 8, 2, 16, 12, + 9, 16, 17, 13, 3, 12, 1, 8, 6, 11, + 7, 6, 14, 7, 15, 8, 10, 2, 18, 4, + 5, } var parserPact = []int{ - 1, -1000, 0, -1000, -1000, 4, -1000, -3, -1000, -5, - -1000, 4, -8, -1000, -1000, 4, -1000, + 7, -1000, 7, -1000, -1000, -1000, -1000, 4, -1000, -2, + 7, -7, -1000, 4, -10, -1000, -1000, 4, -1000, } var parserPgo = []int{ - 0, 0, 10, 14, 17, 11, + 0, 0, 20, 19, 16, 4, 12, 6, } var parserR1 = []int{ - 0, 5, 5, 5, 2, 1, 1, 1, 4, 4, - 4, 3, + 0, 7, 4, 4, 5, 5, 2, 1, 1, 1, + 6, 6, 6, 3, } var parserR2 = []int{ - 0, 1, 1, 2, 3, 1, 1, 4, 0, 3, - 1, 1, + 0, 1, 1, 2, 1, 1, 3, 1, 1, 4, + 0, 3, 1, 1, } var parserChk = []int{ - -1000, -5, -3, -2, 4, 6, -2, -1, -3, 5, - 7, 8, -4, -1, 9, 10, -1, + -1000, -7, -4, -5, -3, -2, 4, 6, -5, -1, + -4, 5, 7, 10, -6, -1, 11, 12, -1, } var parserDef = []int{ - 0, -2, 1, 2, 11, 0, 3, 0, 5, 6, - 4, 8, 0, 10, 7, 0, 9, + 0, -2, 1, 2, 4, 5, 13, 0, 3, 0, + 7, 8, 6, 10, 0, 12, 9, 0, 11, } var parserTok1 = []int{ @@ -96,7 +101,8 @@ var parserTok1 = []int{ } var parserTok2 = []int{ - 2, 3, 4, 5, 6, 7, 8, 9, 10, + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, } var parserTok3 = []int{ 0, @@ -328,59 +334,76 @@ parserdefault: switch parsernt { case 1: - //line lang.y:30 + //line lang.y:31 { parserResult = parserS[parserpt-0].node } case 2: - //line lang.y:34 + //line lang.y:37 { - parserResult = parserS[parserpt-0].node + parserVAL.node = parserS[parserpt-0].node } case 3: - //line lang.y:38 + //line lang.y:41 { - parserResult = &ast.Concat{ - Exprs: []ast.Node{parserS[parserpt-1].node, parserS[parserpt-0].node}, + var result []ast.Node + if c, ok := parserS[parserpt-1].node.(*ast.Concat); ok { + result = append(c.Exprs, parserS[parserpt-0].node) + } else { + result = []ast.Node{parserS[parserpt-1].node, parserS[parserpt-0].node} + } + + parserVAL.node = &ast.Concat{ + Exprs: result, } } case 4: - //line lang.y:46 + //line lang.y:56 { - parserVAL.node = parserS[parserpt-1].node + parserVAL.node = parserS[parserpt-0].node } case 5: - //line lang.y:52 + //line lang.y:60 { parserVAL.node = parserS[parserpt-0].node } case 6: - //line lang.y:56 + //line lang.y:66 + { + parserVAL.node = parserS[parserpt-1].node + } + case 7: + //line lang.y:72 + { + parserVAL.node = parserS[parserpt-0].node + } + case 8: + //line lang.y:76 { parserVAL.node = &ast.VariableAccess{Name: parserS[parserpt-0].str} } - case 7: - //line lang.y:60 + case 9: + //line lang.y:80 { parserVAL.node = &ast.Call{Func: parserS[parserpt-3].str, Args: parserS[parserpt-1].nodeList} } - case 8: - //line lang.y:65 + case 10: + //line lang.y:85 { parserVAL.nodeList = nil } - case 9: - //line lang.y:69 + case 11: + //line lang.y:89 { parserVAL.nodeList = append(parserS[parserpt-2].nodeList, parserS[parserpt-0].node) } - case 10: - //line lang.y:73 + case 12: + //line lang.y:93 { parserVAL.nodeList = append(parserVAL.nodeList, parserS[parserpt-0].node) } - case 11: - //line lang.y:79 + case 13: + //line lang.y:99 { parserVAL.node = &ast.LiteralNode{Value: parserS[parserpt-0].str, Type: ast.TypeString} } diff --git a/config/lang/y.output b/config/lang/y.output index f098e493d..b17ca3ae5 100644 --- a/config/lang/y.output +++ b/config/lang/y.output @@ -2,12 +2,14 @@ state 0 $accept: .top $end - STRING shift 4 - PROGRAM_BRACKET_LEFT shift 5 + STRING shift 6 + PROGRAM_BRACKET_LEFT shift 7 . error - interpolation goto 3 - literal goto 2 + interpolation goto 5 + literal goto 4 + literalModeTop goto 2 + literalModeValue goto 3 top goto 1 state 1 @@ -18,127 +20,160 @@ state 1 state 2 - top: literal. (1) - top: literal.interpolation + top: literalModeTop. (1) + literalModeTop: literalModeTop.literalModeValue - PROGRAM_BRACKET_LEFT shift 5 - . reduce 1 (src line 28) + STRING shift 6 + PROGRAM_BRACKET_LEFT shift 7 + . reduce 1 (src line 29) - interpolation goto 6 + interpolation goto 5 + literal goto 4 + literalModeValue goto 8 state 3 - top: interpolation. (2) + literalModeTop: literalModeValue. (2) - . reduce 2 (src line 33) + . reduce 2 (src line 35) state 4 - literal: STRING. (11) + literalModeValue: literal. (4) - . reduce 11 (src line 77) + . reduce 4 (src line 54) state 5 - interpolation: PROGRAM_BRACKET_LEFT.expr PROGRAM_BRACKET_RIGHT + literalModeValue: interpolation. (5) - STRING shift 4 - IDENTIFIER shift 9 - . error + . reduce 5 (src line 59) - expr goto 7 - literal goto 8 state 6 - top: literal interpolation. (3) + literal: STRING. (13) - . reduce 3 (src line 37) + . reduce 13 (src line 97) state 7 - interpolation: PROGRAM_BRACKET_LEFT expr.PROGRAM_BRACKET_RIGHT + interpolation: PROGRAM_BRACKET_LEFT.expr PROGRAM_BRACKET_RIGHT - PROGRAM_BRACKET_RIGHT shift 10 + STRING shift 6 + IDENTIFIER shift 11 + PROGRAM_BRACKET_LEFT shift 7 . error + expr goto 9 + interpolation goto 5 + literal goto 4 + literalModeTop goto 10 + literalModeValue goto 3 state 8 - expr: literal. (5) + literalModeTop: literalModeTop literalModeValue. (3) - . reduce 5 (src line 50) + . reduce 3 (src line 40) state 9 - expr: IDENTIFIER. (6) - expr: IDENTIFIER.PAREN_LEFT args PAREN_RIGHT + interpolation: PROGRAM_BRACKET_LEFT expr.PROGRAM_BRACKET_RIGHT - PAREN_LEFT shift 11 - . reduce 6 (src line 55) + PROGRAM_BRACKET_RIGHT shift 12 + . error state 10 - interpolation: PROGRAM_BRACKET_LEFT expr PROGRAM_BRACKET_RIGHT. (4) + literalModeTop: literalModeTop.literalModeValue + expr: literalModeTop. (7) - . reduce 4 (src line 44) + STRING shift 6 + PROGRAM_BRACKET_LEFT shift 7 + . reduce 7 (src line 70) + interpolation goto 5 + literal goto 4 + literalModeValue goto 8 state 11 - expr: IDENTIFIER PAREN_LEFT.args PAREN_RIGHT - args: . (8) + expr: IDENTIFIER. (8) + expr: IDENTIFIER.PAREN_LEFT args PAREN_RIGHT - STRING shift 4 - IDENTIFIER shift 9 - . reduce 8 (src line 64) + PAREN_LEFT shift 13 + . reduce 8 (src line 75) - expr goto 13 - literal goto 8 - args goto 12 state 12 - expr: IDENTIFIER PAREN_LEFT args.PAREN_RIGHT - args: args.COMMA expr + interpolation: PROGRAM_BRACKET_LEFT expr PROGRAM_BRACKET_RIGHT. (6) - PAREN_RIGHT shift 14 - COMMA shift 15 - . error + . reduce 6 (src line 64) state 13 - args: expr. (10) + expr: IDENTIFIER PAREN_LEFT.args PAREN_RIGHT + args: . (10) - . reduce 10 (src line 72) + STRING shift 6 + IDENTIFIER shift 11 + PROGRAM_BRACKET_LEFT shift 7 + . reduce 10 (src line 84) + expr goto 15 + interpolation goto 5 + literal goto 4 + literalModeTop goto 10 + literalModeValue goto 3 + args goto 14 state 14 - expr: IDENTIFIER PAREN_LEFT args PAREN_RIGHT. (7) + expr: IDENTIFIER PAREN_LEFT args.PAREN_RIGHT + args: args.COMMA expr - . reduce 7 (src line 59) + PAREN_RIGHT shift 16 + COMMA shift 17 + . error state 15 - args: args COMMA.expr + args: expr. (12) - STRING shift 4 - IDENTIFIER shift 9 - . error + . reduce 12 (src line 92) - expr goto 16 - literal goto 8 state 16 - args: args COMMA expr. (9) + expr: IDENTIFIER PAREN_LEFT args PAREN_RIGHT. (9) - . reduce 9 (src line 68) + . reduce 9 (src line 79) -10 terminals, 6 nonterminals -12 grammar rules, 17/2000 states +state 17 + args: args COMMA.expr + + STRING shift 6 + IDENTIFIER shift 11 + PROGRAM_BRACKET_LEFT shift 7 + . error + + expr goto 18 + interpolation goto 5 + literal goto 4 + literalModeTop goto 10 + literalModeValue goto 3 + +state 18 + args: args COMMA expr. (11) + + . reduce 11 (src line 88) + + +12 terminals, 8 nonterminals +14 grammar rules, 19/2000 states 0 shift/reduce, 0 reduce/reduce conflicts reported -55 working sets used -memory: parser 11/30000 -6 extra closures -13 shift entries, 1 exceptions -9 goto entries -2 entries saved by goto default -Optimizer space used: output 18/30000 -18 table entries, 0 zero -maximum spread: 10, maximum offset: 15 +57 working sets used +memory: parser 25/30000 +14 extra closures +19 shift entries, 1 exceptions +12 goto entries +15 entries saved by goto default +Optimizer space used: output 21/30000 +21 table entries, 0 zero +maximum spread: 12, maximum offset: 17 From fa7891d182a04f1e02bd98d20ecabea151571cf4 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 11 Jan 2015 14:35:14 -0800 Subject: [PATCH 04/31] config/lang/ast: use stringer to generate enums --- config/lang/ast/ast.go | 2 ++ config/lang/ast/type_string.go | 17 +++++++++++++++++ config/lang/lex_test.go | 8 -------- 3 files changed, 19 insertions(+), 8 deletions(-) create mode 100644 config/lang/ast/type_string.go diff --git a/config/lang/ast/ast.go b/config/lang/ast/ast.go index 3bb748ccc..c9ae1b882 100644 --- a/config/lang/ast/ast.go +++ b/config/lang/ast/ast.go @@ -3,6 +3,8 @@ package ast // Node is the interface that all AST nodes must implement. type Node interface{} +//go:generate stringer -type=Type + // Type is the type of a literal. type Type uint diff --git a/config/lang/ast/type_string.go b/config/lang/ast/type_string.go new file mode 100644 index 000000000..59c336f5a --- /dev/null +++ b/config/lang/ast/type_string.go @@ -0,0 +1,17 @@ +// generated by stringer -type=Type; DO NOT EDIT + +package ast + +import "fmt" + +const _Type_name = "TypeInvalidTypeString" + +var _Type_index = [...]uint8{0, 11, 21} + +func (i Type) String() string { + i -= 1 + if i+1 >= Type(len(_Type_index)) { + return fmt.Sprintf("Type(%d)", i+1) + } + return _Type_name[_Type_index[i]:_Type_index[i+1]] +} diff --git a/config/lang/lex_test.go b/config/lang/lex_test.go index ce6a36960..b5cc7c583 100644 --- a/config/lang/lex_test.go +++ b/config/lang/lex_test.go @@ -105,11 +105,3 @@ func TestLex(t *testing.T) { } } } - -/* OTHERS: - -foo ${var.foo} -bar ${"hello"} -foo ${concat("foo ${var.bar}", var.baz)} - -*/ From 4ae8cae9e70863635796e78cc6d6bf3d823a6140 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 11 Jan 2015 15:26:54 -0800 Subject: [PATCH 05/31] config/lang: execution --- config/lang/ast/ast.go | 17 +++- config/lang/ast/call.go | 8 ++ config/lang/ast/concat.go | 8 ++ config/lang/ast/literal.go | 4 + config/lang/ast/type_string.go | 7 +- config/lang/ast/variable_access.go | 4 + config/lang/engine.go | 148 +++++++++++++++++++++++++++++ config/lang/engine_test.go | 59 ++++++++++++ 8 files changed, 248 insertions(+), 7 deletions(-) create mode 100644 config/lang/engine.go create mode 100644 config/lang/engine_test.go diff --git a/config/lang/ast/ast.go b/config/lang/ast/ast.go index c9ae1b882..45c3ddc77 100644 --- a/config/lang/ast/ast.go +++ b/config/lang/ast/ast.go @@ -1,7 +1,18 @@ package ast // Node is the interface that all AST nodes must implement. -type Node interface{} +type Node interface { + // Accept is called to dispatch to the visitors. + Accept(Visitor) +} + +// Visitors are just implementations of this function. +// +// Note that this isn't a true implementation of the visitor pattern, which +// generally requires proper type dispatch on the function. However, +// implementing this basic visitor pattern style is still very useful even +// if you have to type switch. +type Visitor func(Node) //go:generate stringer -type=Type @@ -9,6 +20,6 @@ type Node interface{} type Type uint const ( - TypeInvalid Type = 1 << iota - TypeString + TypeInvalid Type = 0 + TypeString = 1 << iota ) diff --git a/config/lang/ast/call.go b/config/lang/ast/call.go index 35fc03f50..6f2854799 100644 --- a/config/lang/ast/call.go +++ b/config/lang/ast/call.go @@ -5,3 +5,11 @@ type Call struct { Func string Args []Node } + +func (n *Call) Accept(v Visitor) { + for _, a := range n.Args { + a.Accept(v) + } + + v(n) +} diff --git a/config/lang/ast/concat.go b/config/lang/ast/concat.go index c88e3de5a..77f7181ab 100644 --- a/config/lang/ast/concat.go +++ b/config/lang/ast/concat.go @@ -10,6 +10,14 @@ type Concat struct { Exprs []Node } +func (n *Concat) Accept(v Visitor) { + for _, n := range n.Exprs { + n.Accept(v) + } + + v(n) +} + func (n *Concat) GoString() string { return fmt.Sprintf("*%#v", *n) } diff --git a/config/lang/ast/literal.go b/config/lang/ast/literal.go index 3477a5dbd..8b7e2b88c 100644 --- a/config/lang/ast/literal.go +++ b/config/lang/ast/literal.go @@ -11,6 +11,10 @@ type LiteralNode struct { Type Type } +func (n *LiteralNode) Accept(v Visitor) { + v(n) +} + func (n *LiteralNode) GoString() string { return fmt.Sprintf("*%#v", *n) } diff --git a/config/lang/ast/type_string.go b/config/lang/ast/type_string.go index 59c336f5a..55e2af017 100644 --- a/config/lang/ast/type_string.go +++ b/config/lang/ast/type_string.go @@ -4,14 +4,13 @@ package ast import "fmt" -const _Type_name = "TypeInvalidTypeString" +const _Type_name = "TypeInvalid" -var _Type_index = [...]uint8{0, 11, 21} +var _Type_index = [...]uint8{0, 11} func (i Type) String() string { - i -= 1 if i+1 >= Type(len(_Type_index)) { - return fmt.Sprintf("Type(%d)", i+1) + return fmt.Sprintf("Type(%d)", i) } return _Type_name[_Type_index[i]:_Type_index[i+1]] } diff --git a/config/lang/ast/variable_access.go b/config/lang/ast/variable_access.go index 5d63dcba1..987df97d2 100644 --- a/config/lang/ast/variable_access.go +++ b/config/lang/ast/variable_access.go @@ -9,6 +9,10 @@ type VariableAccess struct { Name string } +func (n *VariableAccess) Accept(v Visitor) { + v(n) +} + func (n *VariableAccess) GoString() string { return fmt.Sprintf("*%#v", *n) } diff --git a/config/lang/engine.go b/config/lang/engine.go new file mode 100644 index 000000000..0fd8f7eac --- /dev/null +++ b/config/lang/engine.go @@ -0,0 +1,148 @@ +package lang + +import ( + "bytes" + "fmt" + "sync" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +// Engine is the execution engine for this language. It should be configured +// prior to running Execute. +type Engine struct { + // VarMap and FuncMap are the mappings of identifiers to functions + // and variable values. + VarMap map[string]Variable + FuncMap map[string]Function + + // SemanticChecks is a list of additional semantic checks that will be run + // on the tree prior to executing it. The type checker, identifier checker, + // etc. will be run before these. + SemanticChecks []SemanticChecker +} + +// SemanticChecker is the type that must be implemented to do a +// semantic check on an AST tree. This will be called with the root node. +type SemanticChecker func(ast.Node) error + +// Variable is a variable value for execution given as input to the engine. +// It records the value of a variables along with their type. +type Variable struct { + Value interface{} + Type ast.Type +} + +// Function defines a function that can be executed by the engine. +// The type checker will validate that the proper types will be called +// to the callback. +type Function struct { + Name string + ArgTypes []ast.Type + Callback func([]interface{}) (interface{}, ast.Type, error) +} + +// Execute executes the given ast.Node and returns its final value, its +// type, and an error if one exists. +func (e *Engine) Execute(root ast.Node) (interface{}, ast.Type, error) { + v := &executeVisitor{Engine: e} + return v.Visit(root) +} + +// executeVisitor is the visitor used to do the actual execution of +// a program. Note at this point it is assumed that the types check out +// and the identifiers exist. +type executeVisitor struct { + Engine *Engine + + stack []*ast.LiteralNode + err error + lock sync.Mutex +} + +func (v *executeVisitor) Visit(root ast.Node) (interface{}, ast.Type, error) { + v.lock.Lock() + defer v.lock.Unlock() + + // Run the actual visitor pattern + root.Accept(v.visit) + + // Get our result and clear out everything else + var result *ast.LiteralNode + if len(v.stack) > 0 { + result = v.stack[len(v.stack)-1] + } else { + result = new(ast.LiteralNode) + } + resultErr := v.err + + // Clear everything else so we aren't just dangling + v.stack = nil + v.err = nil + + return result.Value, result.Type, resultErr +} + +func (v *executeVisitor) visit(raw ast.Node) { + if v.err != nil { + return + } + + switch n := raw.(type) { + case *ast.Concat: + v.visitConcat(n) + case *ast.LiteralNode: + v.visitLiteral(n) + case *ast.VariableAccess: + v.visitVariableAccess(n) + default: + v.err = fmt.Errorf("unknown node: %#v", raw) + } +} + +func (v *executeVisitor) visitConcat(n *ast.Concat) { + // The expressions should all be on the stack in reverse + // order. So pop them off, reverse their order, and concatenate. + nodes := make([]*ast.LiteralNode, 0, len(n.Exprs)) + for range n.Exprs { + nodes = append(nodes, v.stackPop()) + } + + var buf bytes.Buffer + for i := len(nodes) - 1; i >= 0; i-- { + buf.WriteString(nodes[i].Value.(string)) + } + + v.stackPush(&ast.LiteralNode{ + Value: buf.String(), + Type: ast.TypeString, + }) +} + +func (v *executeVisitor) visitLiteral(n *ast.LiteralNode) { + v.stack = append(v.stack, n) +} + +func (v *executeVisitor) visitVariableAccess(n *ast.VariableAccess) { + // Look up the variable in the map + variable, ok := v.Engine.VarMap[n.Name] + if !ok { + v.err = fmt.Errorf("unknown variable accessed: %s", n.Name) + return + } + + v.stack = append(v.stack, &ast.LiteralNode{ + Value: variable.Value, + Type: variable.Type, + }) +} + +func (v *executeVisitor) stackPush(n *ast.LiteralNode) { + v.stack = append(v.stack, n) +} + +func (v *executeVisitor) stackPop() *ast.LiteralNode { + var x *ast.LiteralNode + x, v.stack = v.stack[len(v.stack)-1], v.stack[:len(v.stack)-1] + return x +} diff --git a/config/lang/engine_test.go b/config/lang/engine_test.go new file mode 100644 index 000000000..489758a6e --- /dev/null +++ b/config/lang/engine_test.go @@ -0,0 +1,59 @@ +package lang + +import ( + "reflect" + "testing" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +func TestEngineExecute(t *testing.T) { + cases := []struct { + Input string + Engine *Engine + Error bool + Result interface{} + ResultType ast.Type + }{ + { + "foo", + &Engine{}, + false, + "foo", + ast.TypeString, + }, + + { + "foo ${bar}", + &Engine{ + VarMap: map[string]Variable{ + "bar": Variable{ + Value: "baz", + Type: ast.TypeString, + }, + }, + }, + false, + "foo baz", + ast.TypeString, + }, + } + + for _, tc := range cases { + node, err := Parse(tc.Input) + if err != nil { + t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) + } + + out, outType, err := tc.Engine.Execute(node) + if (err != nil) != tc.Error { + t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) + } + if outType != tc.ResultType { + t.Fatalf("Bad: %s\n\nInput: %s", outType, tc.Input) + } + if !reflect.DeepEqual(out, tc.Result) { + t.Fatalf("Bad: %#v\n\nInput: %s", out, tc.Input) + } + } +} From 8f925b93e08581994ec2506864c3456027a24d6c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 11 Jan 2015 15:33:24 -0800 Subject: [PATCH 06/31] config/lang: function calls work --- config/lang/engine.go | 37 ++++++++++++++++++++++++++++++++++--- config/lang/engine_test.go | 17 +++++++++++++++++ config/lang/parse_test.go | 9 +++++++++ 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/config/lang/engine.go b/config/lang/engine.go index 0fd8f7eac..0b3a6c14d 100644 --- a/config/lang/engine.go +++ b/config/lang/engine.go @@ -37,9 +37,9 @@ type Variable struct { // The type checker will validate that the proper types will be called // to the callback. type Function struct { - Name string - ArgTypes []ast.Type - Callback func([]interface{}) (interface{}, ast.Type, error) + ArgTypes []ast.Type + ReturnType ast.Type + Callback func([]interface{}) (interface{}, error) } // Execute executes the given ast.Node and returns its final value, its @@ -89,6 +89,8 @@ func (v *executeVisitor) visit(raw ast.Node) { } switch n := raw.(type) { + case *ast.Call: + v.visitCall(n) case *ast.Concat: v.visitConcat(n) case *ast.LiteralNode: @@ -100,6 +102,35 @@ func (v *executeVisitor) visit(raw ast.Node) { } } +func (v *executeVisitor) visitCall(n *ast.Call) { + // Look up the function in the map + function, ok := v.Engine.FuncMap[n.Func] + if !ok { + v.err = fmt.Errorf("unknown function called: %s", n.Func) + return + } + + // The arguments are on the stack in reverse order, so pop them off. + args := make([]interface{}, len(n.Args)) + for i, _ := range n.Args { + node := v.stackPop() + args[len(n.Args)-1-i] = node.Value + } + + // Call the function + result, err := function.Callback(args) + if err != nil { + v.err = fmt.Errorf("%s: %s", n.Func, err) + return + } + + // Push the result + v.stackPush(&ast.LiteralNode{ + Value: result, + Type: function.ReturnType, + }) +} + func (v *executeVisitor) visitConcat(n *ast.Concat) { // The expressions should all be on the stack in reverse // order. So pop them off, reverse their order, and concatenate. diff --git a/config/lang/engine_test.go b/config/lang/engine_test.go index 489758a6e..96a1105d5 100644 --- a/config/lang/engine_test.go +++ b/config/lang/engine_test.go @@ -37,6 +37,23 @@ func TestEngineExecute(t *testing.T) { "foo baz", ast.TypeString, }, + + { + "foo ${rand()}", + &Engine{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + "foo 42", + ast.TypeString, + }, } for _, tc := range cases { diff --git a/config/lang/parse_test.go b/config/lang/parse_test.go index a635be858..ec508b0e2 100644 --- a/config/lang/parse_test.go +++ b/config/lang/parse_test.go @@ -75,6 +75,15 @@ func TestParse(t *testing.T) { }, }, + { + "${foo()}", + false, + &ast.Call{ + Func: "foo", + Args: nil, + }, + }, + { "${foo(bar)}", false, From d3b10104440e53a0603e9d96954da016fa2664c7 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 11 Jan 2015 23:38:21 -0800 Subject: [PATCH 07/31] config/lang: start implementing type checking --- config/lang/ast/ast.go | 1 + config/lang/visitor_types.go | 100 ++++++++++++++++++++++++++++++ config/lang/visitor_types_test.go | 74 ++++++++++++++++++++++ 3 files changed, 175 insertions(+) create mode 100644 config/lang/visitor_types.go create mode 100644 config/lang/visitor_types_test.go diff --git a/config/lang/ast/ast.go b/config/lang/ast/ast.go index 45c3ddc77..32655e1cb 100644 --- a/config/lang/ast/ast.go +++ b/config/lang/ast/ast.go @@ -22,4 +22,5 @@ type Type uint const ( TypeInvalid Type = 0 TypeString = 1 << iota + TypeInt ) diff --git a/config/lang/visitor_types.go b/config/lang/visitor_types.go new file mode 100644 index 000000000..f06a932d1 --- /dev/null +++ b/config/lang/visitor_types.go @@ -0,0 +1,100 @@ +package lang + +import ( + "fmt" + "sync" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +// TypeVisitor implements ast.Visitor for type checking an AST tree. +// It requires some configuration to look up the type of nodes. +type TypeVisitor struct { + VarMap map[string]Variable + FuncMap map[string]Function + + stack []ast.Type + err error + lock sync.Mutex +} + +func (v *TypeVisitor) Visit(root ast.Node) error { + v.lock.Lock() + defer v.lock.Unlock() + defer v.reset() + root.Accept(v.visit) + return v.err +} + +func (v *TypeVisitor) visit(raw ast.Node) { + if v.err != nil { + return + } + + switch n := raw.(type) { + case *ast.Call: + v.visitCall(n) + case *ast.Concat: + v.visitConcat(n) + case *ast.LiteralNode: + v.visitLiteral(n) + case *ast.VariableAccess: + v.visitVariableAccess(n) + default: + v.err = fmt.Errorf("unknown node: %#v", raw) + } +} + +func (v *TypeVisitor) visitCall(n *ast.Call) { + // TODO + v.stackPush(ast.TypeString) +} + +func (v *TypeVisitor) visitConcat(n *ast.Concat) { + types := make([]ast.Type, len(n.Exprs)) + for i, _ := range n.Exprs { + types[len(n.Exprs)-1-i] = v.stackPop() + } + + // All concat args must be strings, so validate that + for i, t := range types { + if t != ast.TypeString { + v.err = fmt.Errorf("%s: argument %d must be a sting", n, i+1) + return + } + } + + // This always results in type string + v.stackPush(ast.TypeString) +} + +func (v *TypeVisitor) visitLiteral(n *ast.LiteralNode) { + v.stackPush(n.Type) +} + +func (v *TypeVisitor) visitVariableAccess(n *ast.VariableAccess) { + // Look up the variable in the map + variable, ok := v.VarMap[n.Name] + if !ok { + v.err = fmt.Errorf("unknown variable accessed: %s", n.Name) + return + } + + // Add the type to the stack + v.stackPush(variable.Type) +} + +func (v *TypeVisitor) reset() { + v.stack = nil + v.err = nil +} + +func (v *TypeVisitor) stackPush(t ast.Type) { + v.stack = append(v.stack, t) +} + +func (v *TypeVisitor) stackPop() ast.Type { + var x ast.Type + x, v.stack = v.stack[len(v.stack)-1], v.stack[:len(v.stack)-1] + return x +} diff --git a/config/lang/visitor_types_test.go b/config/lang/visitor_types_test.go new file mode 100644 index 000000000..92f81e023 --- /dev/null +++ b/config/lang/visitor_types_test.go @@ -0,0 +1,74 @@ +package lang + +import ( + "testing" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +func TestTypeVisitor(t *testing.T) { + cases := []struct { + Input string + Visitor *TypeVisitor + Error bool + }{ + { + "foo", + &TypeVisitor{}, + false, + }, + + { + "foo ${bar}", + &TypeVisitor{ + VarMap: map[string]Variable{ + "bar": Variable{ + Value: "baz", + Type: ast.TypeString, + }, + }, + }, + false, + }, + + { + "foo ${rand()}", + &TypeVisitor{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, + + { + "foo ${bar}", + &TypeVisitor{ + VarMap: map[string]Variable{ + "bar": Variable{ + Value: 42, + Type: ast.TypeInt, + }, + }, + }, + true, + }, + } + + for _, tc := range cases { + node, err := Parse(tc.Input) + if err != nil { + t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) + } + + err = tc.Visitor.Visit(node) + if (err != nil) != tc.Error { + t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) + } + } +} From 662760da11ddbb818d70fb047d19439b0b7ac22c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 12 Jan 2015 00:28:47 -0800 Subject: [PATCH 08/31] config/lang: have position in AST --- config/lang/ast/ast.go | 16 ++++++ config/lang/ast/call.go | 5 ++ config/lang/ast/concat.go | 5 ++ config/lang/ast/literal.go | 5 ++ config/lang/ast/variable_access.go | 5 ++ config/lang/lang.y | 16 ++++-- config/lang/lex.go | 62 ++++++++++++++++++--- config/lang/parse_test.go | 25 +++++++++ config/lang/visitor_types.go | 12 +++-- config/lang/y.go | 86 ++++++++++++++++-------------- config/lang/y.output | 44 +++++++-------- 11 files changed, 206 insertions(+), 75 deletions(-) diff --git a/config/lang/ast/ast.go b/config/lang/ast/ast.go index 32655e1cb..343deb703 100644 --- a/config/lang/ast/ast.go +++ b/config/lang/ast/ast.go @@ -1,9 +1,25 @@ package ast +import ( + "fmt" +) + // Node is the interface that all AST nodes must implement. type Node interface { // Accept is called to dispatch to the visitors. Accept(Visitor) + + // Pos returns the position of this node in some source. + Pos() Pos +} + +// Pos is the starting position of an AST node +type Pos struct { + Column, Line int // Column/Line number, starting at 1 +} + +func (p *Pos) String() string { + return fmt.Sprintf("%d:%d", p.Line, p.Column) } // Visitors are just implementations of this function. diff --git a/config/lang/ast/call.go b/config/lang/ast/call.go index 6f2854799..d70219983 100644 --- a/config/lang/ast/call.go +++ b/config/lang/ast/call.go @@ -4,6 +4,7 @@ package ast type Call struct { Func string Args []Node + Posx Pos } func (n *Call) Accept(v Visitor) { @@ -13,3 +14,7 @@ func (n *Call) Accept(v Visitor) { v(n) } + +func (n *Call) Pos() Pos { + return n.Posx +} diff --git a/config/lang/ast/concat.go b/config/lang/ast/concat.go index 77f7181ab..9d3d998ee 100644 --- a/config/lang/ast/concat.go +++ b/config/lang/ast/concat.go @@ -8,6 +8,7 @@ import ( // concatenated. The result of all expressions must be a string. type Concat struct { Exprs []Node + Posx Pos } func (n *Concat) Accept(v Visitor) { @@ -18,6 +19,10 @@ func (n *Concat) Accept(v Visitor) { v(n) } +func (n *Concat) Pos() Pos { + return n.Posx +} + func (n *Concat) GoString() string { return fmt.Sprintf("*%#v", *n) } diff --git a/config/lang/ast/literal.go b/config/lang/ast/literal.go index 8b7e2b88c..a20316ac1 100644 --- a/config/lang/ast/literal.go +++ b/config/lang/ast/literal.go @@ -9,12 +9,17 @@ import ( type LiteralNode struct { Value interface{} Type Type + Posx Pos } func (n *LiteralNode) Accept(v Visitor) { v(n) } +func (n *LiteralNode) Pos() Pos { + return n.Posx +} + func (n *LiteralNode) GoString() string { return fmt.Sprintf("*%#v", *n) } diff --git a/config/lang/ast/variable_access.go b/config/lang/ast/variable_access.go index 987df97d2..943ee5140 100644 --- a/config/lang/ast/variable_access.go +++ b/config/lang/ast/variable_access.go @@ -7,12 +7,17 @@ import ( // VariableAccess represents a variable access. type VariableAccess struct { Name string + Posx Pos } func (n *VariableAccess) Accept(v Visitor) { v(n) } +func (n *VariableAccess) Pos() Pos { + return n.Posx +} + func (n *VariableAccess) GoString() string { return fmt.Sprintf("*%#v", *n) } diff --git a/config/lang/lang.y b/config/lang/lang.y index 4ea84f456..d379a6a09 100644 --- a/config/lang/lang.y +++ b/config/lang/lang.y @@ -15,12 +15,15 @@ import ( node ast.Node nodeList []ast.Node str string + token *parserToken } -%token STRING IDENTIFIER PROGRAM_BRACKET_LEFT PROGRAM_BRACKET_RIGHT +%token PROGRAM_BRACKET_LEFT PROGRAM_BRACKET_RIGHT %token PROGRAM_STRING_START PROGRAM_STRING_END %token PAREN_LEFT PAREN_RIGHT COMMA +%token IDENTIFIER STRING + %type expr interpolation literal literalModeTop literalModeValue %type args @@ -48,6 +51,7 @@ literalModeTop: $$ = &ast.Concat{ Exprs: result, + Posx: result[0].Pos(), } } @@ -74,11 +78,11 @@ expr: } | IDENTIFIER { - $$ = &ast.VariableAccess{Name: $1} + $$ = &ast.VariableAccess{Name: $1.Value.(string), Posx: $1.Pos} } | IDENTIFIER PAREN_LEFT args PAREN_RIGHT { - $$ = &ast.Call{Func: $1, Args: $3} + $$ = &ast.Call{Func: $1.Value.(string), Args: $3, Posx: $1.Pos} } args: @@ -97,7 +101,11 @@ args: literal: STRING { - $$ = &ast.LiteralNode{Value: $1, Type: ast.TypeString} + $$ = &ast.LiteralNode{ + Value: $1.Value.(string), + Type: ast.TypeString, + Posx: $1.Pos, + } } %% diff --git a/config/lang/lex.go b/config/lang/lex.go index ff35d8ca5..a4ee94b2a 100644 --- a/config/lang/lex.go +++ b/config/lang/lex.go @@ -5,6 +5,8 @@ import ( "fmt" "unicode" "unicode/utf8" + + "github.com/hashicorp/terraform/config/lang/ast" ) //go:generate go tool yacc -p parser lang.y @@ -22,6 +24,17 @@ type parserLex struct { interpolationDepth int pos int width int + col, line int + lastLine int + astPos *ast.Pos +} + +// parserToken is the token yielded to the parser. The value can be +// determined within the parser type based on the enum value returned +// from Lex. +type parserToken struct { + Value interface{} + Pos ast.Pos } // parserMode keeps track of what mode we're in for the parser. We have @@ -43,6 +56,17 @@ func (x *parserLex) Lex(yylval *parserSymType) int { x.mode = parserModeLiteral } + defer func() { + if yylval.token != nil && yylval.token.Pos.Column == 0 { + yylval.token.Pos = *x.astPos + } + }() + + x.astPos = nil + return x.lex(yylval) +} + +func (x *parserLex) lex(yylval *parserSymType) int { switch x.mode { case parserModeLiteral: return x.lexModeLiteral(yylval) @@ -81,8 +105,8 @@ func (x *parserLex) lexModeLiteral(yylval *parserSymType) int { // If the string is empty, just skip it. We're still in // an interpolation so we do this to avoid empty nodes. - if yylval.str == "" { - return x.Lex(yylval) + if yylval.token.Value.(string) == "" { + return x.lex(yylval) } } @@ -113,8 +137,8 @@ func (x *parserLex) lexModeInterpolation(yylval *parserSymType) int { // If the string is empty and we're starting an interpolation, // then just skip it to avoid empty string AST nodes - if yylval.str == "" { - return x.Lex(yylval) + if yylval.token.Value.(string) == "" { + return x.lex(yylval) } } @@ -165,7 +189,7 @@ func (x *parserLex) lexId(yylval *parserSymType) int { } } - yylval.str = b.String() + yylval.token = &parserToken{Value: b.String()} return IDENTIFIER } @@ -224,7 +248,7 @@ func (x *parserLex) lexString(yylval *parserSymType, quoted bool) (int, bool) { } } - yylval.str = b.String() + yylval.token = &parserToken{Value: b.String()} return STRING, terminated } @@ -238,6 +262,24 @@ func (x *parserLex) next() rune { r, w := utf8.DecodeRuneInString(x.Input[x.pos:]) x.width = w x.pos += x.width + + if x.line == 0 { + x.line = 1 + x.col = 1 + } else { + x.col += 1 + } + + if r == '\n' { + x.lastLine = x.col + x.line += 1 + x.col = 1 + } + + if x.astPos == nil { + x.astPos = &ast.Pos{Column: x.col, Line: x.line} + } + return r } @@ -251,6 +293,14 @@ func (x *parserLex) peek() rune { // backup steps back one rune. Can only be called once per next. func (x *parserLex) backup() { x.pos -= x.width + x.col -= 1 + + // If we are at column 0, we're backing up across a line boundary + // so we need to be careful to get the proper value. + if x.col == 0 { + x.col = x.lastLine + x.line -= 1 + } } // The parser calls this method on a parse error. diff --git a/config/lang/parse_test.go b/config/lang/parse_test.go index ec508b0e2..9d17072e8 100644 --- a/config/lang/parse_test.go +++ b/config/lang/parse_test.go @@ -19,6 +19,7 @@ func TestParse(t *testing.T) { &ast.LiteralNode{ Value: "foo", Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, }, }, @@ -26,13 +27,16 @@ func TestParse(t *testing.T) { "foo ${var.bar}", false, &ast.Concat{ + Posx: ast.Pos{Column: 1, Line: 1}, Exprs: []ast.Node{ &ast.LiteralNode{ Value: "foo ", Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, }, &ast.VariableAccess{ Name: "var.bar", + Posx: ast.Pos{Column: 7, Line: 1}, }, }, }, @@ -42,17 +46,21 @@ func TestParse(t *testing.T) { "foo ${var.bar} baz", false, &ast.Concat{ + Posx: ast.Pos{Column: 1, Line: 1}, Exprs: []ast.Node{ &ast.LiteralNode{ Value: "foo ", Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, }, &ast.VariableAccess{ Name: "var.bar", + Posx: ast.Pos{Column: 7, Line: 1}, }, &ast.LiteralNode{ Value: " baz", Type: ast.TypeString, + Posx: ast.Pos{Column: 15, Line: 1}, }, }, }, @@ -62,14 +70,17 @@ func TestParse(t *testing.T) { "foo ${\"bar\"}", false, &ast.Concat{ + Posx: ast.Pos{Column: 1, Line: 1}, Exprs: []ast.Node{ &ast.LiteralNode{ Value: "foo ", Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, }, &ast.LiteralNode{ Value: "bar", Type: ast.TypeString, + Posx: ast.Pos{Column: 7, Line: 1}, }, }, }, @@ -81,6 +92,7 @@ func TestParse(t *testing.T) { &ast.Call{ Func: "foo", Args: nil, + Posx: ast.Pos{Column: 3, Line: 1}, }, }, @@ -89,9 +101,11 @@ func TestParse(t *testing.T) { false, &ast.Call{ Func: "foo", + Posx: ast.Pos{Column: 3, Line: 1}, Args: []ast.Node{ &ast.VariableAccess{ Name: "bar", + Posx: ast.Pos{Column: 7, Line: 1}, }, }, }, @@ -102,12 +116,15 @@ func TestParse(t *testing.T) { false, &ast.Call{ Func: "foo", + Posx: ast.Pos{Column: 3, Line: 1}, Args: []ast.Node{ &ast.VariableAccess{ Name: "bar", + Posx: ast.Pos{Column: 7, Line: 1}, }, &ast.VariableAccess{ Name: "baz", + Posx: ast.Pos{Column: 11, Line: 1}, }, }, }, @@ -118,12 +135,15 @@ func TestParse(t *testing.T) { false, &ast.Call{ Func: "foo", + Posx: ast.Pos{Column: 3, Line: 1}, Args: []ast.Node{ &ast.Call{ Func: "bar", + Posx: ast.Pos{Column: 7, Line: 1}, Args: []ast.Node{ &ast.VariableAccess{ Name: "baz", + Posx: ast.Pos{Column: 11, Line: 1}, }, }, }, @@ -135,19 +155,24 @@ func TestParse(t *testing.T) { `foo ${"bar ${baz}"}`, false, &ast.Concat{ + Posx: ast.Pos{Column: 1, Line: 1}, Exprs: []ast.Node{ &ast.LiteralNode{ Value: "foo ", Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, }, &ast.Concat{ + Posx: ast.Pos{Column: 7, Line: 1}, Exprs: []ast.Node{ &ast.LiteralNode{ Value: "bar ", Type: ast.TypeString, + Posx: ast.Pos{Column: 7, Line: 1}, }, &ast.VariableAccess{ Name: "baz", + Posx: ast.Pos{Column: 14, Line: 1}, }, }, }, diff --git a/config/lang/visitor_types.go b/config/lang/visitor_types.go index f06a932d1..e9e1cd0c3 100644 --- a/config/lang/visitor_types.go +++ b/config/lang/visitor_types.go @@ -41,7 +41,7 @@ func (v *TypeVisitor) visit(raw ast.Node) { case *ast.VariableAccess: v.visitVariableAccess(n) default: - v.err = fmt.Errorf("unknown node: %#v", raw) + v.createErr(n, fmt.Sprintf("unknown node: %#v", raw)) } } @@ -59,7 +59,8 @@ func (v *TypeVisitor) visitConcat(n *ast.Concat) { // All concat args must be strings, so validate that for i, t := range types { if t != ast.TypeString { - v.err = fmt.Errorf("%s: argument %d must be a sting", n, i+1) + v.createErr(n, fmt.Sprintf( + "argument %d must be a sting", n, i+1)) return } } @@ -76,7 +77,8 @@ func (v *TypeVisitor) visitVariableAccess(n *ast.VariableAccess) { // Look up the variable in the map variable, ok := v.VarMap[n.Name] if !ok { - v.err = fmt.Errorf("unknown variable accessed: %s", n.Name) + v.createErr(n, fmt.Sprintf( + "unknown variable accessed: %s", n.Name)) return } @@ -84,6 +86,10 @@ func (v *TypeVisitor) visitVariableAccess(n *ast.VariableAccess) { v.stackPush(variable.Type) } +func (v *TypeVisitor) createErr(n ast.Node, str string) { + v.err = fmt.Errorf("%s: %s", n.Pos(), str) +} + func (v *TypeVisitor) reset() { v.stack = nil v.err = nil diff --git a/config/lang/y.go b/config/lang/y.go index 173c1208c..e8ce25a93 100644 --- a/config/lang/y.go +++ b/config/lang/y.go @@ -14,21 +14,20 @@ type parserSymType struct { node ast.Node nodeList []ast.Node str string + token *parserToken } -const STRING = 57346 -const IDENTIFIER = 57347 -const PROGRAM_BRACKET_LEFT = 57348 -const PROGRAM_BRACKET_RIGHT = 57349 -const PROGRAM_STRING_START = 57350 -const PROGRAM_STRING_END = 57351 -const PAREN_LEFT = 57352 -const PAREN_RIGHT = 57353 -const COMMA = 57354 +const PROGRAM_BRACKET_LEFT = 57346 +const PROGRAM_BRACKET_RIGHT = 57347 +const PROGRAM_STRING_START = 57348 +const PROGRAM_STRING_END = 57349 +const PAREN_LEFT = 57350 +const PAREN_RIGHT = 57351 +const COMMA = 57352 +const IDENTIFIER = 57353 +const STRING = 57354 var parserToknames = []string{ - "STRING", - "IDENTIFIER", "PROGRAM_BRACKET_LEFT", "PROGRAM_BRACKET_RIGHT", "PROGRAM_STRING_START", @@ -36,6 +35,8 @@ var parserToknames = []string{ "PAREN_LEFT", "PAREN_RIGHT", "COMMA", + "IDENTIFIER", + "STRING", } var parserStatenames = []string{} @@ -43,7 +44,7 @@ const parserEofCode = 1 const parserErrCode = 2 const parserMaxDepth = 200 -//line lang.y:103 +//line lang.y:111 //line yacctab:1 var parserExca = []int{ @@ -62,18 +63,18 @@ const parserLast = 21 var parserAct = []int{ - 9, 16, 17, 13, 3, 12, 1, 8, 6, 11, - 7, 6, 14, 7, 15, 8, 10, 2, 18, 4, + 9, 7, 7, 13, 3, 16, 17, 8, 11, 6, + 6, 12, 10, 2, 15, 8, 1, 14, 18, 4, 5, } var parserPact = []int{ - 7, -1000, 7, -1000, -1000, -1000, -1000, 4, -1000, -2, - 7, -7, -1000, 4, -10, -1000, -1000, 4, -1000, + -2, -1000, -2, -1000, -1000, -1000, -1000, -3, -1000, 6, + -2, -5, -1000, -3, -4, -1000, -1000, -3, -1000, } var parserPgo = []int{ - 0, 0, 20, 19, 16, 4, 12, 6, + 0, 0, 20, 19, 12, 4, 17, 16, } var parserR1 = []int{ @@ -87,8 +88,8 @@ var parserR2 = []int{ } var parserChk = []int{ - -1000, -7, -4, -5, -3, -2, 4, 6, -5, -1, - -4, 5, 7, 10, -6, -1, 11, 12, -1, + -1000, -7, -4, -5, -3, -2, 12, 4, -5, -1, + -4, 11, 5, 8, -6, -1, 9, 10, -1, } var parserDef = []int{ @@ -334,17 +335,17 @@ parserdefault: switch parsernt { case 1: - //line lang.y:31 + //line lang.y:34 { parserResult = parserS[parserpt-0].node } case 2: - //line lang.y:37 + //line lang.y:40 { parserVAL.node = parserS[parserpt-0].node } case 3: - //line lang.y:41 + //line lang.y:44 { var result []ast.Node if c, ok := parserS[parserpt-1].node.(*ast.Concat); ok { @@ -355,57 +356,62 @@ parserdefault: parserVAL.node = &ast.Concat{ Exprs: result, + Posx: result[0].Pos(), } } case 4: - //line lang.y:56 - { - parserVAL.node = parserS[parserpt-0].node - } - case 5: //line lang.y:60 { parserVAL.node = parserS[parserpt-0].node } + case 5: + //line lang.y:64 + { + parserVAL.node = parserS[parserpt-0].node + } case 6: - //line lang.y:66 + //line lang.y:70 { parserVAL.node = parserS[parserpt-1].node } case 7: - //line lang.y:72 + //line lang.y:76 { parserVAL.node = parserS[parserpt-0].node } case 8: - //line lang.y:76 - { - parserVAL.node = &ast.VariableAccess{Name: parserS[parserpt-0].str} - } - case 9: //line lang.y:80 { - parserVAL.node = &ast.Call{Func: parserS[parserpt-3].str, Args: parserS[parserpt-1].nodeList} + parserVAL.node = &ast.VariableAccess{Name: parserS[parserpt-0].token.Value.(string), Posx: parserS[parserpt-0].token.Pos} + } + case 9: + //line lang.y:84 + { + parserVAL.node = &ast.Call{Func: parserS[parserpt-3].token.Value.(string), Args: parserS[parserpt-1].nodeList, Posx: parserS[parserpt-3].token.Pos} } case 10: - //line lang.y:85 + //line lang.y:89 { parserVAL.nodeList = nil } case 11: - //line lang.y:89 + //line lang.y:93 { parserVAL.nodeList = append(parserS[parserpt-2].nodeList, parserS[parserpt-0].node) } case 12: - //line lang.y:93 + //line lang.y:97 { parserVAL.nodeList = append(parserVAL.nodeList, parserS[parserpt-0].node) } case 13: - //line lang.y:99 + //line lang.y:103 { - parserVAL.node = &ast.LiteralNode{Value: parserS[parserpt-0].str, Type: ast.TypeString} + parserVAL.node = &ast.LiteralNode{ + Value: parserS[parserpt-0].token.Value.(string), + Type: ast.TypeString, + Posx: parserS[parserpt-0].token.Pos, + } } } goto parserstack /* stack new state and value */ diff --git a/config/lang/y.output b/config/lang/y.output index b17ca3ae5..86f1ffef2 100644 --- a/config/lang/y.output +++ b/config/lang/y.output @@ -2,8 +2,8 @@ state 0 $accept: .top $end - STRING shift 6 PROGRAM_BRACKET_LEFT shift 7 + STRING shift 6 . error interpolation goto 5 @@ -23,9 +23,9 @@ state 2 top: literalModeTop. (1) literalModeTop: literalModeTop.literalModeValue - STRING shift 6 PROGRAM_BRACKET_LEFT shift 7 - . reduce 1 (src line 29) + STRING shift 6 + . reduce 1 (src line 32) interpolation goto 5 literal goto 4 @@ -34,33 +34,33 @@ state 2 state 3 literalModeTop: literalModeValue. (2) - . reduce 2 (src line 35) + . reduce 2 (src line 38) state 4 literalModeValue: literal. (4) - . reduce 4 (src line 54) + . reduce 4 (src line 58) state 5 literalModeValue: interpolation. (5) - . reduce 5 (src line 59) + . reduce 5 (src line 63) state 6 literal: STRING. (13) - . reduce 13 (src line 97) + . reduce 13 (src line 101) state 7 interpolation: PROGRAM_BRACKET_LEFT.expr PROGRAM_BRACKET_RIGHT - STRING shift 6 - IDENTIFIER shift 11 PROGRAM_BRACKET_LEFT shift 7 + IDENTIFIER shift 11 + STRING shift 6 . error expr goto 9 @@ -72,7 +72,7 @@ state 7 state 8 literalModeTop: literalModeTop literalModeValue. (3) - . reduce 3 (src line 40) + . reduce 3 (src line 43) state 9 @@ -86,9 +86,9 @@ state 10 literalModeTop: literalModeTop.literalModeValue expr: literalModeTop. (7) - STRING shift 6 PROGRAM_BRACKET_LEFT shift 7 - . reduce 7 (src line 70) + STRING shift 6 + . reduce 7 (src line 74) interpolation goto 5 literal goto 4 @@ -99,23 +99,23 @@ state 11 expr: IDENTIFIER.PAREN_LEFT args PAREN_RIGHT PAREN_LEFT shift 13 - . reduce 8 (src line 75) + . reduce 8 (src line 79) state 12 interpolation: PROGRAM_BRACKET_LEFT expr PROGRAM_BRACKET_RIGHT. (6) - . reduce 6 (src line 64) + . reduce 6 (src line 68) state 13 expr: IDENTIFIER PAREN_LEFT.args PAREN_RIGHT args: . (10) - STRING shift 6 - IDENTIFIER shift 11 PROGRAM_BRACKET_LEFT shift 7 - . reduce 10 (src line 84) + IDENTIFIER shift 11 + STRING shift 6 + . reduce 10 (src line 88) expr goto 15 interpolation goto 5 @@ -136,21 +136,21 @@ state 14 state 15 args: expr. (12) - . reduce 12 (src line 92) + . reduce 12 (src line 96) state 16 expr: IDENTIFIER PAREN_LEFT args PAREN_RIGHT. (9) - . reduce 9 (src line 79) + . reduce 9 (src line 83) state 17 args: args COMMA.expr - STRING shift 6 - IDENTIFIER shift 11 PROGRAM_BRACKET_LEFT shift 7 + IDENTIFIER shift 11 + STRING shift 6 . error expr goto 18 @@ -162,7 +162,7 @@ state 17 state 18 args: args COMMA expr. (11) - . reduce 11 (src line 88) + . reduce 11 (src line 92) 12 terminals, 8 nonterminals From a0926de4a90d3d979ba79b362c1ce3ae50d892bd Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 12 Jan 2015 00:35:43 -0800 Subject: [PATCH 09/31] config/lang: completed type checking --- config/lang/visitor_types.go | 27 ++++++++++++++++-- config/lang/visitor_types_test.go | 47 +++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/config/lang/visitor_types.go b/config/lang/visitor_types.go index e9e1cd0c3..f97449073 100644 --- a/config/lang/visitor_types.go +++ b/config/lang/visitor_types.go @@ -46,8 +46,31 @@ func (v *TypeVisitor) visit(raw ast.Node) { } func (v *TypeVisitor) visitCall(n *ast.Call) { - // TODO - v.stackPush(ast.TypeString) + // Look up the function in the map + function, ok := v.FuncMap[n.Func] + if !ok { + v.createErr(n, fmt.Sprintf("unknown function called: %s", n.Func)) + return + } + + // The arguments are on the stack in reverse order, so pop them off. + args := make([]ast.Type, len(n.Args)) + for i, _ := range n.Args { + args[len(n.Args)-1-i] = v.stackPop() + } + + // Verify the args + for i, expected := range function.ArgTypes { + if args[i] != expected { + v.createErr(n, fmt.Sprintf( + "%s: argument %d should be %s, got %s", + n.Func, i+1, expected, args[i])) + return + } + } + + // Return type + v.stackPush(function.ReturnType) } func (v *TypeVisitor) visitConcat(n *ast.Concat) { diff --git a/config/lang/visitor_types_test.go b/config/lang/visitor_types_test.go index 92f81e023..7163108ac 100644 --- a/config/lang/visitor_types_test.go +++ b/config/lang/visitor_types_test.go @@ -46,6 +46,38 @@ func TestTypeVisitor(t *testing.T) { false, }, + { + `foo ${rand("42")}`, + &TypeVisitor{ + FuncMap: map[string]Function{ + "rand": Function{ + ArgTypes: []ast.Type{ast.TypeString}, + ReturnType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, + + { + `foo ${rand(42)}`, + &TypeVisitor{ + FuncMap: map[string]Function{ + "rand": Function{ + ArgTypes: []ast.Type{ast.TypeString}, + ReturnType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + true, + }, + { "foo ${bar}", &TypeVisitor{ @@ -58,6 +90,21 @@ func TestTypeVisitor(t *testing.T) { }, true, }, + + { + "foo ${rand()}", + &TypeVisitor{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeInt, + Callback: func([]interface{}) (interface{}, error) { + return 42, nil + }, + }, + }, + }, + true, + }, } for _, tc := range cases { From 25a2fbc9029d525825017b8ea9df0c081402aa52 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 12 Jan 2015 08:53:27 -0800 Subject: [PATCH 10/31] config/lang: integer and float types --- config/lang/ast/ast.go | 1 + config/lang/lang.y | 18 ++++++- config/lang/lex.go | 62 +++++++++++++++++++++++++ config/lang/lex_test.go | 14 ++++++ config/lang/parse_test.go | 40 ++++++++++++++++ config/lang/y.go | 81 +++++++++++++++++++++----------- config/lang/y.output | 98 +++++++++++++++++++++++---------------- 7 files changed, 245 insertions(+), 69 deletions(-) diff --git a/config/lang/ast/ast.go b/config/lang/ast/ast.go index 343deb703..a03070180 100644 --- a/config/lang/ast/ast.go +++ b/config/lang/ast/ast.go @@ -39,4 +39,5 @@ const ( TypeInvalid Type = 0 TypeString = 1 << iota TypeInt + TypeFloat ) diff --git a/config/lang/lang.y b/config/lang/lang.y index d379a6a09..241b286fb 100644 --- a/config/lang/lang.y +++ b/config/lang/lang.y @@ -22,7 +22,7 @@ import ( %token PROGRAM_STRING_START PROGRAM_STRING_END %token PAREN_LEFT PAREN_RIGHT COMMA -%token IDENTIFIER STRING +%token IDENTIFIER INTEGER FLOAT STRING %type expr interpolation literal literalModeTop literalModeValue %type args @@ -76,6 +76,22 @@ expr: { $$ = $1 } +| INTEGER + { + $$ = &ast.LiteralNode{ + Value: $1.Value.(int), + Type: ast.TypeInt, + Posx: $1.Pos, + } + } +| FLOAT + { + $$ = &ast.LiteralNode{ + Value: $1.Value.(float64), + Type: ast.TypeFloat, + Posx: $1.Pos, + } + } | IDENTIFIER { $$ = &ast.VariableAccess{Name: $1.Value.(string), Posx: $1.Pos} diff --git a/config/lang/lex.go b/config/lang/lex.go index a4ee94b2a..30be9b072 100644 --- a/config/lang/lex.go +++ b/config/lang/lex.go @@ -3,6 +3,7 @@ package lang import ( "bytes" "fmt" + "strconv" "unicode" "unicode/utf8" @@ -145,6 +146,12 @@ func (x *parserLex) lexModeInterpolation(yylval *parserSymType) int { return result } + // If we are seeing a number, it is the start of a number. Lex it. + if c >= '0' && c <= '9' { + x.backup() + return x.lexNumber(yylval) + } + switch c { case '}': x.interpolationDepth-- @@ -193,6 +200,61 @@ func (x *parserLex) lexId(yylval *parserSymType) int { return IDENTIFIER } +// lexNumber lexes out a number: an integer or a float. +func (x *parserLex) lexNumber(yylval *parserSymType) int { + var b bytes.Buffer + gotPeriod := false + for { + c := x.next() + if c == lexEOF { + break + } + + // If we see a period, we might be getting a float.. + if c == '.' { + // If we've already seen a period, then ignore it, and + // exit. This will probably result in a syntax error later. + if gotPeriod { + x.backup() + break + } + + gotPeriod = true + } else if c < '0' || c > '9' { + // If we're not seeing a number, then also exit. + x.backup() + break + } + + if _, err := b.WriteRune(c); err != nil { + x.Error(fmt.Sprintf("internal error: %s", err)) + return lexEOF + } + } + + // If we didn't see a period, it is an int + if !gotPeriod { + v, err := strconv.ParseInt(b.String(), 0, 0) + if err != nil { + x.Error(fmt.Sprintf("expected number: %s", err)) + return lexEOF + } + + yylval.token = &parserToken{Value: int(v)} + return INTEGER + } + + // If we did see a period, it is a float + f, err := strconv.ParseFloat(b.String(), 64) + if err != nil { + x.Error(fmt.Sprintf("expected float: %s", err)) + return lexEOF + } + + yylval.token = &parserToken{Value: f} + return FLOAT +} + func (x *parserLex) lexString(yylval *parserSymType, quoted bool) (int, bool) { var b bytes.Buffer terminated := false diff --git a/config/lang/lex_test.go b/config/lang/lex_test.go index b5cc7c583..73913adf6 100644 --- a/config/lang/lex_test.go +++ b/config/lang/lex_test.go @@ -46,6 +46,20 @@ func TestLex(t *testing.T) { PROGRAM_BRACKET_RIGHT, lexEOF}, }, + { + "${bar(42)}", + []int{PROGRAM_BRACKET_LEFT, + IDENTIFIER, PAREN_LEFT, INTEGER, PAREN_RIGHT, + PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + + { + "${bar(3.14159)}", + []int{PROGRAM_BRACKET_LEFT, + IDENTIFIER, PAREN_LEFT, FLOAT, PAREN_RIGHT, + PROGRAM_BRACKET_RIGHT, lexEOF}, + }, + { "${bar(inner(baz))}", []int{PROGRAM_BRACKET_LEFT, diff --git a/config/lang/parse_test.go b/config/lang/parse_test.go index 9d17072e8..d8e2c7236 100644 --- a/config/lang/parse_test.go +++ b/config/lang/parse_test.go @@ -86,6 +86,46 @@ func TestParse(t *testing.T) { }, }, + { + "foo ${42}", + false, + &ast.Concat{ + Posx: ast.Pos{Column: 1, Line: 1}, + Exprs: []ast.Node{ + &ast.LiteralNode{ + Value: "foo ", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + }, + &ast.LiteralNode{ + Value: 42, + Type: ast.TypeInt, + Posx: ast.Pos{Column: 7, Line: 1}, + }, + }, + }, + }, + + { + "foo ${3.14159}", + false, + &ast.Concat{ + Posx: ast.Pos{Column: 1, Line: 1}, + Exprs: []ast.Node{ + &ast.LiteralNode{ + Value: "foo ", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + }, + &ast.LiteralNode{ + Value: 3.14159, + Type: ast.TypeFloat, + Posx: ast.Pos{Column: 7, Line: 1}, + }, + }, + }, + }, + { "${foo()}", false, diff --git a/config/lang/y.go b/config/lang/y.go index e8ce25a93..07ae55cdc 100644 --- a/config/lang/y.go +++ b/config/lang/y.go @@ -25,7 +25,9 @@ const PAREN_LEFT = 57350 const PAREN_RIGHT = 57351 const COMMA = 57352 const IDENTIFIER = 57353 -const STRING = 57354 +const INTEGER = 57354 +const FLOAT = 57355 +const STRING = 57356 var parserToknames = []string{ "PROGRAM_BRACKET_LEFT", @@ -36,6 +38,8 @@ var parserToknames = []string{ "PAREN_RIGHT", "COMMA", "IDENTIFIER", + "INTEGER", + "FLOAT", "STRING", } var parserStatenames = []string{} @@ -44,7 +48,7 @@ const parserEofCode = 1 const parserErrCode = 2 const parserMaxDepth = 200 -//line lang.y:111 +//line lang.y:127 //line yacctab:1 var parserExca = []int{ @@ -53,48 +57,51 @@ var parserExca = []int{ -2, 0, } -const parserNprod = 14 +const parserNprod = 16 const parserPrivate = 57344 var parserTokenNames []string var parserStates []string -const parserLast = 21 +const parserLast = 23 var parserAct = []int{ - 9, 7, 7, 13, 3, 16, 17, 8, 11, 6, - 6, 12, 10, 2, 15, 8, 1, 14, 18, 4, - 5, + 9, 7, 7, 3, 18, 19, 8, 15, 13, 11, + 12, 6, 6, 14, 8, 1, 17, 10, 2, 16, + 20, 4, 5, } var parserPact = []int{ - -2, -1000, -2, -1000, -1000, -1000, -1000, -3, -1000, 6, - -2, -5, -1000, -3, -4, -1000, -1000, -3, -1000, + -2, -1000, -2, -1000, -1000, -1000, -1000, -3, -1000, 8, + -2, -1000, -1000, -1, -1000, -3, -5, -1000, -1000, -3, + -1000, } var parserPgo = []int{ - 0, 0, 20, 19, 12, 4, 17, 16, + 0, 0, 22, 21, 17, 3, 19, 15, } var parserR1 = []int{ 0, 7, 4, 4, 5, 5, 2, 1, 1, 1, - 6, 6, 6, 3, + 1, 1, 6, 6, 6, 3, } var parserR2 = []int{ - 0, 1, 1, 2, 1, 1, 3, 1, 1, 4, - 0, 3, 1, 1, + 0, 1, 1, 2, 1, 1, 3, 1, 1, 1, + 1, 4, 0, 3, 1, 1, } var parserChk = []int{ - -1000, -7, -4, -5, -3, -2, 12, 4, -5, -1, - -4, 11, 5, 8, -6, -1, 9, 10, -1, + -1000, -7, -4, -5, -3, -2, 14, 4, -5, -1, + -4, 12, 13, 11, 5, 8, -6, -1, 9, 10, + -1, } var parserDef = []int{ - 0, -2, 1, 2, 4, 5, 13, 0, 3, 0, - 7, 8, 6, 10, 0, 12, 9, 0, 11, + 0, -2, 1, 2, 4, 5, 15, 0, 3, 0, + 7, 8, 9, 10, 6, 12, 0, 14, 11, 0, + 13, } var parserTok1 = []int{ @@ -103,7 +110,7 @@ var parserTok1 = []int{ var parserTok2 = []int{ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, - 12, + 12, 13, 14, } var parserTok3 = []int{ 0, @@ -382,30 +389,48 @@ parserdefault: case 8: //line lang.y:80 { - parserVAL.node = &ast.VariableAccess{Name: parserS[parserpt-0].token.Value.(string), Posx: parserS[parserpt-0].token.Pos} + parserVAL.node = &ast.LiteralNode{ + Value: parserS[parserpt-0].token.Value.(int), + Type: ast.TypeInt, + Posx: parserS[parserpt-0].token.Pos, + } } case 9: - //line lang.y:84 + //line lang.y:88 + { + parserVAL.node = &ast.LiteralNode{ + Value: parserS[parserpt-0].token.Value.(float64), + Type: ast.TypeFloat, + Posx: parserS[parserpt-0].token.Pos, + } + } + case 10: + //line lang.y:96 + { + parserVAL.node = &ast.VariableAccess{Name: parserS[parserpt-0].token.Value.(string), Posx: parserS[parserpt-0].token.Pos} + } + case 11: + //line lang.y:100 { parserVAL.node = &ast.Call{Func: parserS[parserpt-3].token.Value.(string), Args: parserS[parserpt-1].nodeList, Posx: parserS[parserpt-3].token.Pos} } - case 10: - //line lang.y:89 + case 12: + //line lang.y:105 { parserVAL.nodeList = nil } - case 11: - //line lang.y:93 + case 13: + //line lang.y:109 { parserVAL.nodeList = append(parserS[parserpt-2].nodeList, parserS[parserpt-0].node) } - case 12: - //line lang.y:97 + case 14: + //line lang.y:113 { parserVAL.nodeList = append(parserVAL.nodeList, parserS[parserpt-0].node) } - case 13: - //line lang.y:103 + case 15: + //line lang.y:119 { parserVAL.node = &ast.LiteralNode{ Value: parserS[parserpt-0].token.Value.(string), diff --git a/config/lang/y.output b/config/lang/y.output index 86f1ffef2..ce531c461 100644 --- a/config/lang/y.output +++ b/config/lang/y.output @@ -50,16 +50,18 @@ state 5 state 6 - literal: STRING. (13) + literal: STRING. (15) - . reduce 13 (src line 101) + . reduce 15 (src line 117) state 7 interpolation: PROGRAM_BRACKET_LEFT.expr PROGRAM_BRACKET_RIGHT PROGRAM_BRACKET_LEFT shift 7 - IDENTIFIER shift 11 + IDENTIFIER shift 13 + INTEGER shift 11 + FLOAT shift 12 STRING shift 6 . error @@ -78,7 +80,7 @@ state 8 state 9 interpolation: PROGRAM_BRACKET_LEFT expr.PROGRAM_BRACKET_RIGHT - PROGRAM_BRACKET_RIGHT shift 12 + PROGRAM_BRACKET_RIGHT shift 14 . error @@ -95,85 +97,101 @@ state 10 literalModeValue goto 8 state 11 - expr: IDENTIFIER. (8) - expr: IDENTIFIER.PAREN_LEFT args PAREN_RIGHT + expr: INTEGER. (8) - PAREN_LEFT shift 13 . reduce 8 (src line 79) state 12 + expr: FLOAT. (9) + + . reduce 9 (src line 87) + + +state 13 + expr: IDENTIFIER. (10) + expr: IDENTIFIER.PAREN_LEFT args PAREN_RIGHT + + PAREN_LEFT shift 15 + . reduce 10 (src line 95) + + +state 14 interpolation: PROGRAM_BRACKET_LEFT expr PROGRAM_BRACKET_RIGHT. (6) . reduce 6 (src line 68) -state 13 +state 15 expr: IDENTIFIER PAREN_LEFT.args PAREN_RIGHT - args: . (10) + args: . (12) PROGRAM_BRACKET_LEFT shift 7 - IDENTIFIER shift 11 + IDENTIFIER shift 13 + INTEGER shift 11 + FLOAT shift 12 STRING shift 6 - . reduce 10 (src line 88) + . reduce 12 (src line 104) - expr goto 15 + expr goto 17 interpolation goto 5 literal goto 4 literalModeTop goto 10 literalModeValue goto 3 - args goto 14 + args goto 16 -state 14 +state 16 expr: IDENTIFIER PAREN_LEFT args.PAREN_RIGHT args: args.COMMA expr - PAREN_RIGHT shift 16 - COMMA shift 17 + PAREN_RIGHT shift 18 + COMMA shift 19 . error -state 15 - args: expr. (12) - - . reduce 12 (src line 96) - - -state 16 - expr: IDENTIFIER PAREN_LEFT args PAREN_RIGHT. (9) - - . reduce 9 (src line 83) - - state 17 + args: expr. (14) + + . reduce 14 (src line 112) + + +state 18 + expr: IDENTIFIER PAREN_LEFT args PAREN_RIGHT. (11) + + . reduce 11 (src line 99) + + +state 19 args: args COMMA.expr PROGRAM_BRACKET_LEFT shift 7 - IDENTIFIER shift 11 + IDENTIFIER shift 13 + INTEGER shift 11 + FLOAT shift 12 STRING shift 6 . error - expr goto 18 + expr goto 20 interpolation goto 5 literal goto 4 literalModeTop goto 10 literalModeValue goto 3 -state 18 - args: args COMMA expr. (11) +state 20 + args: args COMMA expr. (13) - . reduce 11 (src line 92) + . reduce 13 (src line 108) -12 terminals, 8 nonterminals -14 grammar rules, 19/2000 states +14 terminals, 8 nonterminals +16 grammar rules, 21/2000 states 0 shift/reduce, 0 reduce/reduce conflicts reported 57 working sets used memory: parser 25/30000 -14 extra closures -19 shift entries, 1 exceptions +16 extra closures +25 shift entries, 1 exceptions 12 goto entries 15 entries saved by goto default -Optimizer space used: output 21/30000 -21 table entries, 0 zero -maximum spread: 12, maximum offset: 17 +Optimizer space used: output 23/30000 +23 table entries, 0 zero +maximum spread: 14, maximum offset: 19 From ec3b5f3886e73f5d87ac0335313f3cc3f3d3ff77 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 12 Jan 2015 09:57:16 -0800 Subject: [PATCH 11/31] config/lang: implement type lookup --- config/lang/ast/ast.go | 2 +- config/lang/engine.go | 65 ++++++++++------- config/lang/engine_test.go | 11 +-- config/lang/transform_implicit_types_test.go | 74 ++++++++++++++++++++ config/lang/types.go | 58 +++++++++++++++ 5 files changed, 180 insertions(+), 30 deletions(-) create mode 100644 config/lang/transform_implicit_types_test.go create mode 100644 config/lang/types.go diff --git a/config/lang/ast/ast.go b/config/lang/ast/ast.go index a03070180..2476098e8 100644 --- a/config/lang/ast/ast.go +++ b/config/lang/ast/ast.go @@ -18,7 +18,7 @@ type Pos struct { Column, Line int // Column/Line number, starting at 1 } -func (p *Pos) String() string { +func (p Pos) String() string { return fmt.Sprintf("%d:%d", p.Line, p.Column) } diff --git a/config/lang/engine.go b/config/lang/engine.go index 0b3a6c14d..4f06887d2 100644 --- a/config/lang/engine.go +++ b/config/lang/engine.go @@ -11,10 +11,8 @@ import ( // Engine is the execution engine for this language. It should be configured // prior to running Execute. type Engine struct { - // VarMap and FuncMap are the mappings of identifiers to functions - // and variable values. - VarMap map[string]Variable - FuncMap map[string]Function + // GlobalScope is the global scope of execution for this engine. + GlobalScope *Scope // SemanticChecks is a list of additional semantic checks that will be run // on the tree prior to executing it. The type checker, identifier checker, @@ -26,26 +24,10 @@ type Engine struct { // semantic check on an AST tree. This will be called with the root node. type SemanticChecker func(ast.Node) error -// Variable is a variable value for execution given as input to the engine. -// It records the value of a variables along with their type. -type Variable struct { - Value interface{} - Type ast.Type -} - -// Function defines a function that can be executed by the engine. -// The type checker will validate that the proper types will be called -// to the callback. -type Function struct { - ArgTypes []ast.Type - ReturnType ast.Type - Callback func([]interface{}) (interface{}, error) -} - // Execute executes the given ast.Node and returns its final value, its // type, and an error if one exists. func (e *Engine) Execute(root ast.Node) (interface{}, ast.Type, error) { - v := &executeVisitor{Engine: e} + v := &executeVisitor{Scope: e.GlobalScope} return v.Visit(root) } @@ -53,7 +35,7 @@ func (e *Engine) Execute(root ast.Node) (interface{}, ast.Type, error) { // a program. Note at this point it is assumed that the types check out // and the identifiers exist. type executeVisitor struct { - Engine *Engine + Scope *Scope stack []*ast.LiteralNode err error @@ -104,7 +86,7 @@ func (v *executeVisitor) visit(raw ast.Node) { func (v *executeVisitor) visitCall(n *ast.Call) { // Look up the function in the map - function, ok := v.Engine.FuncMap[n.Func] + function, ok := v.Scope.FuncMap[n.Func] if !ok { v.err = fmt.Errorf("unknown function called: %s", n.Func) return @@ -156,7 +138,7 @@ func (v *executeVisitor) visitLiteral(n *ast.LiteralNode) { func (v *executeVisitor) visitVariableAccess(n *ast.VariableAccess) { // Look up the variable in the map - variable, ok := v.Engine.VarMap[n.Name] + variable, ok := v.Scope.VarMap[n.Name] if !ok { v.err = fmt.Errorf("unknown variable accessed: %s", n.Name) return @@ -177,3 +159,38 @@ func (v *executeVisitor) stackPop() *ast.LiteralNode { x, v.stack = v.stack[len(v.stack)-1], v.stack[:len(v.stack)-1] return x } + +// Scope represents a lookup scope for execution. +type Scope struct { + // VarMap and FuncMap are the mappings of identifiers to functions + // and variable values. + VarMap map[string]Variable + FuncMap map[string]Function +} + +// Variable is a variable value for execution given as input to the engine. +// It records the value of a variables along with their type. +type Variable struct { + Value interface{} + Type ast.Type +} + +// Function defines a function that can be executed by the engine. +// The type checker will validate that the proper types will be called +// to the callback. +type Function struct { + ArgTypes []ast.Type + ReturnType ast.Type + Callback func([]interface{}) (interface{}, error) +} + +// LookupVar will look up a variable by name. +// TODO test +func (s *Scope) LookupVar(n string) (Variable, bool) { + if s == nil { + return Variable{}, false + } + + v, ok := s.VarMap[n] + return v, ok +} diff --git a/config/lang/engine_test.go b/config/lang/engine_test.go index 96a1105d5..b65d0475a 100644 --- a/config/lang/engine_test.go +++ b/config/lang/engine_test.go @@ -10,14 +10,14 @@ import ( func TestEngineExecute(t *testing.T) { cases := []struct { Input string - Engine *Engine + Scope *Scope Error bool Result interface{} ResultType ast.Type }{ { "foo", - &Engine{}, + nil, false, "foo", ast.TypeString, @@ -25,7 +25,7 @@ func TestEngineExecute(t *testing.T) { { "foo ${bar}", - &Engine{ + &Scope{ VarMap: map[string]Variable{ "bar": Variable{ Value: "baz", @@ -40,7 +40,7 @@ func TestEngineExecute(t *testing.T) { { "foo ${rand()}", - &Engine{ + &Scope{ FuncMap: map[string]Function{ "rand": Function{ ReturnType: ast.TypeString, @@ -62,7 +62,8 @@ func TestEngineExecute(t *testing.T) { t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) } - out, outType, err := tc.Engine.Execute(node) + engine := &Engine{GlobalScope: tc.Scope} + out, outType, err := engine.Execute(node) if (err != nil) != tc.Error { t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) } diff --git a/config/lang/transform_implicit_types_test.go b/config/lang/transform_implicit_types_test.go new file mode 100644 index 000000000..9eb0fd92f --- /dev/null +++ b/config/lang/transform_implicit_types_test.go @@ -0,0 +1,74 @@ +package lang + +import ( + "testing" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +func TestLookupType(t *testing.T) { + cases := []struct { + Input ast.Node + Scope *Scope + Output ast.Type + Error bool + }{ + { + &customUntyped{}, + nil, + ast.TypeInvalid, + true, + }, + + { + &customTyped{}, + nil, + ast.TypeString, + false, + }, + + { + &ast.LiteralNode{ + Value: 42, + Type: ast.TypeInt, + }, + nil, + ast.TypeInt, + false, + }, + + { + &ast.VariableAccess{ + Name: "foo", + }, + &Scope{ + VarMap: map[string]Variable{ + "foo": Variable{Type: ast.TypeInt}, + }, + }, + ast.TypeInt, + false, + }, + } + + for _, tc := range cases { + actual, err := LookupType(tc.Input, tc.Scope) + if (err != nil) != tc.Error { + t.Fatalf("bad: %s\n\nInput: %#v", err, tc.Input) + } + if actual != tc.Output { + t.Fatalf("bad: %s\n\nInput: %#v", actual, tc.Input) + } + } +} + +type customUntyped struct{} + +func (n customUntyped) Accept(ast.Visitor) {} +func (n customUntyped) Pos() (v ast.Pos) { return } + +type customTyped struct{} + +func (n customTyped) Accept(ast.Visitor) {} +func (n customTyped) Pos() (v ast.Pos) { return } +func (n customTyped) Type(*Scope) (ast.Type, error) { return ast.TypeString, nil } diff --git a/config/lang/types.go b/config/lang/types.go new file mode 100644 index 000000000..2d6b58431 --- /dev/null +++ b/config/lang/types.go @@ -0,0 +1,58 @@ +package lang + +import ( + "fmt" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +// LookupType looks up the type of the given node with the given scope. +func LookupType(raw ast.Node, scope *Scope) (ast.Type, error) { + switch n := raw.(type) { + case *ast.LiteralNode: + return typedLiteralNode{n}.Type(scope) + case *ast.VariableAccess: + return typedVariableAccess{n}.Type(scope) + default: + if t, ok := raw.(TypedNode); ok { + return t.Type(scope) + } + + return ast.TypeInvalid, fmt.Errorf( + "unknown node to get type of: %T", raw) + } +} + +// TypedNode is an interface that custom AST nodes should implement +// if they want to work with LookupType. All the builtin AST nodes have +// implementations of this. +type TypedNode interface { + Type(*Scope) (ast.Type, error) +} + +type typedLiteralNode struct { + n *ast.LiteralNode +} + +func (n typedLiteralNode) Type(s *Scope) (ast.Type, error) { + return n.n.Type, nil +} + +type typedVariableAccess struct { + n *ast.VariableAccess +} + +func (n typedVariableAccess) Type(s *Scope) (ast.Type, error) { + v, ok := s.LookupVar(n.n.Name) + if !ok { + return ast.TypeInvalid, fmt.Errorf( + "%s: couldn't find variable %s", n.n.Pos(), n.n.Name) + } + + return v.Type, nil +} + +var supportedTransforms = map[ast.Type]ast.Type{ + ast.TypeString: ast.TypeInt, + ast.TypeInt: ast.TypeString, +} From f836397d8d3b1c032424ba6d3a21e1099619b3c7 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 12 Jan 2015 10:21:18 -0800 Subject: [PATCH 12/31] config/lang: enable type checking in execution --- config/lang/engine.go | 18 ++++++++++++++++++ config/lang/types.go | 5 ----- config/lang/visitor_types.go | 7 +++---- config/lang/visitor_types_test.go | 23 ++++++++++++----------- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/config/lang/engine.go b/config/lang/engine.go index 4f06887d2..8b2a09a6d 100644 --- a/config/lang/engine.go +++ b/config/lang/engine.go @@ -27,6 +27,13 @@ type SemanticChecker func(ast.Node) error // Execute executes the given ast.Node and returns its final value, its // type, and an error if one exists. func (e *Engine) Execute(root ast.Node) (interface{}, ast.Type, error) { + // Run the type checker + tv := &TypeVisitor{Scope: e.GlobalScope} + if err := tv.Visit(root); err != nil { + return nil, ast.TypeInvalid, err + } + + // Execute v := &executeVisitor{Scope: e.GlobalScope} return v.Visit(root) } @@ -184,6 +191,17 @@ type Function struct { Callback func([]interface{}) (interface{}, error) } +// LookupFunc will look up a variable by name. +// TODO test +func (s *Scope) LookupFunc(n string) (Function, bool) { + if s == nil { + return Function{}, false + } + + v, ok := s.FuncMap[n] + return v, ok +} + // LookupVar will look up a variable by name. // TODO test func (s *Scope) LookupVar(n string) (Variable, bool) { diff --git a/config/lang/types.go b/config/lang/types.go index 2d6b58431..8adfcd05e 100644 --- a/config/lang/types.go +++ b/config/lang/types.go @@ -51,8 +51,3 @@ func (n typedVariableAccess) Type(s *Scope) (ast.Type, error) { return v.Type, nil } - -var supportedTransforms = map[ast.Type]ast.Type{ - ast.TypeString: ast.TypeInt, - ast.TypeInt: ast.TypeString, -} diff --git a/config/lang/visitor_types.go b/config/lang/visitor_types.go index f97449073..0eeb756ae 100644 --- a/config/lang/visitor_types.go +++ b/config/lang/visitor_types.go @@ -10,8 +10,7 @@ import ( // TypeVisitor implements ast.Visitor for type checking an AST tree. // It requires some configuration to look up the type of nodes. type TypeVisitor struct { - VarMap map[string]Variable - FuncMap map[string]Function + Scope *Scope stack []ast.Type err error @@ -47,7 +46,7 @@ func (v *TypeVisitor) visit(raw ast.Node) { func (v *TypeVisitor) visitCall(n *ast.Call) { // Look up the function in the map - function, ok := v.FuncMap[n.Func] + function, ok := v.Scope.LookupFunc(n.Func) if !ok { v.createErr(n, fmt.Sprintf("unknown function called: %s", n.Func)) return @@ -98,7 +97,7 @@ func (v *TypeVisitor) visitLiteral(n *ast.LiteralNode) { func (v *TypeVisitor) visitVariableAccess(n *ast.VariableAccess) { // Look up the variable in the map - variable, ok := v.VarMap[n.Name] + variable, ok := v.Scope.LookupVar(n.Name) if !ok { v.createErr(n, fmt.Sprintf( "unknown variable accessed: %s", n.Name)) diff --git a/config/lang/visitor_types_test.go b/config/lang/visitor_types_test.go index 7163108ac..6ce21c81e 100644 --- a/config/lang/visitor_types_test.go +++ b/config/lang/visitor_types_test.go @@ -8,19 +8,19 @@ import ( func TestTypeVisitor(t *testing.T) { cases := []struct { - Input string - Visitor *TypeVisitor - Error bool + Input string + Scope *Scope + Error bool }{ { "foo", - &TypeVisitor{}, + &Scope{}, false, }, { "foo ${bar}", - &TypeVisitor{ + &Scope{ VarMap: map[string]Variable{ "bar": Variable{ Value: "baz", @@ -33,7 +33,7 @@ func TestTypeVisitor(t *testing.T) { { "foo ${rand()}", - &TypeVisitor{ + &Scope{ FuncMap: map[string]Function{ "rand": Function{ ReturnType: ast.TypeString, @@ -48,7 +48,7 @@ func TestTypeVisitor(t *testing.T) { { `foo ${rand("42")}`, - &TypeVisitor{ + &Scope{ FuncMap: map[string]Function{ "rand": Function{ ArgTypes: []ast.Type{ast.TypeString}, @@ -64,7 +64,7 @@ func TestTypeVisitor(t *testing.T) { { `foo ${rand(42)}`, - &TypeVisitor{ + &Scope{ FuncMap: map[string]Function{ "rand": Function{ ArgTypes: []ast.Type{ast.TypeString}, @@ -80,7 +80,7 @@ func TestTypeVisitor(t *testing.T) { { "foo ${bar}", - &TypeVisitor{ + &Scope{ VarMap: map[string]Variable{ "bar": Variable{ Value: 42, @@ -93,7 +93,7 @@ func TestTypeVisitor(t *testing.T) { { "foo ${rand()}", - &TypeVisitor{ + &Scope{ FuncMap: map[string]Function{ "rand": Function{ ReturnType: ast.TypeInt, @@ -113,7 +113,8 @@ func TestTypeVisitor(t *testing.T) { t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) } - err = tc.Visitor.Visit(node) + visitor := &TypeVisitor{Scope: tc.Scope} + err = visitor.Visit(node) if (err != nil) != tc.Error { t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) } From c15c17dfe9fc158a42b967a531b1564d46bb3b62 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 12 Jan 2015 10:48:20 -0800 Subject: [PATCH 13/31] config/lang: remove unused file --- config/lang/ast.go | 1 - 1 file changed, 1 deletion(-) delete mode 100644 config/lang/ast.go diff --git a/config/lang/ast.go b/config/lang/ast.go deleted file mode 100644 index 48e87f869..000000000 --- a/config/lang/ast.go +++ /dev/null @@ -1 +0,0 @@ -package lang From c05d7a6acdcd56f00d84dfd1c48fd784dd18dbef Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 12 Jan 2015 10:59:43 -0800 Subject: [PATCH 14/31] config/lang: escaping interpolations with double dollar signs --- config/lang/lex.go | 16 +++++++++++++--- config/lang/lex_test.go | 10 ++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/config/lang/lex.go b/config/lang/lex.go index 30be9b072..7f2a85ae7 100644 --- a/config/lang/lex.go +++ b/config/lang/lex.go @@ -299,9 +299,19 @@ func (x *parserLex) lexString(yylval *parserSymType, quoted bool) (int, bool) { // If we hit a dollar sign, then check if we're starting // another interpolation. If so, then we're done. - if c == '$' && x.peek() == '{' { - x.backup() - break + if c == '$' { + n := x.peek() + + // If it is '{', then we're starting another interpolation + if n == '{' { + x.backup() + break + } + + // If it is '$', then we're escaping a dollar sign + if n == '$' { + x.next() + } } if _, err := b.WriteRune(c); err != nil { diff --git a/config/lang/lex_test.go b/config/lang/lex_test.go index 73913adf6..8c5207d78 100644 --- a/config/lang/lex_test.go +++ b/config/lang/lex_test.go @@ -25,6 +25,16 @@ func TestLex(t *testing.T) { []int{STRING, PROGRAM_BRACKET_LEFT, IDENTIFIER, PROGRAM_BRACKET_RIGHT, lexEOF}, }, + { + "foo $${bar}", + []int{STRING, lexEOF}, + }, + + { + "foo $$$${bar}", + []int{STRING, lexEOF}, + }, + { "foo ${\"bar\"}", []int{STRING, PROGRAM_BRACKET_LEFT, STRING, PROGRAM_BRACKET_RIGHT, lexEOF}, From aa2c7b2764e6a1feb5d05f6b6ff8d805b096c4ed Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 12 Jan 2015 12:09:30 -0800 Subject: [PATCH 15/31] config: DetectVariables to detect interpolated variables in an AST --- config/interpolate.go | 38 ++++++++++++++++++++++++++++++ config/interpolate_test.go | 48 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/config/interpolate.go b/config/interpolate.go index e96b6b76c..3e95642e0 100644 --- a/config/interpolate.go +++ b/config/interpolate.go @@ -5,6 +5,8 @@ import ( "regexp" "strconv" "strings" + + "github.com/hashicorp/terraform/config/lang/ast" ) // We really need to replace this with a real parser. @@ -317,3 +319,39 @@ func (v *UserVariable) FullKey() string { func (v *UserVariable) GoString() string { return fmt.Sprintf("*%#v", *v) } + +// DetectVariables takes an AST root and returns all the interpolated +// variables that are detected in the AST tree. +func DetectVariables(root ast.Node) ([]InterpolatedVariable, error) { + var result []InterpolatedVariable + var resultErr error + + // Visitor callback + fn := func(n ast.Node) { + if resultErr != nil { + return + } + + vn, ok := n.(*ast.VariableAccess) + if !ok { + return + } + + v, err := NewInterpolatedVariable(vn.Name) + if err != nil { + resultErr = err + return + } + + result = append(result, v) + } + + // Visitor pattern + root.Accept(fn) + + if resultErr != nil { + return nil, resultErr + } + + return result, nil +} diff --git a/config/interpolate_test.go b/config/interpolate_test.go index 9ecbb9a71..5af01841f 100644 --- a/config/interpolate_test.go +++ b/config/interpolate_test.go @@ -4,6 +4,8 @@ import ( "reflect" "strings" "testing" + + "github.com/hashicorp/terraform/config/lang" ) func TestNewInterpolatedVariable(t *testing.T) { @@ -291,3 +293,49 @@ func TestVariableInterpolation_missing(t *testing.T) { t.Fatal("should error") } } + +func TestDetectVariables(t *testing.T) { + cases := []struct { + Input string + Result []InterpolatedVariable + }{ + { + "foo ${var.foo}", + []InterpolatedVariable{ + &UserVariable{ + Name: "foo", + key: "var.foo", + }, + }, + }, + + { + "foo ${var.foo} ${var.bar}", + []InterpolatedVariable{ + &UserVariable{ + Name: "foo", + key: "var.foo", + }, + &UserVariable{ + Name: "bar", + key: "var.bar", + }, + }, + }, + } + + for _, tc := range cases { + ast, err := lang.Parse(tc.Input) + if err != nil { + t.Fatalf("%s\n\nInput: %s", err, tc.Input) + } + + actual, err := DetectVariables(ast) + if err != nil { + t.Fatalf("err: %s", err) + } + if !reflect.DeepEqual(actual, tc.Result) { + t.Fatalf("bad: %#v\n\nInput: %s", actual, tc.Input) + } + } +} From abca82a84e8b6fbd1cd6f2553e13c431a780afe7 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 12 Jan 2015 12:13:30 -0800 Subject: [PATCH 16/31] config: another test --- config/interpolate_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/interpolate_test.go b/config/interpolate_test.go index 5af01841f..2bafe4362 100644 --- a/config/interpolate_test.go +++ b/config/interpolate_test.go @@ -299,6 +299,11 @@ func TestDetectVariables(t *testing.T) { Input string Result []InterpolatedVariable }{ + { + "foo $${var.foo}", + nil, + }, + { "foo ${var.foo}", []InterpolatedVariable{ From 6d9db3139c6494641bd1171798c12b58eb5c7bb9 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 13 Jan 2015 08:50:28 -0800 Subject: [PATCH 17/31] config/lang: AST String() methods --- config/lang/ast/ast.go | 2 +- config/lang/ast/call.go | 14 ++++++++++++++ config/lang/ast/literal.go | 4 ++++ config/lang/ast/type_string.go | 26 ++++++++++++++++++++++---- config/lang/ast/variable_access.go | 4 ++++ config/lang/parse_test.go | 10 ++++++++++ 6 files changed, 55 insertions(+), 5 deletions(-) diff --git a/config/lang/ast/ast.go b/config/lang/ast/ast.go index 2476098e8..0d7000da6 100644 --- a/config/lang/ast/ast.go +++ b/config/lang/ast/ast.go @@ -37,7 +37,7 @@ type Type uint const ( TypeInvalid Type = 0 - TypeString = 1 << iota + TypeString Type = 1 << iota TypeInt TypeFloat ) diff --git a/config/lang/ast/call.go b/config/lang/ast/call.go index d70219983..40b0773e1 100644 --- a/config/lang/ast/call.go +++ b/config/lang/ast/call.go @@ -1,5 +1,10 @@ package ast +import ( + "fmt" + "strings" +) + // Call represents a function call. type Call struct { Func string @@ -18,3 +23,12 @@ func (n *Call) Accept(v Visitor) { func (n *Call) Pos() Pos { return n.Posx } + +func (n *Call) String() string { + args := make([]string, len(n.Args)) + for i, arg := range n.Args { + args[i] = fmt.Sprintf("%s", arg) + } + + return fmt.Sprintf("Call(%s, %s)", n.Func, strings.Join(args, ", ")) +} diff --git a/config/lang/ast/literal.go b/config/lang/ast/literal.go index a20316ac1..1fd7669ff 100644 --- a/config/lang/ast/literal.go +++ b/config/lang/ast/literal.go @@ -23,3 +23,7 @@ func (n *LiteralNode) Pos() Pos { func (n *LiteralNode) GoString() string { return fmt.Sprintf("*%#v", *n) } + +func (n *LiteralNode) String() string { + return fmt.Sprintf("Literal(%s, %v)", n.Type, n.Value) +} diff --git a/config/lang/ast/type_string.go b/config/lang/ast/type_string.go index 55e2af017..fd0e9e355 100644 --- a/config/lang/ast/type_string.go +++ b/config/lang/ast/type_string.go @@ -4,13 +4,31 @@ package ast import "fmt" -const _Type_name = "TypeInvalid" +const ( + _Type_name_0 = "TypeInvalid" + _Type_name_1 = "TypeString" + _Type_name_2 = "TypeInt" + _Type_name_3 = "TypeFloat" +) -var _Type_index = [...]uint8{0, 11} +var ( + _Type_index_0 = [...]uint8{0, 11} + _Type_index_1 = [...]uint8{0, 10} + _Type_index_2 = [...]uint8{0, 7} + _Type_index_3 = [...]uint8{0, 9} +) func (i Type) String() string { - if i+1 >= Type(len(_Type_index)) { + switch { + case i == 0: + return _Type_name_0 + case i == 2: + return _Type_name_1 + case i == 4: + return _Type_name_2 + case i == 8: + return _Type_name_3 + default: return fmt.Sprintf("Type(%d)", i) } - return _Type_name[_Type_index[i]:_Type_index[i+1]] } diff --git a/config/lang/ast/variable_access.go b/config/lang/ast/variable_access.go index 943ee5140..1f86a260d 100644 --- a/config/lang/ast/variable_access.go +++ b/config/lang/ast/variable_access.go @@ -21,3 +21,7 @@ func (n *VariableAccess) Pos() Pos { func (n *VariableAccess) GoString() string { return fmt.Sprintf("*%#v", *n) } + +func (n *VariableAccess) String() string { + return fmt.Sprintf("Variable(%s)", n.Name) +} diff --git a/config/lang/parse_test.go b/config/lang/parse_test.go index d8e2c7236..eb4670644 100644 --- a/config/lang/parse_test.go +++ b/config/lang/parse_test.go @@ -23,6 +23,16 @@ func TestParse(t *testing.T) { }, }, + { + "$${var.foo}", + false, + &ast.LiteralNode{ + Value: "${var.foo}", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + }, + }, + { "foo ${var.bar}", false, From d1a0ea9d9b1c8a6ef26c26cd933208bd9f008344 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 13 Jan 2015 09:46:13 -0800 Subject: [PATCH 18/31] config/lang: make formal Stack object --- config/lang/engine.go | 47 +++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/config/lang/engine.go b/config/lang/engine.go index 8b2a09a6d..8b4b60793 100644 --- a/config/lang/engine.go +++ b/config/lang/engine.go @@ -44,7 +44,7 @@ func (e *Engine) Execute(root ast.Node) (interface{}, ast.Type, error) { type executeVisitor struct { Scope *Scope - stack []*ast.LiteralNode + stack EngineStack err error lock sync.Mutex } @@ -58,15 +58,15 @@ func (v *executeVisitor) Visit(root ast.Node) (interface{}, ast.Type, error) { // Get our result and clear out everything else var result *ast.LiteralNode - if len(v.stack) > 0 { - result = v.stack[len(v.stack)-1] + if v.stack.Len() > 0 { + result = v.stack.Pop() } else { result = new(ast.LiteralNode) } resultErr := v.err // Clear everything else so we aren't just dangling - v.stack = nil + v.stack.Reset() v.err = nil return result.Value, result.Type, resultErr @@ -102,7 +102,7 @@ func (v *executeVisitor) visitCall(n *ast.Call) { // The arguments are on the stack in reverse order, so pop them off. args := make([]interface{}, len(n.Args)) for i, _ := range n.Args { - node := v.stackPop() + node := v.stack.Pop() args[len(n.Args)-1-i] = node.Value } @@ -114,7 +114,7 @@ func (v *executeVisitor) visitCall(n *ast.Call) { } // Push the result - v.stackPush(&ast.LiteralNode{ + v.stack.Push(&ast.LiteralNode{ Value: result, Type: function.ReturnType, }) @@ -125,7 +125,7 @@ func (v *executeVisitor) visitConcat(n *ast.Concat) { // order. So pop them off, reverse their order, and concatenate. nodes := make([]*ast.LiteralNode, 0, len(n.Exprs)) for range n.Exprs { - nodes = append(nodes, v.stackPop()) + nodes = append(nodes, v.stack.Pop()) } var buf bytes.Buffer @@ -133,14 +133,14 @@ func (v *executeVisitor) visitConcat(n *ast.Concat) { buf.WriteString(nodes[i].Value.(string)) } - v.stackPush(&ast.LiteralNode{ + v.stack.Push(&ast.LiteralNode{ Value: buf.String(), Type: ast.TypeString, }) } func (v *executeVisitor) visitLiteral(n *ast.LiteralNode) { - v.stack = append(v.stack, n) + v.stack.Push(n) } func (v *executeVisitor) visitVariableAccess(n *ast.VariableAccess) { @@ -151,22 +151,39 @@ func (v *executeVisitor) visitVariableAccess(n *ast.VariableAccess) { return } - v.stack = append(v.stack, &ast.LiteralNode{ + v.stack.Push(&ast.LiteralNode{ Value: variable.Value, Type: variable.Type, }) } -func (v *executeVisitor) stackPush(n *ast.LiteralNode) { - v.stack = append(v.stack, n) +// EngineStack is a stack of ast.LiteralNodes that the Engine keeps track +// of during execution. This is currently backed by a dumb slice, but can be +// replaced with a better data structure at some point in the future if this +// turns out to require optimization. +type EngineStack struct { + stack []*ast.LiteralNode } -func (v *executeVisitor) stackPop() *ast.LiteralNode { - var x *ast.LiteralNode - x, v.stack = v.stack[len(v.stack)-1], v.stack[:len(v.stack)-1] +func (s *EngineStack) Len() int { + return len(s.stack) +} + +func (s *EngineStack) Push(n *ast.LiteralNode) { + s.stack = append(s.stack, n) +} + +func (s *EngineStack) Pop() *ast.LiteralNode { + x := s.stack[len(s.stack)-1] + s.stack[len(s.stack)-1] = nil + s.stack = s.stack[:len(s.stack)-1] return x } +func (s *EngineStack) Reset() { + s.stack = nil +} + // Scope represents a lookup scope for execution. type Scope struct { // VarMap and FuncMap are the mappings of identifiers to functions From c424a8a815a267007a4d677d45a66df031cc0171 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 13 Jan 2015 10:10:33 -0800 Subject: [PATCH 19/31] config/lang: FixedValueTransform --- config/lang/transform_fixed.go | 26 ++++++++++++++++ config/lang/transform_fixed_test.go | 48 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 config/lang/transform_fixed.go create mode 100644 config/lang/transform_fixed_test.go diff --git a/config/lang/transform_fixed.go b/config/lang/transform_fixed.go new file mode 100644 index 000000000..c551423df --- /dev/null +++ b/config/lang/transform_fixed.go @@ -0,0 +1,26 @@ +package lang + +import ( + "github.com/hashicorp/terraform/config/lang/ast" +) + +// FixedValueTransform transforms an AST to return a fixed value for +// all interpolations. i.e. you can make "hello ${anything}" always +// turn into "hello foo". +func FixedValueTransform(root ast.Node, Value *ast.LiteralNode) ast.Node { + // We visit the nodes in top-down order + result := root + switch n := result.(type) { + case *ast.Concat: + for i, v := range n.Exprs { + n.Exprs[i] = FixedValueTransform(v, Value) + } + case *ast.LiteralNode: + // We keep it as-is + default: + // Anything else we replace + result = Value + } + + return result +} diff --git a/config/lang/transform_fixed_test.go b/config/lang/transform_fixed_test.go new file mode 100644 index 000000000..c5e5a1ca3 --- /dev/null +++ b/config/lang/transform_fixed_test.go @@ -0,0 +1,48 @@ +package lang + +import ( + "reflect" + "testing" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +func TestFixedValueTransform(t *testing.T) { + cases := []struct { + Input ast.Node + Output ast.Node + }{ + { + &ast.LiteralNode{Value: 42}, + &ast.LiteralNode{Value: 42}, + }, + + { + &ast.VariableAccess{Name: "bar"}, + &ast.LiteralNode{Value: "foo"}, + }, + + { + &ast.Concat{ + Exprs: []ast.Node{ + &ast.VariableAccess{Name: "bar"}, + &ast.LiteralNode{Value: 42}, + }, + }, + &ast.Concat{ + Exprs: []ast.Node{ + &ast.LiteralNode{Value: "foo"}, + &ast.LiteralNode{Value: 42}, + }, + }, + }, + } + + value := &ast.LiteralNode{Value: "foo"} + for _, tc := range cases { + actual := FixedValueTransform(tc.Input, value) + if !reflect.DeepEqual(actual, tc.Output) { + t.Fatalf("bad: %#v\n\nInput: %#v", actual, tc.Input) + } + } +} From e68fbceebcd6316073ac826866c9ddbfee63891c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 13 Jan 2015 10:27:31 -0800 Subject: [PATCH 20/31] config/lang: fix bug parsing empty strings --- config/lang/lang.y | 9 +++- config/lang/parse_test.go | 10 +++++ config/lang/y.go | 91 +++++++++++++++++++++------------------ config/lang/y.output | 65 ++++++++++++++-------------- 4 files changed, 101 insertions(+), 74 deletions(-) diff --git a/config/lang/lang.y b/config/lang/lang.y index 241b286fb..310af6128 100644 --- a/config/lang/lang.y +++ b/config/lang/lang.y @@ -30,7 +30,14 @@ import ( %% top: - literalModeTop + { + parserResult = &ast.LiteralNode{ + Value: "", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + } + } +| literalModeTop { parserResult = $1 } diff --git a/config/lang/parse_test.go b/config/lang/parse_test.go index eb4670644..05b37ac18 100644 --- a/config/lang/parse_test.go +++ b/config/lang/parse_test.go @@ -13,6 +13,16 @@ func TestParse(t *testing.T) { Error bool Result ast.Node }{ + { + "", + false, + &ast.LiteralNode{ + Value: "", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + }, + }, + { "foo", false, diff --git a/config/lang/y.go b/config/lang/y.go index 07ae55cdc..a6b139520 100644 --- a/config/lang/y.go +++ b/config/lang/y.go @@ -48,7 +48,7 @@ const parserEofCode = 1 const parserErrCode = 2 const parserMaxDepth = 200 -//line lang.y:127 +//line lang.y:134 //line yacctab:1 var parserExca = []int{ @@ -57,7 +57,7 @@ var parserExca = []int{ -2, 0, } -const parserNprod = 16 +const parserNprod = 17 const parserPrivate = 57344 var parserTokenNames []string @@ -83,13 +83,13 @@ var parserPgo = []int{ } var parserR1 = []int{ - 0, 7, 4, 4, 5, 5, 2, 1, 1, 1, - 1, 1, 6, 6, 6, 3, + 0, 7, 7, 4, 4, 5, 5, 2, 1, 1, + 1, 1, 1, 6, 6, 6, 3, } var parserR2 = []int{ - 0, 1, 1, 2, 1, 1, 3, 1, 1, 1, - 1, 4, 0, 3, 1, 1, + 0, 0, 1, 1, 2, 1, 1, 3, 1, 1, + 1, 1, 4, 0, 3, 1, 1, } var parserChk = []int{ @@ -99,9 +99,9 @@ var parserChk = []int{ } var parserDef = []int{ - 0, -2, 1, 2, 4, 5, 15, 0, 3, 0, - 7, 8, 9, 10, 6, 12, 0, 14, 11, 0, - 13, + 1, -2, 2, 3, 5, 6, 16, 0, 4, 0, + 8, 9, 10, 11, 7, 13, 0, 15, 12, 0, + 14, } var parserTok1 = []int{ @@ -342,17 +342,26 @@ parserdefault: switch parsernt { case 1: - //line lang.y:34 + //line lang.y:33 + { + parserResult = &ast.LiteralNode{ + Value: "", + Type: ast.TypeString, + Posx: ast.Pos{Column: 1, Line: 1}, + } + } + case 2: + //line lang.y:41 { parserResult = parserS[parserpt-0].node } - case 2: - //line lang.y:40 + case 3: + //line lang.y:47 { parserVAL.node = parserS[parserpt-0].node } - case 3: - //line lang.y:44 + case 4: + //line lang.y:51 { var result []ast.Node if c, ok := parserS[parserpt-1].node.(*ast.Concat); ok { @@ -366,28 +375,28 @@ parserdefault: Posx: result[0].Pos(), } } - case 4: - //line lang.y:60 - { - parserVAL.node = parserS[parserpt-0].node - } case 5: - //line lang.y:64 + //line lang.y:67 { parserVAL.node = parserS[parserpt-0].node } case 6: - //line lang.y:70 - { - parserVAL.node = parserS[parserpt-1].node - } - case 7: - //line lang.y:76 + //line lang.y:71 { parserVAL.node = parserS[parserpt-0].node } + case 7: + //line lang.y:77 + { + parserVAL.node = parserS[parserpt-1].node + } case 8: - //line lang.y:80 + //line lang.y:83 + { + parserVAL.node = parserS[parserpt-0].node + } + case 9: + //line lang.y:87 { parserVAL.node = &ast.LiteralNode{ Value: parserS[parserpt-0].token.Value.(int), @@ -395,8 +404,8 @@ parserdefault: Posx: parserS[parserpt-0].token.Pos, } } - case 9: - //line lang.y:88 + case 10: + //line lang.y:95 { parserVAL.node = &ast.LiteralNode{ Value: parserS[parserpt-0].token.Value.(float64), @@ -404,33 +413,33 @@ parserdefault: Posx: parserS[parserpt-0].token.Pos, } } - case 10: - //line lang.y:96 + case 11: + //line lang.y:103 { parserVAL.node = &ast.VariableAccess{Name: parserS[parserpt-0].token.Value.(string), Posx: parserS[parserpt-0].token.Pos} } - case 11: - //line lang.y:100 + case 12: + //line lang.y:107 { parserVAL.node = &ast.Call{Func: parserS[parserpt-3].token.Value.(string), Args: parserS[parserpt-1].nodeList, Posx: parserS[parserpt-3].token.Pos} } - case 12: - //line lang.y:105 + case 13: + //line lang.y:112 { parserVAL.nodeList = nil } - case 13: - //line lang.y:109 + case 14: + //line lang.y:116 { parserVAL.nodeList = append(parserS[parserpt-2].nodeList, parserS[parserpt-0].node) } - case 14: - //line lang.y:113 + case 15: + //line lang.y:120 { parserVAL.nodeList = append(parserVAL.nodeList, parserS[parserpt-0].node) } - case 15: - //line lang.y:119 + case 16: + //line lang.y:126 { parserVAL.node = &ast.LiteralNode{ Value: parserS[parserpt-0].token.Value.(string), diff --git a/config/lang/y.output b/config/lang/y.output index ce531c461..df08ec4ad 100644 --- a/config/lang/y.output +++ b/config/lang/y.output @@ -1,10 +1,11 @@ state 0 $accept: .top $end + top: . (1) PROGRAM_BRACKET_LEFT shift 7 STRING shift 6 - . error + . reduce 1 (src line 32) interpolation goto 5 literal goto 4 @@ -20,39 +21,39 @@ state 1 state 2 - top: literalModeTop. (1) + top: literalModeTop. (2) literalModeTop: literalModeTop.literalModeValue PROGRAM_BRACKET_LEFT shift 7 STRING shift 6 - . reduce 1 (src line 32) + . reduce 2 (src line 40) interpolation goto 5 literal goto 4 literalModeValue goto 8 state 3 - literalModeTop: literalModeValue. (2) + literalModeTop: literalModeValue. (3) - . reduce 2 (src line 38) + . reduce 3 (src line 45) state 4 - literalModeValue: literal. (4) + literalModeValue: literal. (5) - . reduce 4 (src line 58) + . reduce 5 (src line 65) state 5 - literalModeValue: interpolation. (5) + literalModeValue: interpolation. (6) - . reduce 5 (src line 63) + . reduce 6 (src line 70) state 6 - literal: STRING. (15) + literal: STRING. (16) - . reduce 15 (src line 117) + . reduce 16 (src line 124) state 7 @@ -72,9 +73,9 @@ state 7 literalModeValue goto 3 state 8 - literalModeTop: literalModeTop literalModeValue. (3) + literalModeTop: literalModeTop literalModeValue. (4) - . reduce 3 (src line 43) + . reduce 4 (src line 50) state 9 @@ -86,52 +87,52 @@ state 9 state 10 literalModeTop: literalModeTop.literalModeValue - expr: literalModeTop. (7) + expr: literalModeTop. (8) PROGRAM_BRACKET_LEFT shift 7 STRING shift 6 - . reduce 7 (src line 74) + . reduce 8 (src line 81) interpolation goto 5 literal goto 4 literalModeValue goto 8 state 11 - expr: INTEGER. (8) + expr: INTEGER. (9) - . reduce 8 (src line 79) + . reduce 9 (src line 86) state 12 - expr: FLOAT. (9) + expr: FLOAT. (10) - . reduce 9 (src line 87) + . reduce 10 (src line 94) state 13 - expr: IDENTIFIER. (10) + expr: IDENTIFIER. (11) expr: IDENTIFIER.PAREN_LEFT args PAREN_RIGHT PAREN_LEFT shift 15 - . reduce 10 (src line 95) + . reduce 11 (src line 102) state 14 - interpolation: PROGRAM_BRACKET_LEFT expr PROGRAM_BRACKET_RIGHT. (6) + interpolation: PROGRAM_BRACKET_LEFT expr PROGRAM_BRACKET_RIGHT. (7) - . reduce 6 (src line 68) + . reduce 7 (src line 75) state 15 expr: IDENTIFIER PAREN_LEFT.args PAREN_RIGHT - args: . (12) + args: . (13) PROGRAM_BRACKET_LEFT shift 7 IDENTIFIER shift 13 INTEGER shift 11 FLOAT shift 12 STRING shift 6 - . reduce 12 (src line 104) + . reduce 13 (src line 111) expr goto 17 interpolation goto 5 @@ -150,15 +151,15 @@ state 16 state 17 - args: expr. (14) + args: expr. (15) - . reduce 14 (src line 112) + . reduce 15 (src line 119) state 18 - expr: IDENTIFIER PAREN_LEFT args PAREN_RIGHT. (11) + expr: IDENTIFIER PAREN_LEFT args PAREN_RIGHT. (12) - . reduce 11 (src line 99) + . reduce 12 (src line 106) state 19 @@ -178,13 +179,13 @@ state 19 literalModeValue goto 3 state 20 - args: args COMMA expr. (13) + args: args COMMA expr. (14) - . reduce 13 (src line 108) + . reduce 14 (src line 115) 14 terminals, 8 nonterminals -16 grammar rules, 21/2000 states +17 grammar rules, 21/2000 states 0 shift/reduce, 0 reduce/reduce conflicts reported 57 working sets used memory: parser 25/30000 From 740c25d4ea1db7576900f73f236cd27beb3f71f6 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 13 Jan 2015 10:27:57 -0800 Subject: [PATCH 21/31] config: convert to config/lang --- config/config.go | 44 ++++++++---- config/interpolate_walk.go | 111 ++++++++++++------------------ config/interpolate_walk_test.go | 118 ++++++-------------------------- config/raw_config.go | 35 ++++++++-- 4 files changed, 125 insertions(+), 183 deletions(-) diff --git a/config/config.go b/config/config.go index abe60db6e..7163e2829 100644 --- a/config/config.go +++ b/config/config.go @@ -8,6 +8,8 @@ import ( "strconv" "strings" + "github.com/hashicorp/terraform/config/lang" + "github.com/hashicorp/terraform/config/lang/ast" "github.com/hashicorp/terraform/flatmap" "github.com/hashicorp/terraform/helper/multierror" "github.com/mitchellh/mapstructure" @@ -169,7 +171,7 @@ func (c *Config) Validate() error { } interp := false - fn := func(i Interpolation) (string, error) { + fn := func(ast.Node) (string, error) { interp = true return "", nil } @@ -353,9 +355,18 @@ func (c *Config) Validate() error { } } - // Interpolate with a fixed number to verify that its a number - r.RawCount.interpolate(func(Interpolation) (string, error) { - return "5", nil + // Interpolate with a fixed number to verify that its a number. + r.RawCount.interpolate(func(root ast.Node) (string, error) { + // Execute the node but transform the AST so that it returns + // a fixed value of "5" for all interpolations. + var engine lang.Engine + out, _, err := engine.Execute(lang.FixedValueTransform( + root, &ast.LiteralNode{Value: "5", Type: ast.TypeString})) + if err != nil { + return "", err + } + + return out.(string), nil }) _, err := strconv.ParseInt(r.RawCount.Value().(string), 0, 0) if err != nil { @@ -465,20 +476,29 @@ func (c *Config) rawConfigs() map[string]*RawConfig { func (c *Config) validateVarContextFn( source string, errs *[]error) interpolationWalkerContextFunc { - return func(loc reflectwalk.Location, i Interpolation) { - vi, ok := i.(*VariableInterpolation) - if !ok { + return func(loc reflectwalk.Location, node ast.Node) { + if loc == reflectwalk.SliceElem { return } - rv, ok := vi.Variable.(*ResourceVariable) - if !ok { + vars, err := DetectVariables(node) + if err != nil { + // Ignore it since this will be caught during parse. This + // actually probably should never happen by the time this + // is called, but its okay. return } - if rv.Multi && rv.Index == -1 && loc != reflectwalk.SliceElem { - *errs = append(*errs, fmt.Errorf( - "%s: multi-variable must be in a slice", source)) + for _, v := range vars { + rv, ok := v.(*ResourceVariable) + if !ok { + return + } + + if rv.Multi && rv.Index == -1 { + *errs = append(*errs, fmt.Errorf( + "%s: multi-variable must be in a slice", source)) + } } } } diff --git a/config/interpolate_walk.go b/config/interpolate_walk.go index 7802537dc..17329e5a8 100644 --- a/config/interpolate_walk.go +++ b/config/interpolate_walk.go @@ -3,9 +3,10 @@ package config import ( "fmt" "reflect" - "regexp" "strings" + "github.com/hashicorp/terraform/config/lang" + "github.com/hashicorp/terraform/config/lang/ast" "github.com/mitchellh/reflectwalk" ) @@ -14,10 +15,6 @@ import ( // a value that a user is very unlikely to use (such as UUID). const InterpSplitDelim = `B780FFEC-B661-4EB8-9236-A01737AD98B6` -// interpRegexp is a regexp that matches interpolations such as ${foo.bar} -var interpRegexp *regexp.Regexp = regexp.MustCompile( - `(?i)(\$+)\{([\s*-.,\\/\(\):a-z0-9_"]+)\}`) - // interpolationWalker implements interfaces for the reflectwalk package // (github.com/mitchellh/reflectwalk) that can be used to automatically // execute a callback for an interpolation. @@ -50,7 +47,7 @@ type interpolationWalker struct { // // If Replace is set to false in interpolationWalker, then the replace // value can be anything as it will have no effect. -type interpolationWalkerFunc func(Interpolation) (string, error) +type interpolationWalkerFunc func(ast.Node) (string, error) // interpolationWalkerContextFunc is called by interpolationWalk if // ContextF is set. This receives both the interpolation and the location @@ -58,7 +55,7 @@ type interpolationWalkerFunc func(Interpolation) (string, error) // // This callback can be used to validate the location of the interpolation // within the configuration. -type interpolationWalkerContextFunc func(reflectwalk.Location, Interpolation) +type interpolationWalkerContextFunc func(reflectwalk.Location, ast.Node) func (w *interpolationWalker) Enter(loc reflectwalk.Location) error { w.loc = loc @@ -121,76 +118,54 @@ func (w *interpolationWalker) Primitive(v reflect.Value) error { return nil } - // XXX: This can be a lot more efficient if we used a real - // parser. A regexp is a hammer though that will get this working. + astRoot, err := lang.Parse(v.String()) + if err != nil { + return err + } - matches := interpRegexp.FindAllStringSubmatch(v.String(), -1) - if len(matches) == 0 { + // If the AST we got is just a literal string value, then we ignore it + if _, ok := astRoot.(*ast.LiteralNode); ok { return nil } - result := v.String() - for _, match := range matches { - dollars := len(match[1]) + if w.ContextF != nil { + w.ContextF(w.loc, astRoot) + } - // If there are even amounts of dollar signs, then it is escaped - if dollars%2 == 0 { - continue - } + if w.F == nil { + return nil + } - // Interpolation found, instantiate it - key := match[2] - - i, err := ExprParse(key) - if err != nil { - return err - } - - if w.ContextF != nil { - w.ContextF(w.loc, i) - } - - if w.F == nil { - continue - } - - replaceVal, err := w.F(i) - if err != nil { - return fmt.Errorf( - "%s: %s", - key, - err) - } - - if w.Replace { - // We need to determine if we need to remove this element - // if the result contains any "UnknownVariableValue" which is - // set if it is computed. This behavior is different if we're - // splitting (in a SliceElem) or not. - remove := false - if w.loc == reflectwalk.SliceElem { - parts := strings.Split(replaceVal, InterpSplitDelim) - for _, p := range parts { - if p == UnknownVariableValue { - remove = true - break - } - } - } else if replaceVal == UnknownVariableValue { - remove = true - } - if remove { - w.removeCurrent() - return nil - } - - // Replace in our interpolation and continue on. - result = strings.Replace(result, match[0], replaceVal, -1) - } + replaceVal, err := w.F(astRoot) + if err != nil { + return fmt.Errorf( + "%s in:\n\n%s", + err, v.String()) } if w.Replace { - resultVal := reflect.ValueOf(result) + // We need to determine if we need to remove this element + // if the result contains any "UnknownVariableValue" which is + // set if it is computed. This behavior is different if we're + // splitting (in a SliceElem) or not. + remove := false + if w.loc == reflectwalk.SliceElem { + parts := strings.Split(replaceVal, InterpSplitDelim) + for _, p := range parts { + if p == UnknownVariableValue { + remove = true + break + } + } + } else if replaceVal == UnknownVariableValue { + remove = true + } + if remove { + w.removeCurrent() + return nil + } + + resultVal := reflect.ValueOf(replaceVal) switch w.loc { case reflectwalk.MapKey: m := w.cs[len(w.cs)-1] diff --git a/config/interpolate_walk_test.go b/config/interpolate_walk_test.go index 6e8928694..9b2c34133 100644 --- a/config/interpolate_walk_test.go +++ b/config/interpolate_walk_test.go @@ -1,16 +1,18 @@ package config import ( + "fmt" "reflect" "testing" + "github.com/hashicorp/terraform/config/lang/ast" "github.com/mitchellh/reflectwalk" ) func TestInterpolationWalker_detect(t *testing.T) { cases := []struct { Input interface{} - Result []Interpolation + Result []string }{ { Input: map[string]interface{}{ @@ -23,13 +25,8 @@ func TestInterpolationWalker_detect(t *testing.T) { Input: map[string]interface{}{ "foo": "${var.foo}", }, - Result: []Interpolation{ - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "foo", - key: "var.foo", - }, - }, + Result: []string{ + "Variable(var.foo)", }, }, @@ -37,19 +34,8 @@ func TestInterpolationWalker_detect(t *testing.T) { Input: map[string]interface{}{ "foo": "${aws_instance.foo.*.num}", }, - Result: []Interpolation{ - &VariableInterpolation{ - Variable: &ResourceVariable{ - Type: "aws_instance", - Name: "foo", - Field: "num", - - Multi: true, - Index: -1, - - key: "aws_instance.foo.*.num", - }, - }, + Result: []string{ + "Variable(aws_instance.foo.*.num)", }, }, @@ -57,18 +43,8 @@ func TestInterpolationWalker_detect(t *testing.T) { Input: map[string]interface{}{ "foo": "${lookup(var.foo)}", }, - Result: []Interpolation{ - &FunctionInterpolation{ - Func: nil, - Args: []Interpolation{ - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "foo", - key: "var.foo", - }, - }, - }, - }, + Result: []string{ + "Call(lookup, Variable(var.foo))", }, }, @@ -76,15 +52,8 @@ func TestInterpolationWalker_detect(t *testing.T) { Input: map[string]interface{}{ "foo": `${file("test.txt")}`, }, - Result: []Interpolation{ - &FunctionInterpolation{ - Func: nil, - Args: []Interpolation{ - &LiteralInterpolation{ - Literal: "test.txt", - }, - }, - }, + Result: []string{ + "Call(file, Literal(TypeString, test.txt))", }, }, @@ -92,15 +61,8 @@ func TestInterpolationWalker_detect(t *testing.T) { Input: map[string]interface{}{ "foo": `${file("foo/bar.txt")}`, }, - Result: []Interpolation{ - &FunctionInterpolation{ - Func: nil, - Args: []Interpolation{ - &LiteralInterpolation{ - Literal: "foo/bar.txt", - }, - }, - }, + Result: []string{ + "Call(file, Literal(TypeString, foo/bar.txt))", }, }, @@ -108,25 +70,8 @@ func TestInterpolationWalker_detect(t *testing.T) { Input: map[string]interface{}{ "foo": `${join(",", foo.bar.*.id)}`, }, - Result: []Interpolation{ - &FunctionInterpolation{ - Func: nil, - Args: []Interpolation{ - &LiteralInterpolation{ - Literal: ",", - }, - &VariableInterpolation{ - Variable: &ResourceVariable{ - Type: "foo", - Name: "bar", - Field: "id", - Multi: true, - Index: -1, - key: "foo.bar.*.id", - }, - }, - }, - }, + Result: []string{ + "Call(join, Literal(TypeString, ,), Variable(foo.bar.*.id))", }, }, @@ -134,27 +79,16 @@ func TestInterpolationWalker_detect(t *testing.T) { Input: map[string]interface{}{ "foo": `${concat("localhost", ":8080")}`, }, - Result: []Interpolation{ - &FunctionInterpolation{ - Func: nil, - Args: []Interpolation{ - &LiteralInterpolation{ - Literal: "localhost", - }, - &LiteralInterpolation{ - Literal: ":8080", - }, - }, - }, + Result: []string{ + "Call(concat, Literal(TypeString, localhost), Literal(TypeString, :8080))", }, }, } for i, tc := range cases { - var actual []Interpolation - - detectFn := func(i Interpolation) (string, error) { - actual = append(actual, i) + var actual []string + detectFn := func(root ast.Node) (string, error) { + actual = append(actual, fmt.Sprintf("%s", root)) return "", nil } @@ -163,14 +97,6 @@ func TestInterpolationWalker_detect(t *testing.T) { t.Fatalf("err: %s", err) } - for _, a := range actual { - // This is jank, but reflect.DeepEqual never has functions - // being the same. - if f, ok := a.(*FunctionInterpolation); ok { - f.Func = nil - } - } - if !reflect.DeepEqual(actual, tc.Result) { t.Fatalf("%d: bad:\n\n%#v", i, actual) } @@ -198,7 +124,7 @@ func TestInterpolationWalker_replace(t *testing.T) { "foo": "hello, ${var.foo}", }, Output: map[string]interface{}{ - "foo": "hello, bar", + "foo": "bar", }, Value: "bar", }, @@ -247,7 +173,7 @@ func TestInterpolationWalker_replace(t *testing.T) { } for i, tc := range cases { - fn := func(i Interpolation) (string, error) { + fn := func(ast.Node) (string, error) { return tc.Value, nil } diff --git a/config/raw_config.go b/config/raw_config.go index e1c6c4108..11481b087 100644 --- a/config/raw_config.go +++ b/config/raw_config.go @@ -4,6 +4,8 @@ import ( "bytes" "encoding/gob" + "github.com/hashicorp/terraform/config/lang" + "github.com/hashicorp/terraform/config/lang/ast" "github.com/mitchellh/copystructure" "github.com/mitchellh/reflectwalk" ) @@ -26,7 +28,7 @@ const UnknownVariableValue = "74D93920-ED26-11E3-AC10-0800200C9A66" type RawConfig struct { Key string Raw map[string]interface{} - Interpolations []Interpolation + Interpolations []ast.Node Variables map[string]InterpolatedVariable config map[string]interface{} @@ -79,8 +81,23 @@ func (r *RawConfig) Config() map[string]interface{} { // // If a variable key is missing, this will panic. func (r *RawConfig) Interpolate(vs map[string]string) error { - return r.interpolate(func(i Interpolation) (string, error) { - return i.Interpolate(vs) + varMap := make(map[string]lang.Variable) + for k, v := range vs { + varMap[k] = lang.Variable{Value: v, Type: ast.TypeString} + } + engine := &lang.Engine{ + GlobalScope: &lang.Scope{ + VarMap: varMap, + }, + } + + return r.interpolate(func(root ast.Node) (string, error) { + out, _, err := engine.Execute(root) + if err != nil { + return "", err + } + + return out.(string), nil }) } @@ -89,15 +106,19 @@ func (r *RawConfig) init() error { r.Interpolations = nil r.Variables = nil - fn := func(i Interpolation) (string, error) { - r.Interpolations = append(r.Interpolations, i) + fn := func(node ast.Node) (string, error) { + r.Interpolations = append(r.Interpolations, node) + vars, err := DetectVariables(node) + if err != nil { + return "", err + } - for k, v := range i.Variables() { + for _, v := range vars { if r.Variables == nil { r.Variables = make(map[string]InterpolatedVariable) } - r.Variables[k] = v + r.Variables[v.FullKey()] = v } return "", nil From 5abbde3ac933bb4fbfe2cf25bedf29b997ca878b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 13 Jan 2015 10:32:03 -0800 Subject: [PATCH 22/31] config: remove unused files --- .gitignore | 2 - Makefile | 21 ++---- config/expr.y | 91 ----------------------- config/expr_lex.go | 134 ---------------------------------- config/expr_lex_test.go | 51 ------------- config/expr_parse.go | 40 ----------- config/expr_parse_test.go | 148 -------------------------------------- 7 files changed, 7 insertions(+), 480 deletions(-) delete mode 100644 config/expr.y delete mode 100644 config/expr_lex.go delete mode 100644 config/expr_lex_test.go delete mode 100644 config/expr_parse.go delete mode 100644 config/expr_parse_test.go diff --git a/.gitignore b/.gitignore index 72a053db9..e852cc3b5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,6 @@ example.tf terraform.tfplan terraform.tfstate bin/ -config/y.go -config/y.output modules-dev/ pkg/ vendor/ diff --git a/Makefile b/Makefile index cfc16cb07..44922e283 100644 --- a/Makefile +++ b/Makefile @@ -2,26 +2,26 @@ TEST?=./... default: test -bin: config/y.go generate +bin: generate @sh -c "'$(CURDIR)/scripts/build.sh'" -dev: config/y.go generate +dev: generate @TF_DEV=1 sh -c "'$(CURDIR)/scripts/build.sh'" -test: config/y.go generate +test: generate TF_ACC= go test $(TEST) $(TESTARGS) -timeout=10s -parallel=4 -testacc: config/y.go generate +testacc: generate @if [ "$(TEST)" = "./..." ]; then \ echo "ERROR: Set TEST to a specific package"; \ exit 1; \ fi TF_ACC=1 go test $(TEST) -v $(TESTARGS) -timeout 45m -testrace: config/y.go generate +testrace: generate TF_ACC= go test -race $(TEST) $(TESTARGS) -updatedeps: config/y.go +updatedeps: go get -u golang.org/x/tools/cmd/stringer # Go 1.4 changed the format of `go get` a bit by requiring the # canonical full path. We work around this and just force. @@ -31,14 +31,7 @@ updatedeps: config/y.go go get -f -u -v ./...; \ fi -config/y.go: config/expr.y - cd config/ && \ - go tool yacc -p "expr" expr.y - -clean: - rm config/y.go - generate: go generate ./... -.PHONY: bin clean default generate test updatedeps +.PHONY: bin default generate test updatedeps diff --git a/config/expr.y b/config/expr.y deleted file mode 100644 index 1c295cce6..000000000 --- a/config/expr.y +++ /dev/null @@ -1,91 +0,0 @@ -// This is the yacc input for creating the parser for interpolation -// expressions in Go. - -// To build it: -// -// go tool yacc -p "expr" expr.y (produces y.go) -// - -%{ -package config - -import ( - "fmt" -) - -%} - -%union { - expr Interpolation - str string - variable InterpolatedVariable - args []Interpolation -} - -%type args -%type expr -%type string -%type variable - -%token STRING IDENTIFIER -%token COMMA LEFTPAREN RIGHTPAREN - -%% - -top: - expr - { - exprResult = $1 - } - -expr: - string - { - $$ = &LiteralInterpolation{Literal: $1} - } -| variable - { - $$ = &VariableInterpolation{Variable: $1} - } -| IDENTIFIER LEFTPAREN args RIGHTPAREN - { - f, ok := Funcs[$1] - if !ok { - exprErrors = append(exprErrors, fmt.Errorf( - "Unknown function: %s", $1)) - } - - $$ = &FunctionInterpolation{Func: f, Args: $3} - } - -args: - { - $$ = nil - } -| args COMMA expr - { - $$ = append($1, $3) - } -| expr - { - $$ = append($$, $1) - } - -string: - STRING - { - $$ = $1 - } - -variable: - IDENTIFIER - { - var err error - $$, err = NewInterpolatedVariable($1) - if err != nil { - exprErrors = append(exprErrors, fmt.Errorf( - "Error parsing variable '%s': %s", $1, err)) - } - } - -%% diff --git a/config/expr_lex.go b/config/expr_lex.go deleted file mode 100644 index f94e947ba..000000000 --- a/config/expr_lex.go +++ /dev/null @@ -1,134 +0,0 @@ -package config - -import ( - "bytes" - "fmt" - "log" - "unicode" - "unicode/utf8" -) - -// The parser expects the lexer to return 0 on EOF. -const lexEOF = 0 - -// The parser uses the type Lex as a lexer. It must provide -// the methods Lex(*SymType) int and Error(string). -type exprLex struct { - Err error - Input string - - pos int - width int -} - -// The parser calls this method to get each new token. -func (x *exprLex) Lex(yylval *exprSymType) int { - for { - c := x.next() - if c == lexEOF { - return lexEOF - } - - // Ignore all whitespace - if unicode.IsSpace(c) { - continue - } - - switch c { - case '"': - return x.lexString(yylval) - case ',': - return COMMA - case '(': - return LEFTPAREN - case ')': - return RIGHTPAREN - default: - x.backup() - return x.lexId(yylval) - } - } -} - -func (x *exprLex) lexId(yylval *exprSymType) int { - var b bytes.Buffer - for { - c := x.next() - if c == lexEOF { - break - } - - // If this isn't a character we want in an ID, return out. - // One day we should make this a regexp. - if c != '_' && - c != '-' && - c != '.' && - c != '*' && - !unicode.IsLetter(c) && - !unicode.IsNumber(c) { - x.backup() - break - } - - if _, err := b.WriteRune(c); err != nil { - log.Printf("ERR: %s", err) - return lexEOF - } - } - - yylval.str = b.String() - return IDENTIFIER -} - -func (x *exprLex) lexString(yylval *exprSymType) int { - var b bytes.Buffer - for { - c := x.next() - if c == lexEOF { - break - } - - // String end - if c == '"' { - break - } - - if _, err := b.WriteRune(c); err != nil { - log.Printf("ERR: %s", err) - return lexEOF - } - } - - yylval.str = b.String() - return STRING -} - -// Return the next rune for the lexer. -func (x *exprLex) next() rune { - if int(x.pos) >= len(x.Input) { - x.width = 0 - return lexEOF - } - - r, w := utf8.DecodeRuneInString(x.Input[x.pos:]) - x.width = w - x.pos += x.width - return r -} - -// peek returns but does not consume the next rune in the input -func (x *exprLex) peek() rune { - r := x.next() - x.backup() - return r -} - -// backup steps back one rune. Can only be called once per next. -func (x *exprLex) backup() { - x.pos -= x.width -} - -// The parser calls this method on a parse error. -func (x *exprLex) Error(s string) { - x.Err = fmt.Errorf("parse error: %s", s) -} diff --git a/config/expr_lex_test.go b/config/expr_lex_test.go deleted file mode 100644 index 8e94d86f3..000000000 --- a/config/expr_lex_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package config - -import ( - "io/ioutil" - "path/filepath" - "reflect" - "testing" -) - -func TestLex(t *testing.T) { - cases := []struct { - Input string - Output []int - }{ - { - "concat.hcl", - []int{IDENTIFIER, LEFTPAREN, - STRING, COMMA, STRING, COMMA, STRING, - RIGHTPAREN, lexEOF}, - }, - } - - for _, tc := range cases { - d, err := ioutil.ReadFile(filepath.Join( - fixtureDir, "interpolations", tc.Input)) - if err != nil { - t.Fatalf("err: %s", err) - } - - l := &exprLex{Input: string(d)} - var actual []int - for { - token := l.Lex(new(exprSymType)) - actual = append(actual, token) - - if token == lexEOF { - break - } - - if len(actual) > 500 { - t.Fatalf("Input:%s\n\nExausted.", tc.Input) - } - } - - if !reflect.DeepEqual(actual, tc.Output) { - t.Fatalf( - "Input: %s\n\nBad: %#v\n\nExpected: %#v", - tc.Input, actual, tc.Output) - } - } -} diff --git a/config/expr_parse.go b/config/expr_parse.go deleted file mode 100644 index c6fb39f87..000000000 --- a/config/expr_parse.go +++ /dev/null @@ -1,40 +0,0 @@ -package config - -import ( - "sync" - - "github.com/hashicorp/terraform/helper/multierror" -) - -// exprErrors are the errors built up from parsing. These should not -// be accessed directly. -var exprErrors []error -var exprLock sync.Mutex -var exprResult Interpolation - -// ExprParse parses the given expression and returns an executable -// Interpolation. -func ExprParse(v string) (Interpolation, error) { - exprLock.Lock() - defer exprLock.Unlock() - exprErrors = nil - exprResult = nil - - // Parse - lex := &exprLex{Input: v} - exprParse(lex) - - // Build up the errors - var err error - if lex.Err != nil { - err = multierror.ErrorAppend(err, lex.Err) - } - if len(exprErrors) > 0 { - err = multierror.ErrorAppend(err, exprErrors...) - } - if err != nil { - exprResult = nil - } - - return exprResult, err -} diff --git a/config/expr_parse_test.go b/config/expr_parse_test.go deleted file mode 100644 index da77c7bda..000000000 --- a/config/expr_parse_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package config - -import ( - "reflect" - "testing" -) - -func TestExprParse(t *testing.T) { - cases := []struct { - Input string - Result Interpolation - Error bool - }{ - { - "foo", - nil, - true, - }, - - { - `"foo"`, - &LiteralInterpolation{Literal: "foo"}, - false, - }, - - { - "var.foo", - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "foo", - key: "var.foo", - }, - }, - false, - }, - - { - "module.foo.bar", - &VariableInterpolation{ - Variable: &ModuleVariable{ - Name: "foo", - Field: "bar", - key: "module.foo.bar", - }, - }, - false, - }, - - { - "lookup(var.foo, var.bar)", - &FunctionInterpolation{ - Func: nil, // Funcs["lookup"] - Args: []Interpolation{ - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "foo", - key: "var.foo", - }, - }, - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "bar", - key: "var.bar", - }, - }, - }, - }, - false, - }, - - { - "lookup(var.foo, lookup(var.baz, var.bar))", - &FunctionInterpolation{ - Func: nil, // Funcs["lookup"] - Args: []Interpolation{ - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "foo", - key: "var.foo", - }, - }, - &FunctionInterpolation{ - Func: nil, // Funcs["lookup"] - Args: []Interpolation{ - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "baz", - key: "var.baz", - }, - }, - &VariableInterpolation{ - Variable: &UserVariable{ - Name: "bar", - key: "var.bar", - }, - }, - }, - }, - }, - }, - false, - }, - - { - `concat("foo","-","0.0/16")`, - &FunctionInterpolation{ - Func: nil, // Funcs["lookup"] - Args: []Interpolation{ - &LiteralInterpolation{Literal: "foo"}, - &LiteralInterpolation{Literal: "-"}, - &LiteralInterpolation{Literal: "0.0/16"}, - }, - }, - false, - }, - } - - for i, tc := range cases { - actual, err := ExprParse(tc.Input) - if (err != nil) != tc.Error { - t.Fatalf("%d. Error: %s", i, err) - } - - // This is jank, but reflect.DeepEqual never has functions - // being the same. - f, ok := actual.(*FunctionInterpolation) - if ok { - fs := make([]*FunctionInterpolation, 1) - fs[0] = f - for len(fs) > 0 { - f := fs[0] - fs = fs[1:] - - f.Func = nil - for _, a := range f.Args { - f, ok := a.(*FunctionInterpolation) - if ok { - fs = append(fs, f) - } - } - } - } - - if !reflect.DeepEqual(actual, tc.Result) { - t.Fatalf("%d bad: %#v", i, actual) - } - } -} From 8ce7ef6188dc174cbab7befeb6bde3be69719926 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 13 Jan 2015 11:24:42 -0800 Subject: [PATCH 23/31] config/lang: implement identifier semantic check --- config/lang/check_identifier.go | 79 ++++++++++++++++++++++++ config/lang/check_identifier_test.go | 89 ++++++++++++++++++++++++++++ config/lang/engine.go | 18 +++++- 3 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 config/lang/check_identifier.go create mode 100644 config/lang/check_identifier_test.go diff --git a/config/lang/check_identifier.go b/config/lang/check_identifier.go new file mode 100644 index 000000000..2cc49bad1 --- /dev/null +++ b/config/lang/check_identifier.go @@ -0,0 +1,79 @@ +package lang + +import ( + "fmt" + "sync" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +// IdentifierCheck is a SemanticCheck that checks that all identifiers +// resolve properly and that the right number of arguments are passed +// to functions. +type IdentifierCheck struct { + Scope *Scope + + err error + lock sync.Mutex +} + +func (c *IdentifierCheck) Visit(root ast.Node) error { + c.lock.Lock() + defer c.lock.Unlock() + defer c.reset() + root.Accept(c.visit) + return c.err +} + +func (c *IdentifierCheck) visit(raw ast.Node) { + if c.err != nil { + return + } + + switch n := raw.(type) { + case *ast.Call: + c.visitCall(n) + case *ast.VariableAccess: + c.visitVariableAccess(n) + case *ast.Concat: + // Ignore + case *ast.LiteralNode: + // Ignore + default: + c.createErr(n, fmt.Sprintf("unknown node: %#v", raw)) + } +} + +func (c *IdentifierCheck) visitCall(n *ast.Call) { + // Look up the function in the map + function, ok := c.Scope.LookupFunc(n.Func) + if !ok { + c.createErr(n, fmt.Sprintf("unknown function called: %s", n.Func)) + return + } + + // Verify the number of arguments + if len(n.Args) != len(function.ArgTypes) { + c.createErr(n, fmt.Sprintf( + "%s: expected %d arguments, got %d", + n.Func, len(function.ArgTypes), len(n.Args))) + return + } +} + +func (c *IdentifierCheck) visitVariableAccess(n *ast.VariableAccess) { + // Look up the variable in the map + if _, ok := c.Scope.LookupVar(n.Name); !ok { + c.createErr(n, fmt.Sprintf( + "unknown variable accessed: %s", n.Name)) + return + } +} + +func (c *IdentifierCheck) createErr(n ast.Node, str string) { + c.err = fmt.Errorf("%s: %s", n.Pos(), str) +} + +func (c *IdentifierCheck) reset() { + c.err = nil +} diff --git a/config/lang/check_identifier_test.go b/config/lang/check_identifier_test.go new file mode 100644 index 000000000..52460aa52 --- /dev/null +++ b/config/lang/check_identifier_test.go @@ -0,0 +1,89 @@ +package lang + +import ( + "testing" + + "github.com/hashicorp/terraform/config/lang/ast" +) + +func TestIdentifierCheck(t *testing.T) { + cases := []struct { + Input string + Scope *Scope + Error bool + }{ + { + "foo", + &Scope{}, + false, + }, + + { + "foo ${bar} success", + &Scope{ + VarMap: map[string]Variable{ + "bar": Variable{ + Value: "baz", + Type: ast.TypeString, + }, + }, + }, + false, + }, + + { + "foo ${bar}", + &Scope{}, + true, + }, + + { + "foo ${rand()} success", + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, + + { + "foo ${rand()}", + &Scope{}, + true, + }, + + { + "foo ${rand(42)} ", + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + true, + }, + } + + for _, tc := range cases { + node, err := Parse(tc.Input) + if err != nil { + t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) + } + + visitor := &IdentifierCheck{Scope: tc.Scope} + err = visitor.Visit(node) + if (err != nil) != tc.Error { + t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) + } + } +} diff --git a/config/lang/engine.go b/config/lang/engine.go index 8b4b60793..96eb27c64 100644 --- a/config/lang/engine.go +++ b/config/lang/engine.go @@ -27,10 +27,22 @@ type SemanticChecker func(ast.Node) error // Execute executes the given ast.Node and returns its final value, its // type, and an error if one exists. func (e *Engine) Execute(root ast.Node) (interface{}, ast.Type, error) { - // Run the type checker + // Build our own semantic checks that we always run tv := &TypeVisitor{Scope: e.GlobalScope} - if err := tv.Visit(root); err != nil { - return nil, ast.TypeInvalid, err + ic := &IdentifierCheck{Scope: e.GlobalScope} + + // Build up the semantic checks for execution + checks := make( + []SemanticChecker, len(e.SemanticChecks), len(e.SemanticChecks)+2) + copy(checks, e.SemanticChecks) + checks = append(checks, ic.Visit) + checks = append(checks, tv.Visit) + + // Run the semantic checks + for _, check := range checks { + if err := check(root); err != nil { + return nil, ast.TypeInvalid, err + } } // Execute From 2b679572b4744ca8332d2987c424dd3574f9f8aa Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 13 Jan 2015 11:25:46 -0800 Subject: [PATCH 24/31] config/lang: TypeCheck --- .../lang/{visitor_types.go => check_types.go} | 24 +++++++++---------- ...itor_types_test.go => check_types_test.go} | 4 ++-- config/lang/engine.go | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) rename config/lang/{visitor_types.go => check_types.go} (77%) rename config/lang/{visitor_types_test.go => check_types_test.go} (95%) diff --git a/config/lang/visitor_types.go b/config/lang/check_types.go similarity index 77% rename from config/lang/visitor_types.go rename to config/lang/check_types.go index 0eeb756ae..347a30614 100644 --- a/config/lang/visitor_types.go +++ b/config/lang/check_types.go @@ -7,9 +7,9 @@ import ( "github.com/hashicorp/terraform/config/lang/ast" ) -// TypeVisitor implements ast.Visitor for type checking an AST tree. +// TypeCheck implements ast.Visitor for type checking an AST tree. // It requires some configuration to look up the type of nodes. -type TypeVisitor struct { +type TypeCheck struct { Scope *Scope stack []ast.Type @@ -17,7 +17,7 @@ type TypeVisitor struct { lock sync.Mutex } -func (v *TypeVisitor) Visit(root ast.Node) error { +func (v *TypeCheck) Visit(root ast.Node) error { v.lock.Lock() defer v.lock.Unlock() defer v.reset() @@ -25,7 +25,7 @@ func (v *TypeVisitor) Visit(root ast.Node) error { return v.err } -func (v *TypeVisitor) visit(raw ast.Node) { +func (v *TypeCheck) visit(raw ast.Node) { if v.err != nil { return } @@ -44,7 +44,7 @@ func (v *TypeVisitor) visit(raw ast.Node) { } } -func (v *TypeVisitor) visitCall(n *ast.Call) { +func (v *TypeCheck) visitCall(n *ast.Call) { // Look up the function in the map function, ok := v.Scope.LookupFunc(n.Func) if !ok { @@ -72,7 +72,7 @@ func (v *TypeVisitor) visitCall(n *ast.Call) { v.stackPush(function.ReturnType) } -func (v *TypeVisitor) visitConcat(n *ast.Concat) { +func (v *TypeCheck) visitConcat(n *ast.Concat) { types := make([]ast.Type, len(n.Exprs)) for i, _ := range n.Exprs { types[len(n.Exprs)-1-i] = v.stackPop() @@ -91,11 +91,11 @@ func (v *TypeVisitor) visitConcat(n *ast.Concat) { v.stackPush(ast.TypeString) } -func (v *TypeVisitor) visitLiteral(n *ast.LiteralNode) { +func (v *TypeCheck) visitLiteral(n *ast.LiteralNode) { v.stackPush(n.Type) } -func (v *TypeVisitor) visitVariableAccess(n *ast.VariableAccess) { +func (v *TypeCheck) visitVariableAccess(n *ast.VariableAccess) { // Look up the variable in the map variable, ok := v.Scope.LookupVar(n.Name) if !ok { @@ -108,20 +108,20 @@ func (v *TypeVisitor) visitVariableAccess(n *ast.VariableAccess) { v.stackPush(variable.Type) } -func (v *TypeVisitor) createErr(n ast.Node, str string) { +func (v *TypeCheck) createErr(n ast.Node, str string) { v.err = fmt.Errorf("%s: %s", n.Pos(), str) } -func (v *TypeVisitor) reset() { +func (v *TypeCheck) reset() { v.stack = nil v.err = nil } -func (v *TypeVisitor) stackPush(t ast.Type) { +func (v *TypeCheck) stackPush(t ast.Type) { v.stack = append(v.stack, t) } -func (v *TypeVisitor) stackPop() ast.Type { +func (v *TypeCheck) stackPop() ast.Type { var x ast.Type x, v.stack = v.stack[len(v.stack)-1], v.stack[:len(v.stack)-1] return x diff --git a/config/lang/visitor_types_test.go b/config/lang/check_types_test.go similarity index 95% rename from config/lang/visitor_types_test.go rename to config/lang/check_types_test.go index 6ce21c81e..0c0747d65 100644 --- a/config/lang/visitor_types_test.go +++ b/config/lang/check_types_test.go @@ -6,7 +6,7 @@ import ( "github.com/hashicorp/terraform/config/lang/ast" ) -func TestTypeVisitor(t *testing.T) { +func TestTypeCheck(t *testing.T) { cases := []struct { Input string Scope *Scope @@ -113,7 +113,7 @@ func TestTypeVisitor(t *testing.T) { t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) } - visitor := &TypeVisitor{Scope: tc.Scope} + visitor := &TypeCheck{Scope: tc.Scope} err = visitor.Visit(node) if (err != nil) != tc.Error { t.Fatalf("Error: %s\n\nInput: %s", err, tc.Input) diff --git a/config/lang/engine.go b/config/lang/engine.go index 96eb27c64..558cf85fa 100644 --- a/config/lang/engine.go +++ b/config/lang/engine.go @@ -28,7 +28,7 @@ type SemanticChecker func(ast.Node) error // type, and an error if one exists. func (e *Engine) Execute(root ast.Node) (interface{}, ast.Type, error) { // Build our own semantic checks that we always run - tv := &TypeVisitor{Scope: e.GlobalScope} + tv := &TypeCheck{Scope: e.GlobalScope} ic := &IdentifierCheck{Scope: e.GlobalScope} // Build up the semantic checks for execution From 4ba7de17a9f48132bcee4b84bc07ed015594021e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 13 Jan 2015 11:27:10 -0800 Subject: [PATCH 25/31] config/lang: call the proper functions on Scope --- config/lang/engine.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/lang/engine.go b/config/lang/engine.go index 558cf85fa..5b4e6a2ea 100644 --- a/config/lang/engine.go +++ b/config/lang/engine.go @@ -105,7 +105,7 @@ func (v *executeVisitor) visit(raw ast.Node) { func (v *executeVisitor) visitCall(n *ast.Call) { // Look up the function in the map - function, ok := v.Scope.FuncMap[n.Func] + function, ok := v.Scope.LookupFunc(n.Func) if !ok { v.err = fmt.Errorf("unknown function called: %s", n.Func) return @@ -157,7 +157,7 @@ func (v *executeVisitor) visitLiteral(n *ast.LiteralNode) { func (v *executeVisitor) visitVariableAccess(n *ast.VariableAccess) { // Look up the variable in the map - variable, ok := v.Scope.VarMap[n.Name] + variable, ok := v.Scope.LookupVar(n.Name) if !ok { v.err = fmt.Errorf("unknown variable accessed: %s", n.Name) return From 1ccad4d7292cd41360a6beb220fc518b93ec7c2e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 13 Jan 2015 11:50:44 -0800 Subject: [PATCH 26/31] config: convert fucntions, put functions into Scope --- config/interpolate.go | 46 --------- config/interpolate_funcs.go | 110 ++++++++++------------ config/interpolate_funcs_test.go | 157 +++++++++++-------------------- config/interpolate_test.go | 49 ---------- config/raw_config.go | 25 +++-- 5 files changed, 118 insertions(+), 269 deletions(-) diff --git a/config/interpolate.go b/config/interpolate.go index 3e95642e0..806bac24d 100644 --- a/config/interpolate.go +++ b/config/interpolate.go @@ -2,17 +2,12 @@ package config import ( "fmt" - "regexp" "strconv" "strings" "github.com/hashicorp/terraform/config/lang/ast" ) -// We really need to replace this with a real parser. -var funcRegexp *regexp.Regexp = regexp.MustCompile( - `(?i)([a-z0-9_]+)\(\s*(?:([.a-z0-9_]+)\s*,\s*)*([.a-z0-9_]+)\s*\)`) - // Interpolation is something that can be contained in a "${}" in a // configuration value. // @@ -23,10 +18,6 @@ type Interpolation interface { Variables() map[string]InterpolatedVariable } -// InterpolationFunc is the function signature for implementing -// callable functions in Terraform configurations. -type InterpolationFunc func(map[string]string, ...string) (string, error) - // An InterpolatedVariable is a variable reference within an interpolation. // // Implementations of this interface represents various sources where @@ -35,13 +26,6 @@ type InterpolatedVariable interface { FullKey() string } -// FunctionInterpolation is an Interpolation that executes a function -// with some variable number of arguments to generate a value. -type FunctionInterpolation struct { - Func InterpolationFunc - Args []Interpolation -} - // LiteralInterpolation implements Interpolation for literals. Ex: // ${"foo"} will equal "foo". type LiteralInterpolation struct { @@ -130,36 +114,6 @@ func NewInterpolatedVariable(v string) (InterpolatedVariable, error) { } } -func (i *FunctionInterpolation) Interpolate( - vs map[string]string) (string, error) { - args := make([]string, len(i.Args)) - for idx, a := range i.Args { - v, err := a.Interpolate(vs) - if err != nil { - return "", err - } - - args[idx] = v - } - - return i.Func(vs, args...) -} - -func (i *FunctionInterpolation) GoString() string { - return fmt.Sprintf("*%#v", *i) -} - -func (i *FunctionInterpolation) Variables() map[string]InterpolatedVariable { - result := make(map[string]InterpolatedVariable) - for _, a := range i.Args { - for k, v := range a.Variables() { - result[k] = v - } - } - - return result -} - func (i *LiteralInterpolation) Interpolate( map[string]string) (string, error) { return i.Literal, nil diff --git a/config/interpolate_funcs.go b/config/interpolate_funcs.go index cbba62d0d..c67e3b733 100644 --- a/config/interpolate_funcs.go +++ b/config/interpolate_funcs.go @@ -1,73 +1,60 @@ package config import ( - "bytes" "fmt" "io/ioutil" "strconv" "strings" + + "github.com/hashicorp/terraform/config/lang" + "github.com/hashicorp/terraform/config/lang/ast" ) // Funcs is the mapping of built-in functions for configuration. -var Funcs map[string]InterpolationFunc +var Funcs map[string]lang.Function func init() { - Funcs = map[string]InterpolationFunc{ - "concat": interpolationFuncConcat, - "file": interpolationFuncFile, - "join": interpolationFuncJoin, - "lookup": interpolationFuncLookup, - "element": interpolationFuncElement, + Funcs = map[string]lang.Function{ + "file": interpolationFuncFile(), + "join": interpolationFuncJoin(), + //"lookup": interpolationFuncLookup(), + "element": interpolationFuncElement(), } } -// interpolationFuncConcat implements the "concat" function that allows -// strings to be joined together. -func interpolationFuncConcat( - vs map[string]string, args ...string) (string, error) { - var buf bytes.Buffer - - for _, a := range args { - if _, err := buf.WriteString(a); err != nil { - return "", err - } - } - - return buf.String(), nil -} - // interpolationFuncFile implements the "file" function that allows // loading contents from a file. -func interpolationFuncFile( - vs map[string]string, args ...string) (string, error) { - if len(args) != 1 { - return "", fmt.Errorf( - "file expects 1 arguments, got %d", len(args)) - } +func interpolationFuncFile() lang.Function { + return lang.Function{ + ArgTypes: []ast.Type{ast.TypeString}, + ReturnType: ast.TypeString, + Callback: func(args []interface{}) (interface{}, error) { + data, err := ioutil.ReadFile(args[0].(string)) + if err != nil { + return "", err + } - data, err := ioutil.ReadFile(args[0]) - if err != nil { - return "", err + return string(data), nil + }, } - - return string(data), nil } // interpolationFuncJoin implements the "join" function that allows // multi-variable values to be joined by some character. -func interpolationFuncJoin( - vs map[string]string, args ...string) (string, error) { - if len(args) < 2 { - return "", fmt.Errorf("join expects 2 arguments") - } +func interpolationFuncJoin() lang.Function { + return lang.Function{ + ArgTypes: []ast.Type{ast.TypeString, ast.TypeString}, + ReturnType: ast.TypeString, + Callback: func(args []interface{}) (interface{}, error) { + var list []string + for _, arg := range args[1:] { + parts := strings.Split(arg.(string), InterpSplitDelim) + list = append(list, parts...) + } - var list []string - for _, arg := range args[1:] { - parts := strings.Split(arg, InterpSplitDelim) - list = append(list, parts...) + return strings.Join(list, args[0].(string)), nil + }, } - - return strings.Join(list, args[0]), nil } // interpolationFuncLookup implements the "lookup" function that allows @@ -93,22 +80,21 @@ func interpolationFuncLookup( // interpolationFuncElement implements the "element" function that allows // a specific index to be looked up in a multi-variable value. Note that this will // wrap if the index is larger than the number of elements in the multi-variable value. -func interpolationFuncElement( - vs map[string]string, args ...string) (string, error) { - if len(args) != 2 { - return "", fmt.Errorf( - "element expects 2 arguments, got %d", len(args)) +func interpolationFuncElement() lang.Function { + return lang.Function{ + ArgTypes: []ast.Type{ast.TypeString, ast.TypeString}, + ReturnType: ast.TypeString, + Callback: func(args []interface{}) (interface{}, error) { + list := strings.Split(args[0].(string), InterpSplitDelim) + + index, err := strconv.Atoi(args[1].(string)) + if err != nil { + return "", fmt.Errorf( + "invalid number for index, got %s", args[1]) + } + + v := list[index%len(list)] + return v, nil + }, } - - list := strings.Split(args[0], InterpSplitDelim) - - index, err := strconv.Atoi(args[1]) - if err != nil { - return "", fmt.Errorf( - "invalid number for index, got %s", args[1]) - } - - v := list[index % len(list)] - - return v, nil } diff --git a/config/interpolate_funcs_test.go b/config/interpolate_funcs_test.go index 332b9af4b..3a9a86fe1 100644 --- a/config/interpolate_funcs_test.go +++ b/config/interpolate_funcs_test.go @@ -4,46 +4,12 @@ import ( "fmt" "io/ioutil" "os" + "reflect" "testing" + + "github.com/hashicorp/terraform/config/lang" ) -func TestInterpolateFuncConcat(t *testing.T) { - cases := []struct { - Args []string - Result string - Error bool - }{ - { - []string{"foo", "bar", "baz"}, - "foobarbaz", - false, - }, - - { - []string{"foo", "bar"}, - "foobar", - false, - }, - - { - []string{"foo"}, - "foo", - false, - }, - } - - for i, tc := range cases { - actual, err := interpolationFuncConcat(nil, tc.Args...) - if (err != nil) != tc.Error { - t.Fatalf("%d: err: %s", i, err) - } - - if actual != tc.Result { - t.Fatalf("%d: bad: %#v", i, actual) - } - } -} - func TestInterpolateFuncFile(t *testing.T) { tf, err := ioutil.TempFile("", "tf") if err != nil { @@ -54,94 +20,67 @@ func TestInterpolateFuncFile(t *testing.T) { tf.Close() defer os.Remove(path) - cases := []struct { - Args []string - Result string - Error bool - }{ + testFunction(t, []testFunctionCase{ { - []string{path}, + fmt.Sprintf(`${file("%s")}`, path), "foo", false, }, // Invalid path { - []string{"/i/dont/exist"}, - "", + `${file("/i/dont/exist")}`, + nil, true, }, // Too many args { - []string{"foo", "bar"}, - "", + `${file("foo", "bar")}`, + nil, true, }, - } - - for i, tc := range cases { - actual, err := interpolationFuncFile(nil, tc.Args...) - if (err != nil) != tc.Error { - t.Fatalf("%d: err: %s", i, err) - } - - if actual != tc.Result { - t.Fatalf("%d: bad: %#v", i, actual) - } - } + }) } func TestInterpolateFuncJoin(t *testing.T) { - cases := []struct { - Args []string - Result string - Error bool - }{ + testFunction(t, []testFunctionCase{ { - []string{","}, - "", + `${join(",")}`, + nil, true, }, { - []string{",", "foo"}, + `${join(",", "foo")}`, "foo", false, }, - { - []string{",", "foo", "bar"}, - "foo,bar", - false, - }, + /* + TODO + { + `${join(",", "foo", "bar")}`, + "foo,bar", + false, + }, + */ { - []string{ - ".", + fmt.Sprintf(`${join(".", "%s")}`, fmt.Sprintf( "foo%sbar%sbaz", InterpSplitDelim, - InterpSplitDelim), - }, + InterpSplitDelim)), "foo.bar.baz", false, }, - } - - for i, tc := range cases { - actual, err := interpolationFuncJoin(nil, tc.Args...) - if (err != nil) != tc.Error { - t.Fatalf("%d: err: %s", i, err) - } - - if actual != tc.Result { - t.Fatalf("%d: bad: %#v", i, actual) - } - } + }) } +/* func TestInterpolateFuncLookup(t *testing.T) { + testFunction(t, []testFunctionCase{ cases := []struct { M map[string]string Args []string @@ -189,48 +128,62 @@ func TestInterpolateFuncLookup(t *testing.T) { } } } +*/ func TestInterpolateFuncElement(t *testing.T) { - cases := []struct { - Args []string - Result string - Error bool - }{ + testFunction(t, []testFunctionCase{ { - []string{"foo" + InterpSplitDelim + "baz", "1"}, + fmt.Sprintf(`${element("%s", "1")}`, + "foo"+InterpSplitDelim+"baz"), "baz", false, }, { - []string{"foo", "0"}, + `${element("foo", "0")}`, "foo", false, }, // Invalid index should wrap vs. out-of-bounds { - []string{"foo" + InterpSplitDelim + "baz", "2"}, + fmt.Sprintf(`${element("%s", "2")}`, + "foo"+InterpSplitDelim+"baz"), "foo", false, }, // Too many args { - []string{"foo" + InterpSplitDelim + "baz", "0", "1"}, - "", + fmt.Sprintf(`${element("%s", "0", "2")}`, + "foo"+InterpSplitDelim+"baz"), + nil, true, }, - } + }) +} +type testFunctionCase struct { + Input string + Result interface{} + Error bool +} + +func testFunction(t *testing.T, cases []testFunctionCase) { for i, tc := range cases { - actual, err := interpolationFuncElement(nil, tc.Args...) + ast, err := lang.Parse(tc.Input) + if err != nil { + t.Fatalf("%d: err: %s", i, err) + } + + engine := langEngine(nil) + out, _, err := engine.Execute(ast) if (err != nil) != tc.Error { t.Fatalf("%d: err: %s", i, err) } - if actual != tc.Result { - t.Fatalf("%d: bad: %#v", i, actual) + if !reflect.DeepEqual(out, tc.Result) { + t.Fatalf("%d: bad: %#v", i, out) } } } diff --git a/config/interpolate_test.go b/config/interpolate_test.go index 2bafe4362..923ff660d 100644 --- a/config/interpolate_test.go +++ b/config/interpolate_test.go @@ -2,7 +2,6 @@ package config import ( "reflect" - "strings" "testing" "github.com/hashicorp/terraform/config/lang" @@ -123,54 +122,6 @@ func TestNewUserVariable_map(t *testing.T) { } } -func TestFunctionInterpolation_impl(t *testing.T) { - var _ Interpolation = new(FunctionInterpolation) -} - -func TestFunctionInterpolation(t *testing.T) { - v1, err := NewInterpolatedVariable("var.foo") - if err != nil { - t.Fatalf("err: %s", err) - } - - v2, err := NewInterpolatedVariable("var.bar") - if err != nil { - t.Fatalf("err: %s", err) - } - - fn := func(vs map[string]string, args ...string) (string, error) { - return strings.Join(args, " "), nil - } - - i := &FunctionInterpolation{ - Func: fn, - Args: []Interpolation{ - &VariableInterpolation{Variable: v1}, - &VariableInterpolation{Variable: v2}, - }, - } - - expected := map[string]InterpolatedVariable{ - "var.foo": v1, - "var.bar": v2, - } - if !reflect.DeepEqual(i.Variables(), expected) { - t.Fatalf("bad: %#v", i.Variables()) - } - - actual, err := i.Interpolate(map[string]string{ - "var.foo": "bar", - "var.bar": "baz", - }) - if err != nil { - t.Fatalf("err: %s", err) - } - - if actual != "bar baz" { - t.Fatalf("bad: %#v", actual) - } -} - func TestLiteralInterpolation_impl(t *testing.T) { var _ Interpolation = new(LiteralInterpolation) } diff --git a/config/raw_config.go b/config/raw_config.go index 11481b087..37867c638 100644 --- a/config/raw_config.go +++ b/config/raw_config.go @@ -81,16 +81,7 @@ func (r *RawConfig) Config() map[string]interface{} { // // If a variable key is missing, this will panic. func (r *RawConfig) Interpolate(vs map[string]string) error { - varMap := make(map[string]lang.Variable) - for k, v := range vs { - varMap[k] = lang.Variable{Value: v, Type: ast.TypeString} - } - engine := &lang.Engine{ - GlobalScope: &lang.Scope{ - VarMap: varMap, - }, - } - + engine := langEngine(vs) return r.interpolate(func(root ast.Node) (string, error) { out, _, err := engine.Execute(root) if err != nil { @@ -210,3 +201,17 @@ type gobRawConfig struct { Key string Raw map[string]interface{} } + +// langEngine returns the lang.Engine to use for evaluating configurations. +func langEngine(vs map[string]string) *lang.Engine { + varMap := make(map[string]lang.Variable) + for k, v := range vs { + varMap[k] = lang.Variable{Value: v, Type: ast.TypeString} + } + return &lang.Engine{ + GlobalScope: &lang.Scope{ + VarMap: varMap, + FuncMap: Funcs, + }, + } +} From 49fe0d5c7f2f7914ba13460db2a0c713d7583354 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 13 Jan 2015 11:54:30 -0800 Subject: [PATCH 27/31] config: remove a lot of unused stuff --- config/interpolate.go | 51 ----------------------------- config/interpolate_test.go | 67 -------------------------------------- 2 files changed, 118 deletions(-) diff --git a/config/interpolate.go b/config/interpolate.go index 806bac24d..e6a709d94 100644 --- a/config/interpolate.go +++ b/config/interpolate.go @@ -8,16 +8,6 @@ import ( "github.com/hashicorp/terraform/config/lang/ast" ) -// Interpolation is something that can be contained in a "${}" in a -// configuration value. -// -// Interpolations might be simple variable references, or it might be -// function calls, or even nested function calls. -type Interpolation interface { - Interpolate(map[string]string) (string, error) - Variables() map[string]InterpolatedVariable -} - // An InterpolatedVariable is a variable reference within an interpolation. // // Implementations of this interface represents various sources where @@ -26,18 +16,6 @@ type InterpolatedVariable interface { FullKey() string } -// LiteralInterpolation implements Interpolation for literals. Ex: -// ${"foo"} will equal "foo". -type LiteralInterpolation struct { - Literal string -} - -// VariableInterpolation implements Interpolation for simple variable -// interpolation. Ex: "${var.foo}" or "${aws_instance.foo.bar}" -type VariableInterpolation struct { - Variable InterpolatedVariable -} - // CountVariable is a variable for referencing information about // the count. type CountVariable struct { @@ -114,35 +92,6 @@ func NewInterpolatedVariable(v string) (InterpolatedVariable, error) { } } -func (i *LiteralInterpolation) Interpolate( - map[string]string) (string, error) { - return i.Literal, nil -} - -func (i *LiteralInterpolation) Variables() map[string]InterpolatedVariable { - return nil -} - -func (i *VariableInterpolation) Interpolate( - vs map[string]string) (string, error) { - v, ok := vs[i.Variable.FullKey()] - if !ok { - return "", fmt.Errorf( - "%s: value for variable not found", - i.Variable.FullKey()) - } - - return v, nil -} - -func (i *VariableInterpolation) GoString() string { - return fmt.Sprintf("*%#v", *i) -} - -func (i *VariableInterpolation) Variables() map[string]InterpolatedVariable { - return map[string]InterpolatedVariable{i.Variable.FullKey(): i.Variable} -} - func NewCountVariable(key string) (*CountVariable, error) { var fieldType CountValueType parts := strings.SplitN(key, ".", 2) diff --git a/config/interpolate_test.go b/config/interpolate_test.go index 923ff660d..46945ee81 100644 --- a/config/interpolate_test.go +++ b/config/interpolate_test.go @@ -122,29 +122,6 @@ func TestNewUserVariable_map(t *testing.T) { } } -func TestLiteralInterpolation_impl(t *testing.T) { - var _ Interpolation = new(LiteralInterpolation) -} - -func TestLiteralInterpolation(t *testing.T) { - i := &LiteralInterpolation{ - Literal: "bar", - } - - if i.Variables() != nil { - t.Fatalf("bad: %#v", i.Variables()) - } - - actual, err := i.Interpolate(nil) - if err != nil { - t.Fatalf("err: %s", err) - } - - if actual != "bar" { - t.Fatalf("bad: %#v", actual) - } -} - func TestResourceVariable_impl(t *testing.T) { var _ InterpolatedVariable = new(ResourceVariable) } @@ -201,50 +178,6 @@ func TestUserVariable_impl(t *testing.T) { var _ InterpolatedVariable = new(UserVariable) } -func TestVariableInterpolation_impl(t *testing.T) { - var _ Interpolation = new(VariableInterpolation) -} - -func TestVariableInterpolation(t *testing.T) { - uv, err := NewUserVariable("var.foo") - if err != nil { - t.Fatalf("err: %s", err) - } - - i := &VariableInterpolation{Variable: uv} - - expected := map[string]InterpolatedVariable{"var.foo": uv} - if !reflect.DeepEqual(i.Variables(), expected) { - t.Fatalf("bad: %#v", i.Variables()) - } - - actual, err := i.Interpolate(map[string]string{ - "var.foo": "bar", - }) - if err != nil { - t.Fatalf("err: %s", err) - } - - if actual != "bar" { - t.Fatalf("bad: %#v", actual) - } -} - -func TestVariableInterpolation_missing(t *testing.T) { - uv, err := NewUserVariable("var.foo") - if err != nil { - t.Fatalf("err: %s", err) - } - - i := &VariableInterpolation{Variable: uv} - _, err = i.Interpolate(map[string]string{ - "var.bar": "bar", - }) - if err == nil { - t.Fatal("should error") - } -} - func TestDetectVariables(t *testing.T) { cases := []struct { Input string From 4af4c9e16c084393e943eda2d75240c94ff05cca Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 13 Jan 2015 12:06:04 -0800 Subject: [PATCH 28/31] config: add lookup function back --- config/interpolate_funcs.go | 33 +++-- config/interpolate_funcs_test.go | 214 +++++++++++++++---------------- config/raw_config.go | 9 +- 3 files changed, 124 insertions(+), 132 deletions(-) diff --git a/config/interpolate_funcs.go b/config/interpolate_funcs.go index c67e3b733..8f19d3b95 100644 --- a/config/interpolate_funcs.go +++ b/config/interpolate_funcs.go @@ -15,9 +15,8 @@ var Funcs map[string]lang.Function func init() { Funcs = map[string]lang.Function{ - "file": interpolationFuncFile(), - "join": interpolationFuncJoin(), - //"lookup": interpolationFuncLookup(), + "file": interpolationFuncFile(), + "join": interpolationFuncJoin(), "element": interpolationFuncElement(), } } @@ -59,22 +58,22 @@ func interpolationFuncJoin() lang.Function { // interpolationFuncLookup implements the "lookup" function that allows // dynamic lookups of map types within a Terraform configuration. -func interpolationFuncLookup( - vs map[string]string, args ...string) (string, error) { - if len(args) != 2 { - return "", fmt.Errorf( - "lookup expects 2 arguments, got %d", len(args)) - } +func interpolationFuncLookup(vs map[string]string) lang.Function { + return lang.Function{ + ArgTypes: []ast.Type{ast.TypeString, ast.TypeString}, + ReturnType: ast.TypeString, + Callback: func(args []interface{}) (interface{}, error) { + k := fmt.Sprintf("var.%s.%s", args[0].(string), args[1].(string)) + v, ok := vs[k] + if !ok { + return "", fmt.Errorf( + "lookup in '%s' failed to find '%s'", + args[0].(string), args[1].(string)) + } - k := fmt.Sprintf("var.%s", strings.Join(args, ".")) - v, ok := vs[k] - if !ok { - return "", fmt.Errorf( - "lookup in '%s' failed to find '%s'", - args[0], args[1]) + return v, nil + }, } - - return v, nil } // interpolationFuncElement implements the "element" function that allows diff --git a/config/interpolate_funcs_test.go b/config/interpolate_funcs_test.go index 3a9a86fe1..9ea2ac1cb 100644 --- a/config/interpolate_funcs_test.go +++ b/config/interpolate_funcs_test.go @@ -20,163 +20,149 @@ func TestInterpolateFuncFile(t *testing.T) { tf.Close() defer os.Remove(path) - testFunction(t, []testFunctionCase{ - { - fmt.Sprintf(`${file("%s")}`, path), - "foo", - false, - }, + testFunction(t, testFunctionConfig{ + Cases: []testFunctionCase{ + { + fmt.Sprintf(`${file("%s")}`, path), + "foo", + false, + }, - // Invalid path - { - `${file("/i/dont/exist")}`, - nil, - true, - }, + // Invalid path + { + `${file("/i/dont/exist")}`, + nil, + true, + }, - // Too many args - { - `${file("foo", "bar")}`, - nil, - true, + // Too many args + { + `${file("foo", "bar")}`, + nil, + true, + }, }, }) } func TestInterpolateFuncJoin(t *testing.T) { - testFunction(t, []testFunctionCase{ - { - `${join(",")}`, - nil, - true, - }, - - { - `${join(",", "foo")}`, - "foo", - false, - }, - - /* - TODO + testFunction(t, testFunctionConfig{ + Cases: []testFunctionCase{ { - `${join(",", "foo", "bar")}`, - "foo,bar", + `${join(",")}`, + nil, + true, + }, + + { + `${join(",", "foo")}`, + "foo", false, }, - */ - { - fmt.Sprintf(`${join(".", "%s")}`, - fmt.Sprintf( - "foo%sbar%sbaz", - InterpSplitDelim, - InterpSplitDelim)), - "foo.bar.baz", - false, + /* + TODO + { + `${join(",", "foo", "bar")}`, + "foo,bar", + false, + }, + */ + + { + fmt.Sprintf(`${join(".", "%s")}`, + fmt.Sprintf( + "foo%sbar%sbaz", + InterpSplitDelim, + InterpSplitDelim)), + "foo.bar.baz", + false, + }, }, }) } -/* func TestInterpolateFuncLookup(t *testing.T) { - testFunction(t, []testFunctionCase{ - cases := []struct { - M map[string]string - Args []string - Result string - Error bool - }{ - { - map[string]string{ - "var.foo.bar": "baz", + testFunction(t, testFunctionConfig{ + Vars: map[string]string{"var.foo.bar": "baz"}, + Cases: []testFunctionCase{ + { + `${lookup("foo", "bar")}`, + "baz", + false, }, - []string{"foo", "bar"}, - "baz", - false, - }, - // Invalid key - { - map[string]string{ - "var.foo.bar": "baz", + // Invalid key + { + `${lookup("foo", "baz")}`, + nil, + true, }, - []string{"foo", "baz"}, - "", - true, - }, - // Too many args - { - map[string]string{ - "var.foo.bar": "baz", + // Too many args + { + `${lookup("foo", "bar", "baz")}`, + nil, + true, }, - []string{"foo", "bar", "baz"}, - "", - true, }, - } - - for i, tc := range cases { - actual, err := interpolationFuncLookup(tc.M, tc.Args...) - if (err != nil) != tc.Error { - t.Fatalf("%d: err: %s", i, err) - } - - if actual != tc.Result { - t.Fatalf("%d: bad: %#v", i, actual) - } - } + }) } -*/ func TestInterpolateFuncElement(t *testing.T) { - testFunction(t, []testFunctionCase{ - { - fmt.Sprintf(`${element("%s", "1")}`, - "foo"+InterpSplitDelim+"baz"), - "baz", - false, - }, + testFunction(t, testFunctionConfig{ + Cases: []testFunctionCase{ + { + fmt.Sprintf(`${element("%s", "1")}`, + "foo"+InterpSplitDelim+"baz"), + "baz", + false, + }, - { - `${element("foo", "0")}`, - "foo", - false, - }, + { + `${element("foo", "0")}`, + "foo", + false, + }, - // Invalid index should wrap vs. out-of-bounds - { - fmt.Sprintf(`${element("%s", "2")}`, - "foo"+InterpSplitDelim+"baz"), - "foo", - false, - }, + // Invalid index should wrap vs. out-of-bounds + { + fmt.Sprintf(`${element("%s", "2")}`, + "foo"+InterpSplitDelim+"baz"), + "foo", + false, + }, - // Too many args - { - fmt.Sprintf(`${element("%s", "0", "2")}`, - "foo"+InterpSplitDelim+"baz"), - nil, - true, + // Too many args + { + fmt.Sprintf(`${element("%s", "0", "2")}`, + "foo"+InterpSplitDelim+"baz"), + nil, + true, + }, }, }) } +type testFunctionConfig struct { + Cases []testFunctionCase + Vars map[string]string +} + type testFunctionCase struct { Input string Result interface{} Error bool } -func testFunction(t *testing.T, cases []testFunctionCase) { - for i, tc := range cases { +func testFunction(t *testing.T, config testFunctionConfig) { + for i, tc := range config.Cases { ast, err := lang.Parse(tc.Input) if err != nil { t.Fatalf("%d: err: %s", i, err) } - engine := langEngine(nil) + engine := langEngine(config.Vars) out, _, err := engine.Execute(ast) if (err != nil) != tc.Error { t.Fatalf("%d: err: %s", i, err) diff --git a/config/raw_config.go b/config/raw_config.go index 37867c638..0e1939184 100644 --- a/config/raw_config.go +++ b/config/raw_config.go @@ -208,10 +208,17 @@ func langEngine(vs map[string]string) *lang.Engine { for k, v := range vs { varMap[k] = lang.Variable{Value: v, Type: ast.TypeString} } + + funcMap := make(map[string]lang.Function) + for k, v := range Funcs { + funcMap[k] = v + } + funcMap["lookup"] = interpolationFuncLookup(vs) + return &lang.Engine{ GlobalScope: &lang.Scope{ VarMap: varMap, - FuncMap: Funcs, + FuncMap: funcMap, }, } } From 8d51b6b1d4982d5a9d4f3a5ba6b6f73b2b2fa93b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 13 Jan 2015 12:40:47 -0800 Subject: [PATCH 29/31] config/lang: variadic functions --- config/lang/ast/ast.go | 2 +- config/lang/check_identifier.go | 8 ++++- config/lang/check_identifier_test.go | 52 +++++++++++++++++++++++++++ config/lang/check_types.go | 14 ++++++++ config/lang/check_types_test.go | 54 ++++++++++++++++++++++++++++ config/lang/engine.go | 18 +++++++++- config/lang/engine_test.go | 23 ++++++++++++ 7 files changed, 168 insertions(+), 3 deletions(-) diff --git a/config/lang/ast/ast.go b/config/lang/ast/ast.go index 0d7000da6..31951621f 100644 --- a/config/lang/ast/ast.go +++ b/config/lang/ast/ast.go @@ -33,7 +33,7 @@ type Visitor func(Node) //go:generate stringer -type=Type // Type is the type of a literal. -type Type uint +type Type uint32 const ( TypeInvalid Type = 0 diff --git a/config/lang/check_identifier.go b/config/lang/check_identifier.go index 2cc49bad1..2e467c098 100644 --- a/config/lang/check_identifier.go +++ b/config/lang/check_identifier.go @@ -52,8 +52,14 @@ func (c *IdentifierCheck) visitCall(n *ast.Call) { return } + // Break up the args into what is variadic and what is required + args := n.Args + if function.Variadic && len(args) > len(function.ArgTypes) { + args = n.Args[:len(function.ArgTypes)] + } + // Verify the number of arguments - if len(n.Args) != len(function.ArgTypes) { + if len(args) != len(function.ArgTypes) { c.createErr(n, fmt.Sprintf( "%s: expected %d arguments, got %d", n.Func, len(function.ArgTypes), len(n.Args))) diff --git a/config/lang/check_identifier_test.go b/config/lang/check_identifier_test.go index 52460aa52..526128424 100644 --- a/config/lang/check_identifier_test.go +++ b/config/lang/check_identifier_test.go @@ -72,6 +72,58 @@ func TestIdentifierCheck(t *testing.T) { }, true, }, + + { + "foo ${rand()} ", + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeInt, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, + + { + "foo ${rand(42)} ", + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeInt, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, + + { + "foo ${rand(\"foo\", 42)} ", + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ArgTypes: []ast.Type{ast.TypeString}, + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeInt, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, } for _, tc := range cases { diff --git a/config/lang/check_types.go b/config/lang/check_types.go index 347a30614..5015781e8 100644 --- a/config/lang/check_types.go +++ b/config/lang/check_types.go @@ -68,6 +68,20 @@ func (v *TypeCheck) visitCall(n *ast.Call) { } } + // If we're variadic, then verify the types there + if function.Variadic { + args = args[len(function.ArgTypes):] + for i, t := range args { + if t != function.VariadicType { + v.createErr(n, fmt.Sprintf( + "%s: argument %d should be %s, got %s", + n.Func, i+len(function.ArgTypes), + function.VariadicType, t)) + return + } + } + } + // Return type v.stackPush(function.ReturnType) } diff --git a/config/lang/check_types_test.go b/config/lang/check_types_test.go index 0c0747d65..f39ff8743 100644 --- a/config/lang/check_types_test.go +++ b/config/lang/check_types_test.go @@ -78,6 +78,60 @@ func TestTypeCheck(t *testing.T) { true, }, + { + `foo ${rand()}`, + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ArgTypes: nil, + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, + + { + `foo ${rand("42")}`, + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ArgTypes: nil, + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + false, + }, + + { + `foo ${rand("42", 42)}`, + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ArgTypes: nil, + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeString, + Callback: func([]interface{}) (interface{}, error) { + return "42", nil + }, + }, + }, + }, + true, + }, + { "foo ${bar}", &Scope{ diff --git a/config/lang/engine.go b/config/lang/engine.go index 5b4e6a2ea..594048ed9 100644 --- a/config/lang/engine.go +++ b/config/lang/engine.go @@ -215,9 +215,25 @@ type Variable struct { // The type checker will validate that the proper types will be called // to the callback. type Function struct { + // ArgTypes is the list of types in argument order. These are the + // required arguments. + // + // ReturnType is the type of the returned value. The Callback MUST + // return this type. ArgTypes []ast.Type ReturnType ast.Type - Callback func([]interface{}) (interface{}, error) + + // Variadic, if true, says that this function is variadic, meaning + // it takes a variable number of arguments. In this case, the + // VariadicType must be set. + Variadic bool + VariadicType ast.Type + + // Callback is the function called for a function. The argument + // types are guaranteed to match the spec above by the type checker. + // The length of the args is strictly == len(ArgTypes) unless Varidiac + // is true, in which case its >= len(ArgTypes). + Callback func([]interface{}) (interface{}, error) } // LookupFunc will look up a variable by name. diff --git a/config/lang/engine_test.go b/config/lang/engine_test.go index b65d0475a..e5a840cbc 100644 --- a/config/lang/engine_test.go +++ b/config/lang/engine_test.go @@ -54,6 +54,29 @@ func TestEngineExecute(t *testing.T) { "foo 42", ast.TypeString, }, + + { + `foo ${rand("foo", "bar")}`, + &Scope{ + FuncMap: map[string]Function{ + "rand": Function{ + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeString, + Callback: func(args []interface{}) (interface{}, error) { + var result string + for _, a := range args { + result += a.(string) + } + return result, nil + }, + }, + }, + }, + false, + "foo foobar", + ast.TypeString, + }, } for _, tc := range cases { From 92af4801a1e7bd3051a8de58bfea7a56b1c54a2d Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 13 Jan 2015 12:47:54 -0800 Subject: [PATCH 30/31] config: reintroduce concat --- config/interpolate_funcs.go | 23 +++++++++++++++++++++++ config/interpolate_funcs_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/config/interpolate_funcs.go b/config/interpolate_funcs.go index 8f19d3b95..65cebbfa1 100644 --- a/config/interpolate_funcs.go +++ b/config/interpolate_funcs.go @@ -1,6 +1,7 @@ package config import ( + "bytes" "fmt" "io/ioutil" "strconv" @@ -15,12 +16,34 @@ var Funcs map[string]lang.Function func init() { Funcs = map[string]lang.Function{ + "concat": interpolationFuncConcat(), "file": interpolationFuncFile(), "join": interpolationFuncJoin(), "element": interpolationFuncElement(), } } +// interpolationFuncConcat implements the "concat" function that +// concatenates multiple strings. This isn't actually necessary anymore +// since our language supports string concat natively, but for backwards +// compat we do this. +func interpolationFuncConcat() lang.Function { + return lang.Function{ + ArgTypes: []ast.Type{ast.TypeString}, + ReturnType: ast.TypeString, + Variadic: true, + VariadicType: ast.TypeString, + Callback: func(args []interface{}) (interface{}, error) { + var b bytes.Buffer + for _, v := range args { + b.WriteString(v.(string)) + } + + return b.String(), nil + }, + } +} + // interpolationFuncFile implements the "file" function that allows // loading contents from a file. func interpolationFuncFile() lang.Function { diff --git a/config/interpolate_funcs_test.go b/config/interpolate_funcs_test.go index 9ea2ac1cb..757f8aa5b 100644 --- a/config/interpolate_funcs_test.go +++ b/config/interpolate_funcs_test.go @@ -10,6 +10,30 @@ import ( "github.com/hashicorp/terraform/config/lang" ) +func TestInterpolateFuncConcat(t *testing.T) { + testFunction(t, testFunctionConfig{ + Cases: []testFunctionCase{ + { + `${concat("foo", "bar")}`, + "foobar", + false, + }, + + { + `${concat("foo")}`, + "foo", + false, + }, + + { + `${concat()}`, + nil, + true, + }, + }, + }) +} + func TestInterpolateFuncFile(t *testing.T) { tf, err := ioutil.TempFile("", "tf") if err != nil { From dd456871e9321bea00a18e967e2f8ad5c1db973a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 14 Jan 2015 10:11:29 -0800 Subject: [PATCH 31/31] config/lang: remove unused code --- config/lang/lex.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/config/lang/lex.go b/config/lang/lex.go index 7f2a85ae7..ed1f8fb64 100644 --- a/config/lang/lex.go +++ b/config/lang/lex.go @@ -53,10 +53,13 @@ const ( // The parser calls this method to get each new token. func (x *parserLex) Lex(yylval *parserSymType) int { + // We always start in literal mode, since programs don't start + // in an interpolation. ex. "foo ${bar}" vs "bar" (and assuming interp.) if x.mode == parserModeInvalid { x.mode = parserModeLiteral } + // Defer an update to set the proper column/line we read the next token. defer func() { if yylval.token != nil && yylval.token.Pos.Column == 0 { yylval.token.Pos = *x.astPos @@ -127,8 +130,8 @@ func (x *parserLex) lexModeInterpolation(yylval *parserSymType) int { continue } - // If we see a double quote and we're in an interpolation, then - // we are lexing a string. + // If we see a double quote then we're lexing a string since + // we're in interpolation mode. if c == '"' { result, terminated := x.lexString(yylval, true) if !terminated { @@ -154,6 +157,8 @@ func (x *parserLex) lexModeInterpolation(yylval *parserSymType) int { switch c { case '}': + // '}' means we ended the interpolation. Pop back into + // literal mode and reduce our interpolation depth. x.interpolationDepth-- x.mode = parserModeLiteral return PROGRAM_BRACKET_RIGHT @@ -291,12 +296,6 @@ func (x *parserLex) lexString(yylval *parserSymType, quoted bool) (int, bool) { } } - // If we hit a '}' and we're in a program, then end it. - if c == '}' && x.interpolationDepth > 0 { - x.backup() - break - } - // If we hit a dollar sign, then check if we're starting // another interpolation. If so, then we're done. if c == '$' {