resilien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #32 from hashicorp/f-parser 

Parser
This commit is contained in:
Mitchell Hashimoto 2014-07-22 16:28:41 -07:00
parent dde83f7cdd 1e2de8f91c
commit c6cf449bc4
10 changed files with 454 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -4,4 +4,6 @@ example.tf
terraform.tfplan
terraform.tfstate
bin/
config/y.go
config/y.output
vendor/

12
Makefile
View File

@ -16,27 +16,31 @@ export CGO_CFLAGS CGO_LDFLAGS PATH
default: test
dev: libucl
dev: config/y.go libucl
@sh -c "$(CURDIR)/scripts/build.sh"
libucl: vendor/libucl/$(LIBUCL_NAME)
test: libucl
test: config/y.go libucl
TF_ACC= go test $(TEST) $(TESTARGS) -timeout=10s
testacc: libucl
testacc: config/y.go libucl
@if [ "$(TEST)" = "./..." ]; then \
echo "ERROR: Set TEST to a specific package"; \
exit 1; \
fi
TF_ACC=1 go test $(TEST) -v $(TESTARGS)
testrace: libucl
testrace: config/y.go libucl
TF_ACC= go test -race $(TEST) $(TESTARGS)
updatedeps:
go get -u -v ./...
config/y.go: config/expr.y
cd config/ && \
go tool yacc -p "expr" expr.y
vendor/libucl/libucl.a: vendor/libucl
cd vendor/libucl && \
cmake cmake/ && \

91
config/expr.y Normal file
View File

@ -0,0 +1,91 @@
// This is the yacc input for creating the parser for interpolation
// expressions in Go.
// To build it:
//
// go tool yacc -p "expr" expr.y (produces y.go)
//
%{
package config
import (
"fmt"
)
%}
%union {
expr Interpolation
str string
variable InterpolatedVariable
args []Interpolation
}
%type <args> args
%type <expr> expr
%type <str> string
%type <variable> variable
%token <str> STRING IDENTIFIER
%token <str> COMMA LEFTPAREN RIGHTPAREN
%%
top:
expr
{
exprResult = $1
}
expr:
string
{
$$ = &LiteralInterpolation{Literal: $1}
}
| variable
{
$$ = &VariableInterpolation{Variable: $1}
}
| IDENTIFIER LEFTPAREN args RIGHTPAREN
{
f, ok := Funcs[$1]
if !ok {
exprErrors = append(exprErrors, fmt.Errorf(
"Unknown function: %s", $1))
}
$$ = &FunctionInterpolation{Func: f, Args: $3}
}
args:
{
$$ = nil
}
| expr COMMA expr
{
$$ = append($$, $1, $3)
}
| expr
{
$$ = append($$, $1)
}
string:
STRING
{
$$ = $1
}
variable:
IDENTIFIER
{
var err error
$$, err = NewInterpolatedVariable($1)
if err != nil {
exprErrors = append(exprErrors, fmt.Errorf(
"Error parsing variable '%s': %s", $1, err))
}
}
%%

131
config/expr_lex.go Normal file
View File

@ -0,0 +1,131 @@
package config
import (
"bytes"
"log"
"unicode"
"unicode/utf8"
)
// The parser expects the lexer to return 0 on EOF.
const lexEOF = 0
// The parser uses the type <prefix>Lex as a lexer. It must provide
// the methods Lex(*<prefix>SymType) int and Error(string).
type exprLex struct {
input string
pos int
width int
}
// The parser calls this method to get each new token.
func (x *exprLex) Lex(yylval *exprSymType) int {
for {
c := x.next()
if c == lexEOF {
return lexEOF
}
// Ignore all whitespace
if unicode.IsSpace(c) {
continue
}
switch c {
case '"':
return x.lexString(yylval)
case ',':
return COMMA
case '(':
return LEFTPAREN
case ')':
return RIGHTPAREN
default:
x.backup()
return x.lexId(yylval)
}
}
}
func (x *exprLex) lexId(yylval *exprSymType) int {
var b bytes.Buffer
for {
c := x.next()
if c == lexEOF {
break
}
// If this isn't a character we want in an ID, return out.
// One day we should make this a regexp.
if c != '_' &&
c != '-' &&
c != '.' &&
c != '*' &&
!unicode.IsLetter(c) &&
!unicode.IsNumber(c) {
x.backup()
break
}
if _, err := b.WriteRune(c); err != nil {
log.Printf("ERR: %s", err)
return lexEOF
}
}
yylval.str = b.String()
return IDENTIFIER
}
func (x *exprLex) lexString(yylval *exprSymType) int {
var b bytes.Buffer
for {
c := x.next()
if c == lexEOF {
break
}
// String end
if c == '"' {
break
}
if _, err := b.WriteRune(c); err != nil {
log.Printf("ERR: %s", err)
return lexEOF
}
}
yylval.str = b.String()
return STRING
}
// Return the next rune for the lexer.
func (x *exprLex) next() rune {
if int(x.pos) >= len(x.input) {
x.width = 0
return lexEOF
}
r, w := utf8.DecodeRuneInString(x.input[x.pos:])
x.width = w
x.pos += x.width
return r
}
// peek returns but does not consume the next rune in the input
func (x *exprLex) peek() rune {
r := x.next()
x.backup()
return r
}
// backup steps back one rune. Can only be called once per next.
func (x *exprLex) backup() {
x.pos -= x.width
}
// The parser calls this method on a parse error.
func (x *exprLex) Error(s string) {
log.Printf("parse error: %s", s)
}

34
config/expr_parse.go Normal file
View File

@ -0,0 +1,34 @@
package config
import (
"sync"
"github.com/hashicorp/terraform/helper/multierror"
)
// exprErrors are the errors built up from parsing. These should not
// be accessed directly.
var exprErrors []error
var exprLock sync.Mutex
var exprResult Interpolation
// ExprParse parses the given expression and returns an executable
// Interpolation.
func ExprParse(v string) (Interpolation, error) {
exprLock.Lock()
defer exprLock.Unlock()
exprErrors = nil
exprResult = nil
// Parse
exprParse(&exprLex{input: v})
// Build up the errors
var err error
if len(exprErrors) > 0 {
err = &multierror.Error{Errors: exprErrors}
exprResult = nil
}
return exprResult, err
}

123
config/expr_parse_test.go Normal file
View File

@ -0,0 +1,123 @@
package config
import (
"reflect"
"testing"
)
func TestExprParse(t *testing.T) {
cases := []struct {
Input string
Result Interpolation
Error bool
}{
{
"foo",
nil,
true,
},
{
`"foo"`,
&LiteralInterpolation{Literal: "foo"},
false,
},
{
"var.foo",
&VariableInterpolation{
Variable: &UserVariable{
Name: "foo",
key: "var.foo",
},
},
false,
},
{
"lookup(var.foo, var.bar)",
&FunctionInterpolation{
Func: nil, // Funcs["lookup"]
Args: []Interpolation{
&VariableInterpolation{
Variable: &UserVariable{
Name: "foo",
key: "var.foo",
},
},
&VariableInterpolation{
Variable: &UserVariable{
Name: "bar",
key: "var.bar",
},
},
},
},
false,
},
{
"lookup(var.foo, lookup(var.baz, var.bar))",
&FunctionInterpolation{
Func: nil, // Funcs["lookup"]
Args: []Interpolation{
&VariableInterpolation{
Variable: &UserVariable{
Name: "foo",
key: "var.foo",
},
},
&FunctionInterpolation{
Func: nil, // Funcs["lookup"]
Args: []Interpolation{
&VariableInterpolation{
Variable: &UserVariable{
Name: "baz",
key: "var.baz",
},
},
&VariableInterpolation{
Variable: &UserVariable{
Name: "bar",
key: "var.bar",
},
},
},
},
},
},
false,
},
}
for i, tc := range cases {
actual, err := ExprParse(tc.Input)
if (err != nil) != tc.Error {
t.Fatalf("%d. Error: %s", i, err)
}
// This is jank, but reflect.DeepEqual never has functions
// being the same.
f, ok := actual.(*FunctionInterpolation)
if ok {
fs := make([]*FunctionInterpolation, 1)
fs[0] = f
for len(fs) > 0 {
f := fs[0]
fs = fs[1:]
f.Func = nil
for _, a := range f.Args {
f, ok := a.(*FunctionInterpolation)
if ok {
fs = append(fs, f)
}
}
}
}
if !reflect.DeepEqual(actual, tc.Result) {
t.Fatalf("%d bad: %#v", i, actual)
}
}
}

110
config/interpolate.go
View File

@ -17,7 +17,6 @@ var funcRegexp *regexp.Regexp = regexp.MustCompile(
// Interpolations might be simple variable references, or it might be
// function calls, or even nested function calls.
type Interpolation interface {
FullString() string
Interpolate(map[string]string) (string, error)
Variables() map[string]InterpolatedVariable
}
@ -38,17 +37,19 @@ type InterpolatedVariable interface {
// with some variable number of arguments to generate a value.
type FunctionInterpolation struct {
Func InterpolationFunc
Args []InterpolatedVariable
Args []Interpolation
}
key string
// LiteralInterpolation implements Interpolation for literals. Ex:
// ${"foo"} will equal "foo".
type LiteralInterpolation struct {
Literal string
}
// VariableInterpolation implements Interpolation for simple variable
// interpolation. Ex: "${var.foo}" or "${aws_instance.foo.bar}"
type VariableInterpolation struct {
Variable InterpolatedVariable
key string
}
// A ResourceVariable is a variable that is referencing the field
@ -74,61 +75,6 @@ type UserVariable struct {
key string
}
// NewInterpolation takes some string and returns the valid
// Interpolation associated with it, or error if a valid
// interpolation could not be found or the interpolation itself
// is invalid.
func NewInterpolation(v string) (Interpolation, error) {
match := funcRegexp.FindStringSubmatch(v)
if match != nil {
fn, ok := Funcs[match[1]]
if !ok {
return nil, fmt.Errorf(
"%s: Unknown function '%s'",
v, match[1])
}
args := make([]InterpolatedVariable, 0, len(match)-2)
for i := 2; i < len(match); i++ {
// This can be empty if we have a single argument
// due to the format of the regexp.
if match[i] == "" {
continue
}
v, err := NewInterpolatedVariable(match[i])
if err != nil {
return nil, err
}
args = append(args, v)
}
return &FunctionInterpolation{
Func: fn,
Args: args,
key: v,
}, nil
}
if idx := strings.Index(v, "."); idx >= 0 {
v, err := NewInterpolatedVariable(v)
if err != nil {
return nil, err
}
return &VariableInterpolation{
Variable: v,
key: v.FullKey(),
}, nil
}
return nil, fmt.Errorf(
"Interpolation '%s' is not a valid interpolation. " +
"Please check your syntax and try again.")
}
func NewInterpolatedVariable(v string) (InterpolatedVariable, error) {
if !strings.HasPrefix(v, "var.") {
return NewResourceVariable(v)
@ -137,21 +83,13 @@ func NewInterpolatedVariable(v string) (InterpolatedVariable, error) {
return NewUserVariable(v)
}
func (i *FunctionInterpolation) FullString() string {
return i.key
}
func (i *FunctionInterpolation) Interpolate(
vs map[string]string) (string, error) {
args := make([]string, len(i.Args))
for idx, a := range i.Args {
k := a.FullKey()
v, ok := vs[k]
if !ok {
return "", fmt.Errorf(
"%s: variable argument value unknown: %s",
i.FullString(),
k)
v, err := a.Interpolate(vs)
if err != nil {
return "", err
}
args[idx] = v
@ -160,38 +98,48 @@ func (i *FunctionInterpolation) Interpolate(
return i.Func(vs, args...)
}
func (i *FunctionInterpolation) GoString() string {
return fmt.Sprintf("*%#v", *i)
}
func (i *FunctionInterpolation) Variables() map[string]InterpolatedVariable {
result := make(map[string]InterpolatedVariable)
for _, a := range i.Args {
k := a.FullKey()
if _, ok := result[k]; ok {
continue
for k, v := range a.Variables() {
result[k] = v
}
result[k] = a
}
return result
}
func (i *VariableInterpolation) FullString() string {
return i.key
func (i *LiteralInterpolation) Interpolate(
map[string]string) (string, error) {
return i.Literal, nil
}
func (i *LiteralInterpolation) Variables() map[string]InterpolatedVariable {
return nil
}
func (i *VariableInterpolation) Interpolate(
vs map[string]string) (string, error) {
v, ok := vs[i.key]
v, ok := vs[i.Variable.FullKey()]
if !ok {
return "", fmt.Errorf(
"%s: value for variable not found",
i.key)
i.Variable.FullKey())
}
return v, nil
}
func (i *VariableInterpolation) GoString() string {
return fmt.Sprintf("*%#v", *i)
}
func (i *VariableInterpolation) Variables() map[string]InterpolatedVariable {
return map[string]InterpolatedVariable{i.key: i.Variable}
return map[string]InterpolatedVariable{i.Variable.FullKey(): i.Variable}
}
func NewResourceVariable(key string) (*ResourceVariable, error) {

101
config/interpolate_test.go
View File

@ -6,68 +6,6 @@ import (
"testing"
)
func TestNewInterpolation(t *testing.T) {
cases := []struct {
Input string
Result Interpolation
Error bool
}{
{
"foo",
nil,
true,
},
{
"var.foo",
&VariableInterpolation{
Variable: &UserVariable{
Name: "foo",
key: "var.foo",
},
key: "var.foo",
},
false,
},
{
"lookup(var.foo, var.bar)",
&FunctionInterpolation{
Func: nil, // Funcs["lookup"]
Args: []InterpolatedVariable{
&UserVariable{
Name: "foo",
key: "var.foo",
},
&UserVariable{
Name: "bar",
key: "var.bar",
},
},
key: "lookup(var.foo, var.bar)",
},
false,
},
}
for i, tc := range cases {
actual, err := NewInterpolation(tc.Input)
if (err != nil) != tc.Error {
t.Fatalf("%d. Error: %s", i, err)
}
// This is jank, but reflect.DeepEqual never has functions
// being the same.
if f, ok := actual.(*FunctionInterpolation); ok {
f.Func = nil
}
if !reflect.DeepEqual(actual, tc.Result) {
t.Fatalf("%d bad: %#v", i, actual)
}
}
}
func TestNewInterpolatedVariable(t *testing.T) {
cases := []struct {
Input string
@ -171,11 +109,10 @@ func TestFunctionInterpolation(t *testing.T) {
i := &FunctionInterpolation{
Func: fn,
Args: []InterpolatedVariable{v1, v2},
key: "foo",
}
if i.FullString() != "foo" {
t.Fatalf("err: %#v", i)
Args: []Interpolation{
&VariableInterpolation{Variable: v1},
&VariableInterpolation{Variable: v2},
},
}
expected := map[string]InterpolatedVariable{
@ -199,6 +136,29 @@ func TestFunctionInterpolation(t *testing.T) {
}
}
func TestLiteralInterpolation_impl(t *testing.T) {
var _ Interpolation = new(LiteralInterpolation)
}
func TestLiteralInterpolation(t *testing.T) {
i := &LiteralInterpolation{
Literal: "bar",
}
if i.Variables() != nil {
t.Fatalf("bad: %#v", i.Variables())
}
actual, err := i.Interpolate(nil)
if err != nil {
t.Fatalf("err: %s", err)
}
if actual != "bar" {
t.Fatalf("bad: %#v", actual)
}
}
func TestResourceVariable_impl(t *testing.T) {
var _ InterpolatedVariable = new(ResourceVariable)
}
@ -265,10 +225,7 @@ func TestVariableInterpolation(t *testing.T) {
t.Fatalf("err: %s", err)
}
i := &VariableInterpolation{Variable: uv, key: "var.foo"}
if i.FullString() != "var.foo" {
t.Fatalf("err: %#v", i)
}
i := &VariableInterpolation{Variable: uv}
expected := map[string]InterpolatedVariable{"var.foo": uv}
if !reflect.DeepEqual(i.Variables(), expected) {
@ -293,7 +250,7 @@ func TestVariableInterpolation_missing(t *testing.T) {
t.Fatalf("err: %s", err)
}
i := &VariableInterpolation{Variable: uv, key: "var.foo"}
i := &VariableInterpolation{Variable: uv}
_, err = i.Interpolate(map[string]string{
"var.bar": "bar",
})

2
config/interpolate_walk.go
View File

@ -95,7 +95,7 @@ func (w *interpolationWalker) Primitive(v reflect.Value) error {
// Interpolation found, instantiate it
key := match[2]
i, err := NewInterpolation(key)
i, err := ExprParse(key)
if err != nil {
return err
}

12
config/interpolate_walk_test.go
View File

@ -29,7 +29,6 @@ func TestInterpolationWalker_detect(t *testing.T) {
Name: "foo",
key: "var.foo",
},
key: "var.foo",
},
},
},
@ -41,13 +40,14 @@ func TestInterpolationWalker_detect(t *testing.T) {
Result: []Interpolation{
&FunctionInterpolation{
Func: nil,
Args: []InterpolatedVariable{
&UserVariable{
Name: "foo",
key: "var.foo",
Args: []Interpolation{
&VariableInterpolation{
Variable: &UserVariable{
Name: "foo",
key: "var.foo",
},
},
},
key: "lookup(var.foo)",
},
},
},