configs/configupgrade: do not panic on HEREDOCs. (#20281)

Previously, configupgrade would panic if it encountered a HEREDOC. For
the time being, we will simply print out the HEREDOC as-is.

Unfortunately, we discovered that terraform 0.11's version of HCL
allowed for HEREDOCs with the termination delimiter inline (instead of
on a newline, which is technically correct). Since 0.12configupgrade
needs to be bug-compatible with terraform 0.11, we must roll back to the
same version of HCL used in terraform 0.11.
This commit is contained in:
Kristin Laemmert 2019-02-08 15:51:53 -08:00 committed by GitHub
parent 6eb7bfbdfb
commit 5f8916b4fd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 184 additions and 180 deletions

View File

@ -0,0 +1,10 @@
locals {
cert_options = <<EOF
--cert-file=/etc/ssl/etcd/server.crt \
--peer-trusted-ca-file=/etc/ssl/etcd/ca.crt \
--peer-client-cert-auth=trueEOF
}
output "local" {
value = "${local.cert_options}"
}

View File

@ -0,0 +1,11 @@
locals {
cert_options = <<EOF
--cert-file=/etc/ssl/etcd/server.crt \
--peer-trusted-ca-file=/etc/ssl/etcd/ca.crt \
--peer-client-cert-auth=trueEOF
}
output "local" {
value = local.cert_options
}

View File

@ -0,0 +1,3 @@
terraform {
required_version = ">= 0.12"
}

View File

@ -62,8 +62,13 @@ Value:
diags = diags.Append(interpDiags)
case hcl1token.HEREDOC:
// TODO: Implement
panic("HEREDOC not supported yet")
// TODO: Implement more complex handling to upgrade any
// interpolation sequences inside.
// TODO: If a heredoc has its termination delimeter inline (which is
// a bug that worked in terraform 0.11, so we need to support it
// here), move the delimiter to a new line.
buf.WriteString(tv.Text)
case hcl1token.BOOL:
if litVal.(bool) {

4
go.mod
View File

@ -71,13 +71,13 @@ require (
github.com/hashicorp/go-uuid v1.0.0
github.com/hashicorp/go-version v1.0.0
github.com/hashicorp/golang-lru v0.5.0 // indirect
github.com/hashicorp/hcl v1.0.0
github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f
github.com/hashicorp/hcl2 v0.0.0-20190130225218-89dbc5eb3d9e
github.com/hashicorp/hil v0.0.0-20170627220502-fa9f258a9250
github.com/hashicorp/logutils v0.0.0-20150609070431-0dc08b1671f3
github.com/hashicorp/memberlist v0.1.0 // indirect
github.com/hashicorp/serf v0.0.0-20160124182025-e4ec8cc423bb // indirect
github.com/hashicorp/terraform-config-inspect v0.0.0-20190129165904-67302cb0361b
github.com/hashicorp/terraform-config-inspect v0.0.0-20190208230122-b0707673339c
github.com/hashicorp/vault v0.0.0-20161029210149-9a60bf2a50e4
github.com/jen20/awspolicyequivalence v0.0.0-20170831201602-3d48364a137a // indirect
github.com/jonboulle/clockwork v0.1.0 // indirect

9
go.sum
View File

@ -160,6 +160,12 @@ github.com/hashicorp/go-version v1.0.0 h1:21MVWPKDphxa7ineQQTrCU5brh7OuVVAzGOCnn
github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f h1:UdxlrJz4JOnY8W+DbLISwf2B8WXEolNRA8BGCwI9jws=
github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
github.com/hashicorp/hcl v0.0.0-20180320202055-f40e974e75af h1:g6buE1uPu/jBP1YRkIqjoqBz5CraM5TukA+MmymQQXA=
github.com/hashicorp/hcl v0.0.0-20180320202055-f40e974e75af/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
github.com/hashicorp/hcl v0.0.0-20180403142333-25340db58d11 h1:/WXmgbjMlEHeC4Pbb5h/TfV3cAe6HuKDKdlWz1eoBd4=
github.com/hashicorp/hcl v0.0.0-20180403142333-25340db58d11/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/hcl2 v0.0.0-20181208003705-670926858200/go.mod h1:ShfpTh661oAaxo7VcNxg0zcZW6jvMa7Moy2oFx7e5dE=
@ -175,6 +181,9 @@ github.com/hashicorp/serf v0.0.0-20160124182025-e4ec8cc423bb h1:ZbgmOQt8DOg796fi
github.com/hashicorp/serf v0.0.0-20160124182025-e4ec8cc423bb/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE=
github.com/hashicorp/terraform-config-inspect v0.0.0-20190129165904-67302cb0361b h1:A+sv28TB1pIfYHFBnEDUYIKA12RTVLsYjvu9JS+dxz8=
github.com/hashicorp/terraform-config-inspect v0.0.0-20190129165904-67302cb0361b/go.mod h1:nKgb1xGwu9K9mk77DQJoM/XD19kEyrKFNstmjxB6/48=
github.com/hashicorp/terraform-config-inspect v0.0.0-20190208170851-de963d052d6d h1:6sogMHZHRQKJz4qCUZBvNU9ajPeJ7PK/P6c8UexD+pY=
github.com/hashicorp/terraform-config-inspect v0.0.0-20190208170851-de963d052d6d/go.mod h1:ItvqtvbC3K23FFET62ZwnkwtpbKZm8t8eMcWjmVVjD8=
github.com/hashicorp/terraform-config-inspect v0.0.0-20190208230122-b0707673339c/go.mod h1:ItvqtvbC3K23FFET62ZwnkwtpbKZm8t8eMcWjmVVjD8=
github.com/hashicorp/vault v0.0.0-20161029210149-9a60bf2a50e4 h1:SGDekHLK2IRoVS7Fb4olLyWvc2VmwKgyFC05j6X3NII=
github.com/hashicorp/vault v0.0.0-20161029210149-9a60bf2a50e4/go.mod h1:KfSyffbKxoVyspOdlaGVjIuwLobi07qD1bAbosPMpP0=
github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M=

View File

@ -3,8 +3,7 @@ sudo: false
language: go
go:
- 1.x
- tip
- 1.8
branches:
only:

View File

@ -89,7 +89,7 @@ func (d *decoder) decode(name string, node ast.Node, result reflect.Value) error
switch k.Kind() {
case reflect.Bool:
return d.decodeBool(name, node, result)
case reflect.Float32, reflect.Float64:
case reflect.Float64:
return d.decodeFloat(name, node, result)
case reflect.Int, reflect.Int32, reflect.Int64:
return d.decodeInt(name, node, result)
@ -137,13 +137,13 @@ func (d *decoder) decodeBool(name string, node ast.Node, result reflect.Value) e
func (d *decoder) decodeFloat(name string, node ast.Node, result reflect.Value) error {
switch n := node.(type) {
case *ast.LiteralType:
if n.Token.Type == token.FLOAT || n.Token.Type == token.NUMBER {
if n.Token.Type == token.FLOAT {
v, err := strconv.ParseFloat(n.Token.Text, 64)
if err != nil {
return err
}
result.Set(reflect.ValueOf(v).Convert(result.Type()))
result.Set(reflect.ValueOf(v))
return nil
}
}
@ -573,11 +573,7 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value)
// Compile the list of all the fields that we're going to be decoding
// from all the structs.
type field struct {
field reflect.StructField
val reflect.Value
}
fields := []field{}
fields := make(map[*reflect.StructField]reflect.Value)
for len(structs) > 0 {
structVal := structs[0]
structs = structs[1:]
@ -620,7 +616,7 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value)
}
// Normal struct field, store it away
fields = append(fields, field{fieldType, structVal.Field(i)})
fields[&fieldType] = structVal.Field(i)
}
}
@ -628,27 +624,26 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value)
decodedFields := make([]string, 0, len(fields))
decodedFieldsVal := make([]reflect.Value, 0)
unusedKeysVal := make([]reflect.Value, 0)
for _, f := range fields {
field, fieldValue := f.field, f.val
if !fieldValue.IsValid() {
for fieldType, field := range fields {
if !field.IsValid() {
// This should never happen
panic("field is not valid")
}
// If we can't set the field, then it is unexported or something,
// and we just continue onwards.
if !fieldValue.CanSet() {
if !field.CanSet() {
continue
}
fieldName := field.Name
fieldName := fieldType.Name
tagValue := field.Tag.Get(tagName)
tagValue := fieldType.Tag.Get(tagName)
tagParts := strings.SplitN(tagValue, ",", 2)
if len(tagParts) >= 2 {
switch tagParts[1] {
case "decodedFields":
decodedFieldsVal = append(decodedFieldsVal, fieldValue)
decodedFieldsVal = append(decodedFieldsVal, field)
continue
case "key":
if item == nil {
@ -659,10 +654,10 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value)
}
}
fieldValue.SetString(item.Keys[0].Token.Value().(string))
field.SetString(item.Keys[0].Token.Value().(string))
continue
case "unusedKeys":
unusedKeysVal = append(unusedKeysVal, fieldValue)
unusedKeysVal = append(unusedKeysVal, field)
continue
}
}
@ -689,7 +684,7 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value)
// because we actually want the value.
fieldName = fmt.Sprintf("%s.%s", name, fieldName)
if len(prefixMatches.Items) > 0 {
if err := d.decode(fieldName, prefixMatches, fieldValue); err != nil {
if err := d.decode(fieldName, prefixMatches, field); err != nil {
return err
}
}
@ -699,12 +694,12 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value)
decodeNode = &ast.ObjectList{Items: ot.List.Items}
}
if err := d.decode(fieldName, decodeNode, fieldValue); err != nil {
if err := d.decode(fieldName, decodeNode, field); err != nil {
return err
}
}
decodedFields = append(decodedFields, field.Name)
decodedFields = append(decodedFields, fieldType.Name)
}
if len(decodedFieldsVal) > 0 {

View File

@ -1,3 +0,0 @@
module github.com/hashicorp/hcl
require github.com/davecgh/go-spew v1.1.1

View File

@ -1,2 +0,0 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

View File

@ -197,18 +197,9 @@ func (p *Parser) objectItem() (*ast.ObjectItem, error) {
keyStr = append(keyStr, k.Token.Text)
}
return nil, &PosError{
Pos: p.tok.Pos,
Err: fmt.Errorf(
"key '%s' expected start of object ('{') or assignment ('=')",
strings.Join(keyStr, " ")),
}
}
// key=#comment
// val
if p.lineComment != nil {
o.LineComment, p.lineComment = p.lineComment, nil
return nil, fmt.Errorf(
"key '%s' expected start of object ('{') or assignment ('=')",
strings.Join(keyStr, " "))
}
// do a look-ahead for line comment
@ -328,10 +319,7 @@ func (p *Parser) objectType() (*ast.ObjectType, error) {
// No error, scan and expect the ending to be a brace
if tok := p.scan(); tok.Type != token.RBRACE {
return nil, &PosError{
Pos: tok.Pos,
Err: fmt.Errorf("object expected closing RBRACE got: %s", tok.Type),
}
return nil, fmt.Errorf("object expected closing RBRACE got: %s", tok.Type)
}
o.List = l

View File

@ -252,14 +252,6 @@ func (p *printer) objectItem(o *ast.ObjectItem) []byte {
}
}
// If key and val are on different lines, treat line comments like lead comments.
if o.LineComment != nil && o.Val.Pos().Line != o.Keys[0].Pos().Line {
for _, comment := range o.LineComment.List {
buf.WriteString(comment.Text)
buf.WriteByte(newline)
}
}
for i, k := range o.Keys {
buf.WriteString(k.Token.Text)
buf.WriteByte(blank)
@ -273,7 +265,7 @@ func (p *printer) objectItem(o *ast.ObjectItem) []byte {
buf.Write(p.output(o.Val))
if o.LineComment != nil && o.Val.Pos().Line == o.Keys[0].Pos().Line {
if o.Val.Pos().Line == o.Keys[0].Pos().Line && o.LineComment != nil {
buf.WriteByte(blank)
for _, comment := range o.LineComment.List {
buf.WriteString(comment.Text)
@ -517,13 +509,8 @@ func (p *printer) alignedItems(items []*ast.ObjectItem) []byte {
// list returns the printable HCL form of an list type.
func (p *printer) list(l *ast.ListType) []byte {
if p.isSingleLineList(l) {
return p.singleLineList(l)
}
var buf bytes.Buffer
buf.WriteString("[")
buf.WriteByte(newline)
var longestLine int
for _, item := range l.List {
@ -536,112 +523,115 @@ func (p *printer) list(l *ast.ListType) []byte {
}
}
haveEmptyLine := false
insertSpaceBeforeItem := false
lastHadLeadComment := false
for i, item := range l.List {
// If we have a lead comment, then we want to write that first
leadComment := false
if lit, ok := item.(*ast.LiteralType); ok && lit.LeadComment != nil {
leadComment = true
// Ensure an empty line before every element with a
// lead comment (except the first item in a list).
if !haveEmptyLine && i != 0 {
buf.WriteByte(newline)
}
for _, comment := range lit.LeadComment.List {
buf.Write(p.indent([]byte(comment.Text)))
buf.WriteByte(newline)
}
}
// also indent each line
val := p.output(item)
curLen := len(val)
buf.Write(p.indent(val))
// if this item is a heredoc, then we output the comma on
// the next line. This is the only case this happens.
comma := []byte{','}
// Keep track of whether this item is a heredoc since that has
// unique behavior.
heredoc := false
if lit, ok := item.(*ast.LiteralType); ok && lit.Token.Type == token.HEREDOC {
buf.WriteByte(newline)
comma = p.indent(comma)
heredoc = true
}
buf.Write(comma)
if lit, ok := item.(*ast.LiteralType); ok && lit.LineComment != nil {
// if the next item doesn't have any comments, do not align
buf.WriteByte(blank) // align one space
for i := 0; i < longestLine-curLen; i++ {
buf.WriteByte(blank)
}
for _, comment := range lit.LineComment.List {
buf.WriteString(comment.Text)
}
}
buf.WriteByte(newline)
// Ensure an empty line after every element with a
// lead comment (except the first item in a list).
haveEmptyLine = leadComment && i != len(l.List)-1
if haveEmptyLine {
buf.WriteByte(newline)
}
}
buf.WriteString("]")
return buf.Bytes()
}
// isSingleLineList returns true if:
// * they were previously formatted entirely on one line
// * they consist entirely of literals
// * there are either no heredoc strings or the list has exactly one element
// * there are no line comments
func (printer) isSingleLineList(l *ast.ListType) bool {
for _, item := range l.List {
if item.Pos().Line != l.Lbrack.Line {
return false
}
lit, ok := item.(*ast.LiteralType)
if !ok {
return false
}
if lit.Token.Type == token.HEREDOC && len(l.List) != 1 {
return false
}
if lit.LineComment != nil {
return false
}
}
return true
}
// singleLineList prints a simple single line list.
// For a definition of "simple", see isSingleLineList above.
func (p *printer) singleLineList(l *ast.ListType) []byte {
buf := &bytes.Buffer{}
buf.WriteString("[")
for i, item := range l.List {
if i != 0 {
buf.WriteString(", ")
}
// Output the item itself
buf.Write(p.output(item))
// The heredoc marker needs to be at the end of line.
if lit, ok := item.(*ast.LiteralType); ok && lit.Token.Type == token.HEREDOC {
// multiline list, add newline before we add each item
buf.WriteByte(newline)
insertSpaceBeforeItem = false
// If we have a lead comment, then we want to write that first
leadComment := false
if lit, ok := item.(*ast.LiteralType); ok && lit.LeadComment != nil {
leadComment = true
// If this isn't the first item and the previous element
// didn't have a lead comment, then we need to add an extra
// newline to properly space things out. If it did have a
// lead comment previously then this would be done
// automatically.
if i > 0 && !lastHadLeadComment {
buf.WriteByte(newline)
}
for _, comment := range lit.LeadComment.List {
buf.Write(p.indent([]byte(comment.Text)))
buf.WriteByte(newline)
}
}
// also indent each line
val := p.output(item)
curLen := len(val)
buf.Write(p.indent(val))
// if this item is a heredoc, then we output the comma on
// the next line. This is the only case this happens.
comma := []byte{','}
if heredoc {
buf.WriteByte(newline)
comma = p.indent(comma)
}
buf.Write(comma)
if lit, ok := item.(*ast.LiteralType); ok && lit.LineComment != nil {
// if the next item doesn't have any comments, do not align
buf.WriteByte(blank) // align one space
for i := 0; i < longestLine-curLen; i++ {
buf.WriteByte(blank)
}
for _, comment := range lit.LineComment.List {
buf.WriteString(comment.Text)
}
}
lastItem := i == len(l.List)-1
if lastItem {
buf.WriteByte(newline)
}
if leadComment && !lastItem {
buf.WriteByte(newline)
}
lastHadLeadComment = leadComment
} else {
if insertSpaceBeforeItem {
buf.WriteByte(blank)
insertSpaceBeforeItem = false
}
// Output the item itself
// also indent each line
val := p.output(item)
curLen := len(val)
buf.Write(val)
// If this is a heredoc item we always have to output a newline
// so that it parses properly.
if heredoc {
buf.WriteByte(newline)
}
// If this isn't the last element, write a comma.
if i != len(l.List)-1 {
buf.WriteString(",")
insertSpaceBeforeItem = true
}
if lit, ok := item.(*ast.LiteralType); ok && lit.LineComment != nil {
// if the next item doesn't have any comments, do not align
buf.WriteByte(blank) // align one space
for i := 0; i < longestLine-curLen; i++ {
buf.WriteByte(blank)
}
for _, comment := range lit.LineComment.List {
buf.WriteString(comment.Text)
}
}
}
}
buf.WriteString("]")

View File

@ -74,6 +74,14 @@ func (s *Scanner) next() rune {
return eof
}
if ch == utf8.RuneError && size == 1 {
s.srcPos.Column++
s.srcPos.Offset += size
s.lastCharLen = size
s.err("illegal UTF-8 encoding")
return ch
}
// remember last position
s.prevPos = s.srcPos
@ -81,27 +89,18 @@ func (s *Scanner) next() rune {
s.lastCharLen = size
s.srcPos.Offset += size
if ch == utf8.RuneError && size == 1 {
s.err("illegal UTF-8 encoding")
return ch
}
if ch == '\n' {
s.srcPos.Line++
s.lastLineLen = s.srcPos.Column
s.srcPos.Column = 0
}
if ch == '\x00' {
// If we see a null character with data left, then that is an error
if ch == '\x00' && s.buf.Len() > 0 {
s.err("unexpected null character (0x00)")
return eof
}
if ch == '\uE123' {
s.err("unicode code point U+E123 reserved for internal use")
return utf8.RuneError
}
// debug
// fmt.Printf("ch: %q, offset:column: %d:%d\n", ch, s.srcPos.Offset, s.srcPos.Column)
return ch
@ -352,7 +351,7 @@ func (s *Scanner) scanNumber(ch rune) token.Type {
return token.NUMBER
}
// scanMantissa scans the mantissa beginning from the rune. It returns the next
// scanMantissa scans the mantissa begining from the rune. It returns the next
// non decimal rune. It's used to determine wheter it's a fraction or exponent.
func (s *Scanner) scanMantissa(ch rune) rune {
scanned := false
@ -433,16 +432,16 @@ func (s *Scanner) scanHeredoc() {
// Read the identifier
identBytes := s.src[offs : s.srcPos.Offset-s.lastCharLen]
if len(identBytes) == 0 || (len(identBytes) == 1 && identBytes[0] == '-') {
if len(identBytes) == 0 {
s.err("zero-length heredoc anchor")
return
}
var identRegexp *regexp.Regexp
if identBytes[0] == '-' {
identRegexp = regexp.MustCompile(fmt.Sprintf(`^[[:space:]]*%s\r*\z`, identBytes[1:]))
identRegexp = regexp.MustCompile(fmt.Sprintf(`[[:space:]]*%s\z`, identBytes[1:]))
} else {
identRegexp = regexp.MustCompile(fmt.Sprintf(`^[[:space:]]*%s\r*\z`, identBytes))
identRegexp = regexp.MustCompile(fmt.Sprintf(`[[:space:]]*%s\z`, identBytes))
}
// Read the actual string value
@ -552,7 +551,7 @@ func (s *Scanner) scanDigits(ch rune, base, n int) rune {
s.err("illegal char escape")
}
if n != start && ch != eof {
if n != start {
// we scanned all digits, put the last non digit char back,
// only if we read anything at all
s.unread()

View File

@ -246,7 +246,7 @@ func (s *Scanner) scanNumber(ch rune) token.Type {
return token.NUMBER
}
// scanMantissa scans the mantissa beginning from the rune. It returns the next
// scanMantissa scans the mantissa begining from the rune. It returns the next
// non decimal rune. It's used to determine wheter it's a fraction or exponent.
func (s *Scanner) scanMantissa(ch rune) rune {
scanned := false

4
vendor/modules.txt vendored
View File

@ -379,7 +379,7 @@ github.com/hashicorp/go-tfe
github.com/hashicorp/go-uuid
# github.com/hashicorp/go-version v1.0.0
github.com/hashicorp/go-version
# github.com/hashicorp/hcl v1.0.0
# github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f
github.com/hashicorp/hcl
github.com/hashicorp/hcl/hcl/ast
github.com/hashicorp/hcl/hcl/parser
@ -411,7 +411,7 @@ github.com/hashicorp/hil/scanner
github.com/hashicorp/logutils
# github.com/hashicorp/serf v0.0.0-20160124182025-e4ec8cc423bb
github.com/hashicorp/serf/coordinate
# github.com/hashicorp/terraform-config-inspect v0.0.0-20190129165904-67302cb0361b
# github.com/hashicorp/terraform-config-inspect v0.0.0-20190208170851-de963d052d6d
github.com/hashicorp/terraform-config-inspect/tfconfig
# github.com/hashicorp/vault v0.0.0-20161029210149-9a60bf2a50e4
github.com/hashicorp/vault/helper/pgpkeys