From 976d85ae926ef3a35f4d9039b949d3374ea7f0ff Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 1 Feb 2018 17:02:07 -0800 Subject: [PATCH 01/25] govendor fetch github.com/spf13/afero/... --- vendor/github.com/spf13/afero/LICENSE.txt | 174 +++++++ vendor/github.com/spf13/afero/README.md | 452 ++++++++++++++++++ vendor/github.com/spf13/afero/afero.go | 108 +++++ vendor/github.com/spf13/afero/appveyor.yml | 15 + vendor/github.com/spf13/afero/basepath.go | 145 ++++++ .../github.com/spf13/afero/cacheOnReadFs.go | 290 +++++++++++ vendor/github.com/spf13/afero/const_bsds.go | 22 + .../github.com/spf13/afero/const_win_unix.go | 25 + .../github.com/spf13/afero/copyOnWriteFs.go | 253 ++++++++++ vendor/github.com/spf13/afero/httpFs.go | 110 +++++ vendor/github.com/spf13/afero/ioutil.go | 230 +++++++++ vendor/github.com/spf13/afero/match.go | 110 +++++ vendor/github.com/spf13/afero/mem/dir.go | 37 ++ vendor/github.com/spf13/afero/mem/dirmap.go | 43 ++ vendor/github.com/spf13/afero/mem/file.go | 314 ++++++++++++ vendor/github.com/spf13/afero/memmap.go | 365 ++++++++++++++ vendor/github.com/spf13/afero/os.go | 94 ++++ vendor/github.com/spf13/afero/path.go | 108 +++++ vendor/github.com/spf13/afero/readonlyfs.go | 70 +++ vendor/github.com/spf13/afero/regexpfs.go | 214 +++++++++ vendor/github.com/spf13/afero/unionFile.go | 274 +++++++++++ vendor/github.com/spf13/afero/util.go | 330 +++++++++++++ vendor/vendor.json | 12 + 23 files changed, 3795 insertions(+) create mode 100644 vendor/github.com/spf13/afero/LICENSE.txt create mode 100644 vendor/github.com/spf13/afero/README.md create mode 100644 vendor/github.com/spf13/afero/afero.go create mode 100644 vendor/github.com/spf13/afero/appveyor.yml create mode 100644 vendor/github.com/spf13/afero/basepath.go create mode 100644 vendor/github.com/spf13/afero/cacheOnReadFs.go create mode 100644 vendor/github.com/spf13/afero/const_bsds.go create mode 100644 vendor/github.com/spf13/afero/const_win_unix.go create mode 100644 vendor/github.com/spf13/afero/copyOnWriteFs.go create mode 100644 vendor/github.com/spf13/afero/httpFs.go create mode 100644 vendor/github.com/spf13/afero/ioutil.go create mode 100644 vendor/github.com/spf13/afero/match.go create mode 100644 vendor/github.com/spf13/afero/mem/dir.go create mode 100644 vendor/github.com/spf13/afero/mem/dirmap.go create mode 100644 vendor/github.com/spf13/afero/mem/file.go create mode 100644 vendor/github.com/spf13/afero/memmap.go create mode 100644 vendor/github.com/spf13/afero/os.go create mode 100644 vendor/github.com/spf13/afero/path.go create mode 100644 vendor/github.com/spf13/afero/readonlyfs.go create mode 100644 vendor/github.com/spf13/afero/regexpfs.go create mode 100644 vendor/github.com/spf13/afero/unionFile.go create mode 100644 vendor/github.com/spf13/afero/util.go diff --git a/vendor/github.com/spf13/afero/LICENSE.txt b/vendor/github.com/spf13/afero/LICENSE.txt new file mode 100644 index 000000000..298f0e266 --- /dev/null +++ b/vendor/github.com/spf13/afero/LICENSE.txt @@ -0,0 +1,174 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/vendor/github.com/spf13/afero/README.md b/vendor/github.com/spf13/afero/README.md new file mode 100644 index 000000000..0c9b04b53 --- /dev/null +++ b/vendor/github.com/spf13/afero/README.md @@ -0,0 +1,452 @@ +![afero logo-sm](https://cloud.githubusercontent.com/assets/173412/11490338/d50e16dc-97a5-11e5-8b12-019a300d0fcb.png) + +A FileSystem Abstraction System for Go + +[![Build Status](https://travis-ci.org/spf13/afero.svg)](https://travis-ci.org/spf13/afero) [![Build status](https://ci.appveyor.com/api/projects/status/github/spf13/afero?branch=master&svg=true)](https://ci.appveyor.com/project/spf13/afero) [![GoDoc](https://godoc.org/github.com/spf13/afero?status.svg)](https://godoc.org/github.com/spf13/afero) [![Join the chat at https://gitter.im/spf13/afero](https://badges.gitter.im/Dev%20Chat.svg)](https://gitter.im/spf13/afero?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +# Overview + +Afero is an filesystem framework providing a simple, uniform and universal API +interacting with any filesystem, as an abstraction layer providing interfaces, +types and methods. Afero has an exceptionally clean interface and simple design +without needless constructors or initialization methods. + +Afero is also a library providing a base set of interoperable backend +filesystems that make it easy to work with afero while retaining all the power +and benefit of the os and ioutil packages. + +Afero provides significant improvements over using the os package alone, most +notably the ability to create mock and testing filesystems without relying on the disk. + +It is suitable for use in a any situation where you would consider using the OS +package as it provides an additional abstraction that makes it easy to use a +memory backed file system during testing. It also adds support for the http +filesystem for full interoperability. + + +## Afero Features + +* A single consistent API for accessing a variety of filesystems +* Interoperation between a variety of file system types +* A set of interfaces to encourage and enforce interoperability between backends +* An atomic cross platform memory backed file system +* Support for compositional (union) file systems by combining multiple file systems acting as one +* Specialized backends which modify existing filesystems (Read Only, Regexp filtered) +* A set of utility functions ported from io, ioutil & hugo to be afero aware + + +# Using Afero + +Afero is easy to use and easier to adopt. + +A few different ways you could use Afero: + +* Use the interfaces alone to define you own file system. +* Wrap for the OS packages. +* Define different filesystems for different parts of your application. +* Use Afero for mock filesystems while testing + +## Step 1: Install Afero + +First use go get to install the latest version of the library. + + $ go get github.com/spf13/afero + +Next include Afero in your application. +```go +import "github.com/spf13/afero" +``` + +## Step 2: Declare a backend + +First define a package variable and set it to a pointer to a filesystem. +```go +var AppFs = afero.NewMemMapFs() + +or + +var AppFs = afero.NewOsFs() +``` +It is important to note that if you repeat the composite literal you +will be using a completely new and isolated filesystem. In the case of +OsFs it will still use the same underlying filesystem but will reduce +the ability to drop in other filesystems as desired. + +## Step 3: Use it like you would the OS package + +Throughout your application use any function and method like you normally +would. + +So if my application before had: +```go +os.Open('/tmp/foo') +``` +We would replace it with: +```go +AppFs.Open('/tmp/foo') +``` + +`AppFs` being the variable we defined above. + + +## List of all available functions + +File System Methods Available: +```go +Chmod(name string, mode os.FileMode) : error +Chtimes(name string, atime time.Time, mtime time.Time) : error +Create(name string) : File, error +Mkdir(name string, perm os.FileMode) : error +MkdirAll(path string, perm os.FileMode) : error +Name() : string +Open(name string) : File, error +OpenFile(name string, flag int, perm os.FileMode) : File, error +Remove(name string) : error +RemoveAll(path string) : error +Rename(oldname, newname string) : error +Stat(name string) : os.FileInfo, error +``` +File Interfaces and Methods Available: +```go +io.Closer +io.Reader +io.ReaderAt +io.Seeker +io.Writer +io.WriterAt + +Name() : string +Readdir(count int) : []os.FileInfo, error +Readdirnames(n int) : []string, error +Stat() : os.FileInfo, error +Sync() : error +Truncate(size int64) : error +WriteString(s string) : ret int, err error +``` +In some applications it may make sense to define a new package that +simply exports the file system variable for easy access from anywhere. + +## Using Afero's utility functions + +Afero provides a set of functions to make it easier to use the underlying file systems. +These functions have been primarily ported from io & ioutil with some developed for Hugo. + +The afero utilities support all afero compatible backends. + +The list of utilities includes: + +```go +DirExists(path string) (bool, error) +Exists(path string) (bool, error) +FileContainsBytes(filename string, subslice []byte) (bool, error) +GetTempDir(subPath string) string +IsDir(path string) (bool, error) +IsEmpty(path string) (bool, error) +ReadDir(dirname string) ([]os.FileInfo, error) +ReadFile(filename string) ([]byte, error) +SafeWriteReader(path string, r io.Reader) (err error) +TempDir(dir, prefix string) (name string, err error) +TempFile(dir, prefix string) (f File, err error) +Walk(root string, walkFn filepath.WalkFunc) error +WriteFile(filename string, data []byte, perm os.FileMode) error +WriteReader(path string, r io.Reader) (err error) +``` +For a complete list see [Afero's GoDoc](https://godoc.org/github.com/spf13/afero) + +They are available under two different approaches to use. You can either call +them directly where the first parameter of each function will be the file +system, or you can declare a new `Afero`, a custom type used to bind these +functions as methods to a given filesystem. + +### Calling utilities directly + +```go +fs := new(afero.MemMapFs) +f, err := afero.TempFile(fs,"", "ioutil-test") + +``` + +### Calling via Afero + +```go +fs := afero.NewMemMapFs() +afs := &afero.Afero{Fs: fs} +f, err := afs.TempFile("", "ioutil-test") +``` + +## Using Afero for Testing + +There is a large benefit to using a mock filesystem for testing. It has a +completely blank state every time it is initialized and can be easily +reproducible regardless of OS. You could create files to your heart’s content +and the file access would be fast while also saving you from all the annoying +issues with deleting temporary files, Windows file locking, etc. The MemMapFs +backend is perfect for testing. + +* Much faster than performing I/O operations on disk +* Avoid security issues and permissions +* Far more control. 'rm -rf /' with confidence +* Test setup is far more easier to do +* No test cleanup needed + +One way to accomplish this is to define a variable as mentioned above. +In your application this will be set to afero.NewOsFs() during testing you +can set it to afero.NewMemMapFs(). + +It wouldn't be uncommon to have each test initialize a blank slate memory +backend. To do this I would define my `appFS = afero.NewOsFs()` somewhere +appropriate in my application code. This approach ensures that Tests are order +independent, with no test relying on the state left by an earlier test. + +Then in my tests I would initialize a new MemMapFs for each test: +```go +func TestExist(t *testing.T) { + appFS := afero.NewMemMapFs() + // create test files and directories + appFS.MkdirAll("src/a", 0755) + afero.WriteFile(appFS, "src/a/b", []byte("file b"), 0644) + afero.WriteFile(appFS, "src/c", []byte("file c"), 0644) + name := "src/c" + _, err := appFS.Stat(name) + if os.IsNotExist(err) { + t.Errorf("file \"%s\" does not exist.\n", name) + } +} +``` + +# Available Backends + +## Operating System Native + +### OsFs + +The first is simply a wrapper around the native OS calls. This makes it +very easy to use as all of the calls are the same as the existing OS +calls. It also makes it trivial to have your code use the OS during +operation and a mock filesystem during testing or as needed. + +```go +appfs := afero.NewOsFs() +appfs.MkdirAll("src/a", 0755)) +``` + +## Memory Backed Storage + +### MemMapFs + +Afero also provides a fully atomic memory backed filesystem perfect for use in +mocking and to speed up unnecessary disk io when persistence isn’t +necessary. It is fully concurrent and will work within go routines +safely. + +```go +mm := afero.NewMemMapFs() +mm.MkdirAll("src/a", 0755)) +``` + +#### InMemoryFile + +As part of MemMapFs, Afero also provides an atomic, fully concurrent memory +backed file implementation. This can be used in other memory backed file +systems with ease. Plans are to add a radix tree memory stored file +system using InMemoryFile. + +## Network Interfaces + +### SftpFs + +Afero has experimental support for secure file transfer protocol (sftp). Which can +be used to perform file operations over a encrypted channel. + +## Filtering Backends + +### BasePathFs + +The BasePathFs restricts all operations to a given path within an Fs. +The given file name to the operations on this Fs will be prepended with +the base path before calling the source Fs. + +```go +bp := afero.NewBasePathFs(afero.NewOsFs(), "/base/path") +``` + +### ReadOnlyFs + +A thin wrapper around the source Fs providing a read only view. + +```go +fs := afero.NewReadOnlyFs(afero.NewOsFs()) +_, err := fs.Create("/file.txt") +// err = syscall.EPERM +``` + +# RegexpFs + +A filtered view on file names, any file NOT matching +the passed regexp will be treated as non-existing. +Files not matching the regexp provided will not be created. +Directories are not filtered. + +```go +fs := afero.NewRegexpFs(afero.NewMemMapFs(), regexp.MustCompile(`\.txt$`)) +_, err := fs.Create("/file.html") +// err = syscall.ENOENT +``` + +### HttpFs + +Afero provides an http compatible backend which can wrap any of the existing +backends. + +The Http package requires a slightly specific version of Open which +returns an http.File type. + +Afero provides an httpFs file system which satisfies this requirement. +Any Afero FileSystem can be used as an httpFs. + +```go +httpFs := afero.NewHttpFs() +fileserver := http.FileServer(httpFs.Dir())) +http.Handle("/", fileserver) +``` + +## Composite Backends + +Afero provides the ability have two filesystems (or more) act as a single +file system. + +### CacheOnReadFs + +The CacheOnReadFs will lazily make copies of any accessed files from the base +layer into the overlay. Subsequent reads will be pulled from the overlay +directly permitting the request is within the cache duration of when it was +created in the overlay. + +If the base filesystem is writeable, any changes to files will be +done first to the base, then to the overlay layer. Write calls to open file +handles like `Write()` or `Truncate()` to the overlay first. + +To writing files to the overlay only, you can use the overlay Fs directly (not +via the union Fs). + +Cache files in the layer for the given time.Duration, a cache duration of 0 +means "forever" meaning the file will not be re-requested from the base ever. + +A read-only base will make the overlay also read-only but still copy files +from the base to the overlay when they're not present (or outdated) in the +caching layer. + +```go +base := afero.NewOsFs() +layer := afero.NewMemMapFs() +ufs := afero.NewCacheOnReadFs(base, layer, 100 * time.Second) +``` + +### CopyOnWriteFs() + +The CopyOnWriteFs is a read only base file system with a potentially +writeable layer on top. + +Read operations will first look in the overlay and if not found there, will +serve the file from the base. + +Changes to the file system will only be made in the overlay. + +Any attempt to modify a file found only in the base will copy the file to the +overlay layer before modification (including opening a file with a writable +handle). + +Removing and Renaming files present only in the base layer is not currently +permitted. If a file is present in the base layer and the overlay, only the +overlay will be removed/renamed. + +```go + base := afero.NewOsFs() + roBase := afero.NewReadOnlyFs(base) + ufs := afero.NewCopyOnWriteFs(roBase, afero.NewMemMapFs()) + + fh, _ = ufs.Create("/home/test/file2.txt") + fh.WriteString("This is a test") + fh.Close() +``` + +In this example all write operations will only occur in memory (MemMapFs) +leaving the base filesystem (OsFs) untouched. + + +## Desired/possible backends + +The following is a short list of possible backends we hope someone will +implement: + +* SSH +* ZIP +* TAR +* S3 + +# About the project + +## What's in the name + +Afero comes from the latin roots Ad-Facere. + +**"Ad"** is a prefix meaning "to". + +**"Facere"** is a form of the root "faciō" making "make or do". + +The literal meaning of afero is "to make" or "to do" which seems very fitting +for a library that allows one to make files and directories and do things with them. + +The English word that shares the same roots as Afero is "affair". Affair shares +the same concept but as a noun it means "something that is made or done" or "an +object of a particular type". + +It's also nice that unlike some of my other libraries (hugo, cobra, viper) it +Googles very well. + +## Release Notes + +* **0.10.0** 2015.12.10 + * Full compatibility with Windows + * Introduction of afero utilities + * Test suite rewritten to work cross platform + * Normalize paths for MemMapFs + * Adding Sync to the file interface + * **Breaking Change** Walk and ReadDir have changed parameter order + * Moving types used by MemMapFs to a subpackage + * General bugfixes and improvements +* **0.9.0** 2015.11.05 + * New Walk function similar to filepath.Walk + * MemMapFs.OpenFile handles O_CREATE, O_APPEND, O_TRUNC + * MemMapFs.Remove now really deletes the file + * InMemoryFile.Readdir and Readdirnames work correctly + * InMemoryFile functions lock it for concurrent access + * Test suite improvements +* **0.8.0** 2014.10.28 + * First public version + * Interfaces feel ready for people to build using + * Interfaces satisfy all known uses + * MemMapFs passes the majority of the OS test suite + * OsFs passes the majority of the OS test suite + +## Contributing + +1. Fork it +2. Create your feature branch (`git checkout -b my-new-feature`) +3. Commit your changes (`git commit -am 'Add some feature'`) +4. Push to the branch (`git push origin my-new-feature`) +5. Create new Pull Request + +## Contributors + +Names in no particular order: + +* [spf13](https://github.com/spf13) +* [jaqx0r](https://github.com/jaqx0r) +* [mbertschler](https://github.com/mbertschler) +* [xor-gate](https://github.com/xor-gate) + +## License + +Afero is released under the Apache 2.0 license. See +[LICENSE.txt](https://github.com/spf13/afero/blob/master/LICENSE.txt) diff --git a/vendor/github.com/spf13/afero/afero.go b/vendor/github.com/spf13/afero/afero.go new file mode 100644 index 000000000..f5b5e127c --- /dev/null +++ b/vendor/github.com/spf13/afero/afero.go @@ -0,0 +1,108 @@ +// Copyright © 2014 Steve Francia . +// Copyright 2013 tsuru authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package afero provides types and methods for interacting with the filesystem, +// as an abstraction layer. + +// Afero also provides a few implementations that are mostly interoperable. One that +// uses the operating system filesystem, one that uses memory to store files +// (cross platform) and an interface that should be implemented if you want to +// provide your own filesystem. + +package afero + +import ( + "errors" + "io" + "os" + "time" +) + +type Afero struct { + Fs +} + +// File represents a file in the filesystem. +type File interface { + io.Closer + io.Reader + io.ReaderAt + io.Seeker + io.Writer + io.WriterAt + + Name() string + Readdir(count int) ([]os.FileInfo, error) + Readdirnames(n int) ([]string, error) + Stat() (os.FileInfo, error) + Sync() error + Truncate(size int64) error + WriteString(s string) (ret int, err error) +} + +// Fs is the filesystem interface. +// +// Any simulated or real filesystem should implement this interface. +type Fs interface { + // Create creates a file in the filesystem, returning the file and an + // error, if any happens. + Create(name string) (File, error) + + // Mkdir creates a directory in the filesystem, return an error if any + // happens. + Mkdir(name string, perm os.FileMode) error + + // MkdirAll creates a directory path and all parents that does not exist + // yet. + MkdirAll(path string, perm os.FileMode) error + + // Open opens a file, returning it or an error, if any happens. + Open(name string) (File, error) + + // OpenFile opens a file using the given flags and the given mode. + OpenFile(name string, flag int, perm os.FileMode) (File, error) + + // Remove removes a file identified by name, returning an error, if any + // happens. + Remove(name string) error + + // RemoveAll removes a directory path and any children it contains. It + // does not fail if the path does not exist (return nil). + RemoveAll(path string) error + + // Rename renames a file. + Rename(oldname, newname string) error + + // Stat returns a FileInfo describing the named file, or an error, if any + // happens. + Stat(name string) (os.FileInfo, error) + + // The name of this FileSystem + Name() string + + //Chmod changes the mode of the named file to mode. + Chmod(name string, mode os.FileMode) error + + //Chtimes changes the access and modification times of the named file + Chtimes(name string, atime time.Time, mtime time.Time) error +} + +var ( + ErrFileClosed = errors.New("File is closed") + ErrOutOfRange = errors.New("Out of range") + ErrTooLarge = errors.New("Too large") + ErrFileNotFound = os.ErrNotExist + ErrFileExists = os.ErrExist + ErrDestinationExists = os.ErrExist +) diff --git a/vendor/github.com/spf13/afero/appveyor.yml b/vendor/github.com/spf13/afero/appveyor.yml new file mode 100644 index 000000000..a633ad500 --- /dev/null +++ b/vendor/github.com/spf13/afero/appveyor.yml @@ -0,0 +1,15 @@ +version: '{build}' +clone_folder: C:\gopath\src\github.com\spf13\afero +environment: + GOPATH: C:\gopath +build_script: +- cmd: >- + go version + + go env + + go get -v github.com/spf13/afero/... + + go build github.com/spf13/afero +test_script: +- cmd: go test -race -v github.com/spf13/afero/... diff --git a/vendor/github.com/spf13/afero/basepath.go b/vendor/github.com/spf13/afero/basepath.go new file mode 100644 index 000000000..5e4fc2ec0 --- /dev/null +++ b/vendor/github.com/spf13/afero/basepath.go @@ -0,0 +1,145 @@ +package afero + +import ( + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "time" +) + +// The BasePathFs restricts all operations to a given path within an Fs. +// The given file name to the operations on this Fs will be prepended with +// the base path before calling the base Fs. +// Any file name (after filepath.Clean()) outside this base path will be +// treated as non existing file. +// +// Note that it does not clean the error messages on return, so you may +// reveal the real path on errors. +type BasePathFs struct { + source Fs + path string +} + +func NewBasePathFs(source Fs, path string) Fs { + return &BasePathFs{source: source, path: path} +} + +// on a file outside the base path it returns the given file name and an error, +// else the given file with the base path prepended +func (b *BasePathFs) RealPath(name string) (path string, err error) { + if err := validateBasePathName(name); err != nil { + return "", err + } + + bpath := filepath.Clean(b.path) + path = filepath.Clean(filepath.Join(bpath, name)) + if !strings.HasPrefix(path, bpath) { + return name, os.ErrNotExist + } + + return path, nil +} + +func validateBasePathName(name string) error { + if runtime.GOOS != "windows" { + // Not much to do here; + // the virtual file paths all look absolute on *nix. + return nil + } + + // On Windows a common mistake would be to provide an absolute OS path + // We could strip out the base part, but that would not be very portable. + if filepath.IsAbs(name) { + return &os.PathError{Op: "realPath", Path: name, Err: errors.New("got a real OS path instead of a virtual")} + } + + return nil +} + +func (b *BasePathFs) Chtimes(name string, atime, mtime time.Time) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "chtimes", Path: name, Err: err} + } + return b.source.Chtimes(name, atime, mtime) +} + +func (b *BasePathFs) Chmod(name string, mode os.FileMode) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "chmod", Path: name, Err: err} + } + return b.source.Chmod(name, mode) +} + +func (b *BasePathFs) Name() string { + return "BasePathFs" +} + +func (b *BasePathFs) Stat(name string) (fi os.FileInfo, err error) { + if name, err = b.RealPath(name); err != nil { + return nil, &os.PathError{Op: "stat", Path: name, Err: err} + } + return b.source.Stat(name) +} + +func (b *BasePathFs) Rename(oldname, newname string) (err error) { + if oldname, err = b.RealPath(oldname); err != nil { + return &os.PathError{Op: "rename", Path: oldname, Err: err} + } + if newname, err = b.RealPath(newname); err != nil { + return &os.PathError{Op: "rename", Path: newname, Err: err} + } + return b.source.Rename(oldname, newname) +} + +func (b *BasePathFs) RemoveAll(name string) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "remove_all", Path: name, Err: err} + } + return b.source.RemoveAll(name) +} + +func (b *BasePathFs) Remove(name string) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "remove", Path: name, Err: err} + } + return b.source.Remove(name) +} + +func (b *BasePathFs) OpenFile(name string, flag int, mode os.FileMode) (f File, err error) { + if name, err = b.RealPath(name); err != nil { + return nil, &os.PathError{Op: "openfile", Path: name, Err: err} + } + return b.source.OpenFile(name, flag, mode) +} + +func (b *BasePathFs) Open(name string) (f File, err error) { + if name, err = b.RealPath(name); err != nil { + return nil, &os.PathError{Op: "open", Path: name, Err: err} + } + return b.source.Open(name) +} + +func (b *BasePathFs) Mkdir(name string, mode os.FileMode) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "mkdir", Path: name, Err: err} + } + return b.source.Mkdir(name, mode) +} + +func (b *BasePathFs) MkdirAll(name string, mode os.FileMode) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "mkdir", Path: name, Err: err} + } + return b.source.MkdirAll(name, mode) +} + +func (b *BasePathFs) Create(name string) (f File, err error) { + if name, err = b.RealPath(name); err != nil { + return nil, &os.PathError{Op: "create", Path: name, Err: err} + } + return b.source.Create(name) +} + +// vim: ts=4 sw=4 noexpandtab nolist syn=go diff --git a/vendor/github.com/spf13/afero/cacheOnReadFs.go b/vendor/github.com/spf13/afero/cacheOnReadFs.go new file mode 100644 index 000000000..b026e0de8 --- /dev/null +++ b/vendor/github.com/spf13/afero/cacheOnReadFs.go @@ -0,0 +1,290 @@ +package afero + +import ( + "os" + "syscall" + "time" +) + +// If the cache duration is 0, cache time will be unlimited, i.e. once +// a file is in the layer, the base will never be read again for this file. +// +// For cache times greater than 0, the modification time of a file is +// checked. Note that a lot of file system implementations only allow a +// resolution of a second for timestamps... or as the godoc for os.Chtimes() +// states: "The underlying filesystem may truncate or round the values to a +// less precise time unit." +// +// This caching union will forward all write calls also to the base file +// system first. To prevent writing to the base Fs, wrap it in a read-only +// filter - Note: this will also make the overlay read-only, for writing files +// in the overlay, use the overlay Fs directly, not via the union Fs. +type CacheOnReadFs struct { + base Fs + layer Fs + cacheTime time.Duration +} + +func NewCacheOnReadFs(base Fs, layer Fs, cacheTime time.Duration) Fs { + return &CacheOnReadFs{base: base, layer: layer, cacheTime: cacheTime} +} + +type cacheState int + +const ( + // not present in the overlay, unknown if it exists in the base: + cacheMiss cacheState = iota + // present in the overlay and in base, base file is newer: + cacheStale + // present in the overlay - with cache time == 0 it may exist in the base, + // with cacheTime > 0 it exists in the base and is same age or newer in the + // overlay + cacheHit + // happens if someone writes directly to the overlay without + // going through this union + cacheLocal +) + +func (u *CacheOnReadFs) cacheStatus(name string) (state cacheState, fi os.FileInfo, err error) { + var lfi, bfi os.FileInfo + lfi, err = u.layer.Stat(name) + if err == nil { + if u.cacheTime == 0 { + return cacheHit, lfi, nil + } + if lfi.ModTime().Add(u.cacheTime).Before(time.Now()) { + bfi, err = u.base.Stat(name) + if err != nil { + return cacheLocal, lfi, nil + } + if bfi.ModTime().After(lfi.ModTime()) { + return cacheStale, bfi, nil + } + } + return cacheHit, lfi, nil + } + + if err == syscall.ENOENT || os.IsNotExist(err) { + return cacheMiss, nil, nil + } + + return cacheMiss, nil, err +} + +func (u *CacheOnReadFs) copyToLayer(name string) error { + return copyToLayer(u.base, u.layer, name) +} + +func (u *CacheOnReadFs) Chtimes(name string, atime, mtime time.Time) error { + st, _, err := u.cacheStatus(name) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit: + err = u.base.Chtimes(name, atime, mtime) + case cacheStale, cacheMiss: + if err := u.copyToLayer(name); err != nil { + return err + } + err = u.base.Chtimes(name, atime, mtime) + } + if err != nil { + return err + } + return u.layer.Chtimes(name, atime, mtime) +} + +func (u *CacheOnReadFs) Chmod(name string, mode os.FileMode) error { + st, _, err := u.cacheStatus(name) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit: + err = u.base.Chmod(name, mode) + case cacheStale, cacheMiss: + if err := u.copyToLayer(name); err != nil { + return err + } + err = u.base.Chmod(name, mode) + } + if err != nil { + return err + } + return u.layer.Chmod(name, mode) +} + +func (u *CacheOnReadFs) Stat(name string) (os.FileInfo, error) { + st, fi, err := u.cacheStatus(name) + if err != nil { + return nil, err + } + switch st { + case cacheMiss: + return u.base.Stat(name) + default: // cacheStale has base, cacheHit and cacheLocal the layer os.FileInfo + return fi, nil + } +} + +func (u *CacheOnReadFs) Rename(oldname, newname string) error { + st, _, err := u.cacheStatus(oldname) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit: + err = u.base.Rename(oldname, newname) + case cacheStale, cacheMiss: + if err := u.copyToLayer(oldname); err != nil { + return err + } + err = u.base.Rename(oldname, newname) + } + if err != nil { + return err + } + return u.layer.Rename(oldname, newname) +} + +func (u *CacheOnReadFs) Remove(name string) error { + st, _, err := u.cacheStatus(name) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit, cacheStale, cacheMiss: + err = u.base.Remove(name) + } + if err != nil { + return err + } + return u.layer.Remove(name) +} + +func (u *CacheOnReadFs) RemoveAll(name string) error { + st, _, err := u.cacheStatus(name) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit, cacheStale, cacheMiss: + err = u.base.RemoveAll(name) + } + if err != nil { + return err + } + return u.layer.RemoveAll(name) +} + +func (u *CacheOnReadFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + st, _, err := u.cacheStatus(name) + if err != nil { + return nil, err + } + switch st { + case cacheLocal, cacheHit: + default: + if err := u.copyToLayer(name); err != nil { + return nil, err + } + } + if flag&(os.O_WRONLY|syscall.O_RDWR|os.O_APPEND|os.O_CREATE|os.O_TRUNC) != 0 { + bfi, err := u.base.OpenFile(name, flag, perm) + if err != nil { + return nil, err + } + lfi, err := u.layer.OpenFile(name, flag, perm) + if err != nil { + bfi.Close() // oops, what if O_TRUNC was set and file opening in the layer failed...? + return nil, err + } + return &UnionFile{base: bfi, layer: lfi}, nil + } + return u.layer.OpenFile(name, flag, perm) +} + +func (u *CacheOnReadFs) Open(name string) (File, error) { + st, fi, err := u.cacheStatus(name) + if err != nil { + return nil, err + } + + switch st { + case cacheLocal: + return u.layer.Open(name) + + case cacheMiss: + bfi, err := u.base.Stat(name) + if err != nil { + return nil, err + } + if bfi.IsDir() { + return u.base.Open(name) + } + if err := u.copyToLayer(name); err != nil { + return nil, err + } + return u.layer.Open(name) + + case cacheStale: + if !fi.IsDir() { + if err := u.copyToLayer(name); err != nil { + return nil, err + } + return u.layer.Open(name) + } + case cacheHit: + if !fi.IsDir() { + return u.layer.Open(name) + } + } + // the dirs from cacheHit, cacheStale fall down here: + bfile, _ := u.base.Open(name) + lfile, err := u.layer.Open(name) + if err != nil && bfile == nil { + return nil, err + } + return &UnionFile{base: bfile, layer: lfile}, nil +} + +func (u *CacheOnReadFs) Mkdir(name string, perm os.FileMode) error { + err := u.base.Mkdir(name, perm) + if err != nil { + return err + } + return u.layer.MkdirAll(name, perm) // yes, MkdirAll... we cannot assume it exists in the cache +} + +func (u *CacheOnReadFs) Name() string { + return "CacheOnReadFs" +} + +func (u *CacheOnReadFs) MkdirAll(name string, perm os.FileMode) error { + err := u.base.MkdirAll(name, perm) + if err != nil { + return err + } + return u.layer.MkdirAll(name, perm) +} + +func (u *CacheOnReadFs) Create(name string) (File, error) { + bfh, err := u.base.Create(name) + if err != nil { + return nil, err + } + lfh, err := u.layer.Create(name) + if err != nil { + // oops, see comment about OS_TRUNC above, should we remove? then we have to + // remember if the file did not exist before + bfh.Close() + return nil, err + } + return &UnionFile{base: bfh, layer: lfh}, nil +} diff --git a/vendor/github.com/spf13/afero/const_bsds.go b/vendor/github.com/spf13/afero/const_bsds.go new file mode 100644 index 000000000..5728243d9 --- /dev/null +++ b/vendor/github.com/spf13/afero/const_bsds.go @@ -0,0 +1,22 @@ +// Copyright © 2016 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build darwin openbsd freebsd netbsd dragonfly + +package afero + +import ( + "syscall" +) + +const BADFD = syscall.EBADF diff --git a/vendor/github.com/spf13/afero/const_win_unix.go b/vendor/github.com/spf13/afero/const_win_unix.go new file mode 100644 index 000000000..968fc2783 --- /dev/null +++ b/vendor/github.com/spf13/afero/const_win_unix.go @@ -0,0 +1,25 @@ +// Copyright © 2016 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +build !darwin +// +build !openbsd +// +build !freebsd +// +build !dragonfly +// +build !netbsd + +package afero + +import ( + "syscall" +) + +const BADFD = syscall.EBADFD diff --git a/vendor/github.com/spf13/afero/copyOnWriteFs.go b/vendor/github.com/spf13/afero/copyOnWriteFs.go new file mode 100644 index 000000000..f2ebcd226 --- /dev/null +++ b/vendor/github.com/spf13/afero/copyOnWriteFs.go @@ -0,0 +1,253 @@ +package afero + +import ( + "fmt" + "os" + "path/filepath" + "syscall" + "time" +) + +// The CopyOnWriteFs is a union filesystem: a read only base file system with +// a possibly writeable layer on top. Changes to the file system will only +// be made in the overlay: Changing an existing file in the base layer which +// is not present in the overlay will copy the file to the overlay ("changing" +// includes also calls to e.g. Chtimes() and Chmod()). +// +// Reading directories is currently only supported via Open(), not OpenFile(). +type CopyOnWriteFs struct { + base Fs + layer Fs +} + +func NewCopyOnWriteFs(base Fs, layer Fs) Fs { + return &CopyOnWriteFs{base: base, layer: layer} +} + +// Returns true if the file is not in the overlay +func (u *CopyOnWriteFs) isBaseFile(name string) (bool, error) { + if _, err := u.layer.Stat(name); err == nil { + return false, nil + } + _, err := u.base.Stat(name) + if err != nil { + if oerr, ok := err.(*os.PathError); ok { + if oerr.Err == os.ErrNotExist || oerr.Err == syscall.ENOENT || oerr.Err == syscall.ENOTDIR { + return false, nil + } + } + if err == syscall.ENOENT { + return false, nil + } + } + return true, err +} + +func (u *CopyOnWriteFs) copyToLayer(name string) error { + return copyToLayer(u.base, u.layer, name) +} + +func (u *CopyOnWriteFs) Chtimes(name string, atime, mtime time.Time) error { + b, err := u.isBaseFile(name) + if err != nil { + return err + } + if b { + if err := u.copyToLayer(name); err != nil { + return err + } + } + return u.layer.Chtimes(name, atime, mtime) +} + +func (u *CopyOnWriteFs) Chmod(name string, mode os.FileMode) error { + b, err := u.isBaseFile(name) + if err != nil { + return err + } + if b { + if err := u.copyToLayer(name); err != nil { + return err + } + } + return u.layer.Chmod(name, mode) +} + +func (u *CopyOnWriteFs) Stat(name string) (os.FileInfo, error) { + fi, err := u.layer.Stat(name) + if err != nil { + origErr := err + if e, ok := err.(*os.PathError); ok { + err = e.Err + } + if err == os.ErrNotExist || err == syscall.ENOENT || err == syscall.ENOTDIR { + return u.base.Stat(name) + } + return nil, origErr + } + return fi, nil +} + +// Renaming files present only in the base layer is not permitted +func (u *CopyOnWriteFs) Rename(oldname, newname string) error { + b, err := u.isBaseFile(oldname) + if err != nil { + return err + } + if b { + return syscall.EPERM + } + return u.layer.Rename(oldname, newname) +} + +// Removing files present only in the base layer is not permitted. If +// a file is present in the base layer and the overlay, only the overlay +// will be removed. +func (u *CopyOnWriteFs) Remove(name string) error { + err := u.layer.Remove(name) + switch err { + case syscall.ENOENT: + _, err = u.base.Stat(name) + if err == nil { + return syscall.EPERM + } + return syscall.ENOENT + default: + return err + } +} + +func (u *CopyOnWriteFs) RemoveAll(name string) error { + err := u.layer.RemoveAll(name) + switch err { + case syscall.ENOENT: + _, err = u.base.Stat(name) + if err == nil { + return syscall.EPERM + } + return syscall.ENOENT + default: + return err + } +} + +func (u *CopyOnWriteFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + b, err := u.isBaseFile(name) + if err != nil { + return nil, err + } + + if flag&(os.O_WRONLY|os.O_RDWR|os.O_APPEND|os.O_CREATE|os.O_TRUNC) != 0 { + if b { + if err = u.copyToLayer(name); err != nil { + return nil, err + } + return u.layer.OpenFile(name, flag, perm) + } + + dir := filepath.Dir(name) + isaDir, err := IsDir(u.base, dir) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + if isaDir { + if err = u.layer.MkdirAll(dir, 0777); err != nil { + return nil, err + } + return u.layer.OpenFile(name, flag, perm) + } + + isaDir, err = IsDir(u.layer, dir) + if err != nil { + return nil, err + } + if isaDir { + return u.layer.OpenFile(name, flag, perm) + } + + return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOTDIR} // ...or os.ErrNotExist? + } + if b { + return u.base.OpenFile(name, flag, perm) + } + return u.layer.OpenFile(name, flag, perm) +} + +// This function handles the 9 different possibilities caused +// by the union which are the intersection of the following... +// layer: doesn't exist, exists as a file, and exists as a directory +// base: doesn't exist, exists as a file, and exists as a directory +func (u *CopyOnWriteFs) Open(name string) (File, error) { + // Since the overlay overrides the base we check that first + b, err := u.isBaseFile(name) + if err != nil { + return nil, err + } + + // If overlay doesn't exist, return the base (base state irrelevant) + if b { + return u.base.Open(name) + } + + // If overlay is a file, return it (base state irrelevant) + dir, err := IsDir(u.layer, name) + if err != nil { + return nil, err + } + if !dir { + return u.layer.Open(name) + } + + // Overlay is a directory, base state now matters. + // Base state has 3 states to check but 2 outcomes: + // A. It's a file or non-readable in the base (return just the overlay) + // B. It's an accessible directory in the base (return a UnionFile) + + // If base is file or nonreadable, return overlay + dir, err = IsDir(u.base, name) + if !dir || err != nil { + return u.layer.Open(name) + } + + // Both base & layer are directories + // Return union file (if opens are without error) + bfile, bErr := u.base.Open(name) + lfile, lErr := u.layer.Open(name) + + // If either have errors at this point something is very wrong. Return nil and the errors + if bErr != nil || lErr != nil { + return nil, fmt.Errorf("BaseErr: %v\nOverlayErr: %v", bErr, lErr) + } + + return &UnionFile{base: bfile, layer: lfile}, nil +} + +func (u *CopyOnWriteFs) Mkdir(name string, perm os.FileMode) error { + dir, err := IsDir(u.base, name) + if err != nil { + return u.layer.MkdirAll(name, perm) + } + if dir { + return syscall.EEXIST + } + return u.layer.MkdirAll(name, perm) +} + +func (u *CopyOnWriteFs) Name() string { + return "CopyOnWriteFs" +} + +func (u *CopyOnWriteFs) MkdirAll(name string, perm os.FileMode) error { + dir, err := IsDir(u.base, name) + if err != nil { + return u.layer.MkdirAll(name, perm) + } + if dir { + return syscall.EEXIST + } + return u.layer.MkdirAll(name, perm) +} + +func (u *CopyOnWriteFs) Create(name string) (File, error) { + return u.OpenFile(name, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0666) +} diff --git a/vendor/github.com/spf13/afero/httpFs.go b/vendor/github.com/spf13/afero/httpFs.go new file mode 100644 index 000000000..c42193688 --- /dev/null +++ b/vendor/github.com/spf13/afero/httpFs.go @@ -0,0 +1,110 @@ +// Copyright © 2014 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "errors" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "time" +) + +type httpDir struct { + basePath string + fs HttpFs +} + +func (d httpDir) Open(name string) (http.File, error) { + if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 || + strings.Contains(name, "\x00") { + return nil, errors.New("http: invalid character in file path") + } + dir := string(d.basePath) + if dir == "" { + dir = "." + } + + f, err := d.fs.Open(filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name)))) + if err != nil { + return nil, err + } + return f, nil +} + +type HttpFs struct { + source Fs +} + +func NewHttpFs(source Fs) *HttpFs { + return &HttpFs{source: source} +} + +func (h HttpFs) Dir(s string) *httpDir { + return &httpDir{basePath: s, fs: h} +} + +func (h HttpFs) Name() string { return "h HttpFs" } + +func (h HttpFs) Create(name string) (File, error) { + return h.source.Create(name) +} + +func (h HttpFs) Chmod(name string, mode os.FileMode) error { + return h.source.Chmod(name, mode) +} + +func (h HttpFs) Chtimes(name string, atime time.Time, mtime time.Time) error { + return h.source.Chtimes(name, atime, mtime) +} + +func (h HttpFs) Mkdir(name string, perm os.FileMode) error { + return h.source.Mkdir(name, perm) +} + +func (h HttpFs) MkdirAll(path string, perm os.FileMode) error { + return h.source.MkdirAll(path, perm) +} + +func (h HttpFs) Open(name string) (http.File, error) { + f, err := h.source.Open(name) + if err == nil { + if httpfile, ok := f.(http.File); ok { + return httpfile, nil + } + } + return nil, err +} + +func (h HttpFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + return h.source.OpenFile(name, flag, perm) +} + +func (h HttpFs) Remove(name string) error { + return h.source.Remove(name) +} + +func (h HttpFs) RemoveAll(path string) error { + return h.source.RemoveAll(path) +} + +func (h HttpFs) Rename(oldname, newname string) error { + return h.source.Rename(oldname, newname) +} + +func (h HttpFs) Stat(name string) (os.FileInfo, error) { + return h.source.Stat(name) +} diff --git a/vendor/github.com/spf13/afero/ioutil.go b/vendor/github.com/spf13/afero/ioutil.go new file mode 100644 index 000000000..5c3a3d8ff --- /dev/null +++ b/vendor/github.com/spf13/afero/ioutil.go @@ -0,0 +1,230 @@ +// Copyright ©2015 The Go Authors +// Copyright ©2015 Steve Francia +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "bytes" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "sync" + "time" +) + +// byName implements sort.Interface. +type byName []os.FileInfo + +func (f byName) Len() int { return len(f) } +func (f byName) Less(i, j int) bool { return f[i].Name() < f[j].Name() } +func (f byName) Swap(i, j int) { f[i], f[j] = f[j], f[i] } + +// ReadDir reads the directory named by dirname and returns +// a list of sorted directory entries. +func (a Afero) ReadDir(dirname string) ([]os.FileInfo, error) { + return ReadDir(a.Fs, dirname) +} + +func ReadDir(fs Fs, dirname string) ([]os.FileInfo, error) { + f, err := fs.Open(dirname) + if err != nil { + return nil, err + } + list, err := f.Readdir(-1) + f.Close() + if err != nil { + return nil, err + } + sort.Sort(byName(list)) + return list, nil +} + +// ReadFile reads the file named by filename and returns the contents. +// A successful call returns err == nil, not err == EOF. Because ReadFile +// reads the whole file, it does not treat an EOF from Read as an error +// to be reported. +func (a Afero) ReadFile(filename string) ([]byte, error) { + return ReadFile(a.Fs, filename) +} + +func ReadFile(fs Fs, filename string) ([]byte, error) { + f, err := fs.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + // It's a good but not certain bet that FileInfo will tell us exactly how much to + // read, so let's try it but be prepared for the answer to be wrong. + var n int64 + + if fi, err := f.Stat(); err == nil { + // Don't preallocate a huge buffer, just in case. + if size := fi.Size(); size < 1e9 { + n = size + } + } + // As initial capacity for readAll, use n + a little extra in case Size is zero, + // and to avoid another allocation after Read has filled the buffer. The readAll + // call will read into its allocated internal buffer cheaply. If the size was + // wrong, we'll either waste some space off the end or reallocate as needed, but + // in the overwhelmingly common case we'll get it just right. + return readAll(f, n+bytes.MinRead) +} + +// readAll reads from r until an error or EOF and returns the data it read +// from the internal buffer allocated with a specified capacity. +func readAll(r io.Reader, capacity int64) (b []byte, err error) { + buf := bytes.NewBuffer(make([]byte, 0, capacity)) + // If the buffer overflows, we will get bytes.ErrTooLarge. + // Return that as an error. Any other panic remains. + defer func() { + e := recover() + if e == nil { + return + } + if panicErr, ok := e.(error); ok && panicErr == bytes.ErrTooLarge { + err = panicErr + } else { + panic(e) + } + }() + _, err = buf.ReadFrom(r) + return buf.Bytes(), err +} + +// ReadAll reads from r until an error or EOF and returns the data it read. +// A successful call returns err == nil, not err == EOF. Because ReadAll is +// defined to read from src until EOF, it does not treat an EOF from Read +// as an error to be reported. +func ReadAll(r io.Reader) ([]byte, error) { + return readAll(r, bytes.MinRead) +} + +// WriteFile writes data to a file named by filename. +// If the file does not exist, WriteFile creates it with permissions perm; +// otherwise WriteFile truncates it before writing. +func (a Afero) WriteFile(filename string, data []byte, perm os.FileMode) error { + return WriteFile(a.Fs, filename, data, perm) +} + +func WriteFile(fs Fs, filename string, data []byte, perm os.FileMode) error { + f, err := fs.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + n, err := f.Write(data) + if err == nil && n < len(data) { + err = io.ErrShortWrite + } + if err1 := f.Close(); err == nil { + err = err1 + } + return err +} + +// Random number state. +// We generate random temporary file names so that there's a good +// chance the file doesn't exist yet - keeps the number of tries in +// TempFile to a minimum. +var rand uint32 +var randmu sync.Mutex + +func reseed() uint32 { + return uint32(time.Now().UnixNano() + int64(os.Getpid())) +} + +func nextSuffix() string { + randmu.Lock() + r := rand + if r == 0 { + r = reseed() + } + r = r*1664525 + 1013904223 // constants from Numerical Recipes + rand = r + randmu.Unlock() + return strconv.Itoa(int(1e9 + r%1e9))[1:] +} + +// TempFile creates a new temporary file in the directory dir +// with a name beginning with prefix, opens the file for reading +// and writing, and returns the resulting *File. +// If dir is the empty string, TempFile uses the default directory +// for temporary files (see os.TempDir). +// Multiple programs calling TempFile simultaneously +// will not choose the same file. The caller can use f.Name() +// to find the pathname of the file. It is the caller's responsibility +// to remove the file when no longer needed. +func (a Afero) TempFile(dir, prefix string) (f File, err error) { + return TempFile(a.Fs, dir, prefix) +} + +func TempFile(fs Fs, dir, prefix string) (f File, err error) { + if dir == "" { + dir = os.TempDir() + } + + nconflict := 0 + for i := 0; i < 10000; i++ { + name := filepath.Join(dir, prefix+nextSuffix()) + f, err = fs.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + if os.IsExist(err) { + if nconflict++; nconflict > 10 { + randmu.Lock() + rand = reseed() + randmu.Unlock() + } + continue + } + break + } + return +} + +// TempDir creates a new temporary directory in the directory dir +// with a name beginning with prefix and returns the path of the +// new directory. If dir is the empty string, TempDir uses the +// default directory for temporary files (see os.TempDir). +// Multiple programs calling TempDir simultaneously +// will not choose the same directory. It is the caller's responsibility +// to remove the directory when no longer needed. +func (a Afero) TempDir(dir, prefix string) (name string, err error) { + return TempDir(a.Fs, dir, prefix) +} +func TempDir(fs Fs, dir, prefix string) (name string, err error) { + if dir == "" { + dir = os.TempDir() + } + + nconflict := 0 + for i := 0; i < 10000; i++ { + try := filepath.Join(dir, prefix+nextSuffix()) + err = fs.Mkdir(try, 0700) + if os.IsExist(err) { + if nconflict++; nconflict > 10 { + randmu.Lock() + rand = reseed() + randmu.Unlock() + } + continue + } + if err == nil { + name = try + } + break + } + return +} diff --git a/vendor/github.com/spf13/afero/match.go b/vendor/github.com/spf13/afero/match.go new file mode 100644 index 000000000..08b3b7e01 --- /dev/null +++ b/vendor/github.com/spf13/afero/match.go @@ -0,0 +1,110 @@ +// Copyright © 2014 Steve Francia . +// Copyright 2009 The Go Authors. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "path/filepath" + "sort" + "strings" +) + +// Glob returns the names of all files matching pattern or nil +// if there is no matching file. The syntax of patterns is the same +// as in Match. The pattern may describe hierarchical names such as +// /usr/*/bin/ed (assuming the Separator is '/'). +// +// Glob ignores file system errors such as I/O errors reading directories. +// The only possible returned error is ErrBadPattern, when pattern +// is malformed. +// +// This was adapted from (http://golang.org/pkg/path/filepath) and uses several +// built-ins from that package. +func Glob(fs Fs, pattern string) (matches []string, err error) { + if !hasMeta(pattern) { + // afero does not support Lstat directly. + if _, err = lstatIfOs(fs, pattern); err != nil { + return nil, nil + } + return []string{pattern}, nil + } + + dir, file := filepath.Split(pattern) + switch dir { + case "": + dir = "." + case string(filepath.Separator): + // nothing + default: + dir = dir[0 : len(dir)-1] // chop off trailing separator + } + + if !hasMeta(dir) { + return glob(fs, dir, file, nil) + } + + var m []string + m, err = Glob(fs, dir) + if err != nil { + return + } + for _, d := range m { + matches, err = glob(fs, d, file, matches) + if err != nil { + return + } + } + return +} + +// glob searches for files matching pattern in the directory dir +// and appends them to matches. If the directory cannot be +// opened, it returns the existing matches. New matches are +// added in lexicographical order. +func glob(fs Fs, dir, pattern string, matches []string) (m []string, e error) { + m = matches + fi, err := fs.Stat(dir) + if err != nil { + return + } + if !fi.IsDir() { + return + } + d, err := fs.Open(dir) + if err != nil { + return + } + defer d.Close() + + names, _ := d.Readdirnames(-1) + sort.Strings(names) + + for _, n := range names { + matched, err := filepath.Match(pattern, n) + if err != nil { + return m, err + } + if matched { + m = append(m, filepath.Join(dir, n)) + } + } + return +} + +// hasMeta reports whether path contains any of the magic characters +// recognized by Match. +func hasMeta(path string) bool { + // TODO(niemeyer): Should other magic characters be added here? + return strings.IndexAny(path, "*?[") >= 0 +} diff --git a/vendor/github.com/spf13/afero/mem/dir.go b/vendor/github.com/spf13/afero/mem/dir.go new file mode 100644 index 000000000..e104013f4 --- /dev/null +++ b/vendor/github.com/spf13/afero/mem/dir.go @@ -0,0 +1,37 @@ +// Copyright © 2014 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mem + +type Dir interface { + Len() int + Names() []string + Files() []*FileData + Add(*FileData) + Remove(*FileData) +} + +func RemoveFromMemDir(dir *FileData, f *FileData) { + dir.memDir.Remove(f) +} + +func AddToMemDir(dir *FileData, f *FileData) { + dir.memDir.Add(f) +} + +func InitializeDir(d *FileData) { + if d.memDir == nil { + d.dir = true + d.memDir = &DirMap{} + } +} diff --git a/vendor/github.com/spf13/afero/mem/dirmap.go b/vendor/github.com/spf13/afero/mem/dirmap.go new file mode 100644 index 000000000..03a57ee5b --- /dev/null +++ b/vendor/github.com/spf13/afero/mem/dirmap.go @@ -0,0 +1,43 @@ +// Copyright © 2015 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mem + +import "sort" + +type DirMap map[string]*FileData + +func (m DirMap) Len() int { return len(m) } +func (m DirMap) Add(f *FileData) { m[f.name] = f } +func (m DirMap) Remove(f *FileData) { delete(m, f.name) } +func (m DirMap) Files() (files []*FileData) { + for _, f := range m { + files = append(files, f) + } + sort.Sort(filesSorter(files)) + return files +} + +// implement sort.Interface for []*FileData +type filesSorter []*FileData + +func (s filesSorter) Len() int { return len(s) } +func (s filesSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s filesSorter) Less(i, j int) bool { return s[i].name < s[j].name } + +func (m DirMap) Names() (names []string) { + for x := range m { + names = append(names, x) + } + return names +} diff --git a/vendor/github.com/spf13/afero/mem/file.go b/vendor/github.com/spf13/afero/mem/file.go new file mode 100644 index 000000000..885e55429 --- /dev/null +++ b/vendor/github.com/spf13/afero/mem/file.go @@ -0,0 +1,314 @@ +// Copyright © 2015 Steve Francia . +// Copyright 2013 tsuru authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mem + +import ( + "bytes" + "errors" + "io" + "os" + "path/filepath" + "sync" + "sync/atomic" +) + +import "time" + +const FilePathSeparator = string(filepath.Separator) + +type File struct { + // atomic requires 64-bit alignment for struct field access + at int64 + readDirCount int64 + closed bool + readOnly bool + fileData *FileData +} + +func NewFileHandle(data *FileData) *File { + return &File{fileData: data} +} + +func NewReadOnlyFileHandle(data *FileData) *File { + return &File{fileData: data, readOnly: true} +} + +func (f File) Data() *FileData { + return f.fileData +} + +type FileData struct { + sync.Mutex + name string + data []byte + memDir Dir + dir bool + mode os.FileMode + modtime time.Time +} + +func (d *FileData) Name() string { + d.Lock() + defer d.Unlock() + return d.name +} + +func CreateFile(name string) *FileData { + return &FileData{name: name, mode: os.ModeTemporary, modtime: time.Now()} +} + +func CreateDir(name string) *FileData { + return &FileData{name: name, memDir: &DirMap{}, dir: true} +} + +func ChangeFileName(f *FileData, newname string) { + f.Lock() + f.name = newname + f.Unlock() +} + +func SetMode(f *FileData, mode os.FileMode) { + f.Lock() + f.mode = mode + f.Unlock() +} + +func SetModTime(f *FileData, mtime time.Time) { + f.Lock() + setModTime(f, mtime) + f.Unlock() +} + +func setModTime(f *FileData, mtime time.Time) { + f.modtime = mtime +} + +func GetFileInfo(f *FileData) *FileInfo { + return &FileInfo{f} +} + +func (f *File) Open() error { + atomic.StoreInt64(&f.at, 0) + atomic.StoreInt64(&f.readDirCount, 0) + f.fileData.Lock() + f.closed = false + f.fileData.Unlock() + return nil +} + +func (f *File) Close() error { + f.fileData.Lock() + f.closed = true + if !f.readOnly { + setModTime(f.fileData, time.Now()) + } + f.fileData.Unlock() + return nil +} + +func (f *File) Name() string { + return f.fileData.Name() +} + +func (f *File) Stat() (os.FileInfo, error) { + return &FileInfo{f.fileData}, nil +} + +func (f *File) Sync() error { + return nil +} + +func (f *File) Readdir(count int) (res []os.FileInfo, err error) { + var outLength int64 + + f.fileData.Lock() + files := f.fileData.memDir.Files()[f.readDirCount:] + if count > 0 { + if len(files) < count { + outLength = int64(len(files)) + } else { + outLength = int64(count) + } + if len(files) == 0 { + err = io.EOF + } + } else { + outLength = int64(len(files)) + } + f.readDirCount += outLength + f.fileData.Unlock() + + res = make([]os.FileInfo, outLength) + for i := range res { + res[i] = &FileInfo{files[i]} + } + + return res, err +} + +func (f *File) Readdirnames(n int) (names []string, err error) { + fi, err := f.Readdir(n) + names = make([]string, len(fi)) + for i, f := range fi { + _, names[i] = filepath.Split(f.Name()) + } + return names, err +} + +func (f *File) Read(b []byte) (n int, err error) { + f.fileData.Lock() + defer f.fileData.Unlock() + if f.closed == true { + return 0, ErrFileClosed + } + if len(b) > 0 && int(f.at) == len(f.fileData.data) { + return 0, io.EOF + } + if int(f.at) > len(f.fileData.data) { + return 0, io.ErrUnexpectedEOF + } + if len(f.fileData.data)-int(f.at) >= len(b) { + n = len(b) + } else { + n = len(f.fileData.data) - int(f.at) + } + copy(b, f.fileData.data[f.at:f.at+int64(n)]) + atomic.AddInt64(&f.at, int64(n)) + return +} + +func (f *File) ReadAt(b []byte, off int64) (n int, err error) { + atomic.StoreInt64(&f.at, off) + return f.Read(b) +} + +func (f *File) Truncate(size int64) error { + if f.closed == true { + return ErrFileClosed + } + if f.readOnly { + return &os.PathError{Op: "truncate", Path: f.fileData.name, Err: errors.New("file handle is read only")} + } + if size < 0 { + return ErrOutOfRange + } + if size > int64(len(f.fileData.data)) { + diff := size - int64(len(f.fileData.data)) + f.fileData.data = append(f.fileData.data, bytes.Repeat([]byte{00}, int(diff))...) + } else { + f.fileData.data = f.fileData.data[0:size] + } + setModTime(f.fileData, time.Now()) + return nil +} + +func (f *File) Seek(offset int64, whence int) (int64, error) { + if f.closed == true { + return 0, ErrFileClosed + } + switch whence { + case 0: + atomic.StoreInt64(&f.at, offset) + case 1: + atomic.AddInt64(&f.at, int64(offset)) + case 2: + atomic.StoreInt64(&f.at, int64(len(f.fileData.data))+offset) + } + return f.at, nil +} + +func (f *File) Write(b []byte) (n int, err error) { + if f.readOnly { + return 0, &os.PathError{Op: "write", Path: f.fileData.name, Err: errors.New("file handle is read only")} + } + n = len(b) + cur := atomic.LoadInt64(&f.at) + f.fileData.Lock() + defer f.fileData.Unlock() + diff := cur - int64(len(f.fileData.data)) + var tail []byte + if n+int(cur) < len(f.fileData.data) { + tail = f.fileData.data[n+int(cur):] + } + if diff > 0 { + f.fileData.data = append(bytes.Repeat([]byte{00}, int(diff)), b...) + f.fileData.data = append(f.fileData.data, tail...) + } else { + f.fileData.data = append(f.fileData.data[:cur], b...) + f.fileData.data = append(f.fileData.data, tail...) + } + setModTime(f.fileData, time.Now()) + + atomic.StoreInt64(&f.at, int64(len(f.fileData.data))) + return +} + +func (f *File) WriteAt(b []byte, off int64) (n int, err error) { + atomic.StoreInt64(&f.at, off) + return f.Write(b) +} + +func (f *File) WriteString(s string) (ret int, err error) { + return f.Write([]byte(s)) +} + +func (f *File) Info() *FileInfo { + return &FileInfo{f.fileData} +} + +type FileInfo struct { + *FileData +} + +// Implements os.FileInfo +func (s *FileInfo) Name() string { + s.Lock() + _, name := filepath.Split(s.name) + s.Unlock() + return name +} +func (s *FileInfo) Mode() os.FileMode { + s.Lock() + defer s.Unlock() + return s.mode +} +func (s *FileInfo) ModTime() time.Time { + s.Lock() + defer s.Unlock() + return s.modtime +} +func (s *FileInfo) IsDir() bool { + s.Lock() + defer s.Unlock() + return s.dir +} +func (s *FileInfo) Sys() interface{} { return nil } +func (s *FileInfo) Size() int64 { + if s.IsDir() { + return int64(42) + } + s.Lock() + defer s.Unlock() + return int64(len(s.data)) +} + +var ( + ErrFileClosed = errors.New("File is closed") + ErrOutOfRange = errors.New("Out of range") + ErrTooLarge = errors.New("Too large") + ErrFileNotFound = os.ErrNotExist + ErrFileExists = os.ErrExist + ErrDestinationExists = os.ErrExist +) diff --git a/vendor/github.com/spf13/afero/memmap.go b/vendor/github.com/spf13/afero/memmap.go new file mode 100644 index 000000000..09498e70f --- /dev/null +++ b/vendor/github.com/spf13/afero/memmap.go @@ -0,0 +1,365 @@ +// Copyright © 2014 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "fmt" + "log" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/spf13/afero/mem" +) + +type MemMapFs struct { + mu sync.RWMutex + data map[string]*mem.FileData + init sync.Once +} + +func NewMemMapFs() Fs { + return &MemMapFs{} +} + +func (m *MemMapFs) getData() map[string]*mem.FileData { + m.init.Do(func() { + m.data = make(map[string]*mem.FileData) + // Root should always exist, right? + // TODO: what about windows? + m.data[FilePathSeparator] = mem.CreateDir(FilePathSeparator) + }) + return m.data +} + +func (*MemMapFs) Name() string { return "MemMapFS" } + +func (m *MemMapFs) Create(name string) (File, error) { + name = normalizePath(name) + m.mu.Lock() + file := mem.CreateFile(name) + m.getData()[name] = file + m.registerWithParent(file) + m.mu.Unlock() + return mem.NewFileHandle(file), nil +} + +func (m *MemMapFs) unRegisterWithParent(fileName string) error { + f, err := m.lockfreeOpen(fileName) + if err != nil { + return err + } + parent := m.findParent(f) + if parent == nil { + log.Panic("parent of ", f.Name(), " is nil") + } + + parent.Lock() + mem.RemoveFromMemDir(parent, f) + parent.Unlock() + return nil +} + +func (m *MemMapFs) findParent(f *mem.FileData) *mem.FileData { + pdir, _ := filepath.Split(f.Name()) + pdir = filepath.Clean(pdir) + pfile, err := m.lockfreeOpen(pdir) + if err != nil { + return nil + } + return pfile +} + +func (m *MemMapFs) registerWithParent(f *mem.FileData) { + if f == nil { + return + } + parent := m.findParent(f) + if parent == nil { + pdir := filepath.Dir(filepath.Clean(f.Name())) + err := m.lockfreeMkdir(pdir, 0777) + if err != nil { + //log.Println("Mkdir error:", err) + return + } + parent, err = m.lockfreeOpen(pdir) + if err != nil { + //log.Println("Open after Mkdir error:", err) + return + } + } + + parent.Lock() + mem.InitializeDir(parent) + mem.AddToMemDir(parent, f) + parent.Unlock() +} + +func (m *MemMapFs) lockfreeMkdir(name string, perm os.FileMode) error { + name = normalizePath(name) + x, ok := m.getData()[name] + if ok { + // Only return ErrFileExists if it's a file, not a directory. + i := mem.FileInfo{FileData: x} + if !i.IsDir() { + return ErrFileExists + } + } else { + item := mem.CreateDir(name) + m.getData()[name] = item + m.registerWithParent(item) + } + return nil +} + +func (m *MemMapFs) Mkdir(name string, perm os.FileMode) error { + name = normalizePath(name) + + m.mu.RLock() + _, ok := m.getData()[name] + m.mu.RUnlock() + if ok { + return &os.PathError{Op: "mkdir", Path: name, Err: ErrFileExists} + } + + m.mu.Lock() + item := mem.CreateDir(name) + m.getData()[name] = item + m.registerWithParent(item) + m.mu.Unlock() + + m.Chmod(name, perm|os.ModeDir) + + return nil +} + +func (m *MemMapFs) MkdirAll(path string, perm os.FileMode) error { + err := m.Mkdir(path, perm) + if err != nil { + if err.(*os.PathError).Err == ErrFileExists { + return nil + } + return err + } + return nil +} + +// Handle some relative paths +func normalizePath(path string) string { + path = filepath.Clean(path) + + switch path { + case ".": + return FilePathSeparator + case "..": + return FilePathSeparator + default: + return path + } +} + +func (m *MemMapFs) Open(name string) (File, error) { + f, err := m.open(name) + if f != nil { + return mem.NewReadOnlyFileHandle(f), err + } + return nil, err +} + +func (m *MemMapFs) openWrite(name string) (File, error) { + f, err := m.open(name) + if f != nil { + return mem.NewFileHandle(f), err + } + return nil, err +} + +func (m *MemMapFs) open(name string) (*mem.FileData, error) { + name = normalizePath(name) + + m.mu.RLock() + f, ok := m.getData()[name] + m.mu.RUnlock() + if !ok { + return nil, &os.PathError{Op: "open", Path: name, Err: ErrFileNotFound} + } + return f, nil +} + +func (m *MemMapFs) lockfreeOpen(name string) (*mem.FileData, error) { + name = normalizePath(name) + f, ok := m.getData()[name] + if ok { + return f, nil + } else { + return nil, ErrFileNotFound + } +} + +func (m *MemMapFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + chmod := false + file, err := m.openWrite(name) + if os.IsNotExist(err) && (flag&os.O_CREATE > 0) { + file, err = m.Create(name) + chmod = true + } + if err != nil { + return nil, err + } + if flag == os.O_RDONLY { + file = mem.NewReadOnlyFileHandle(file.(*mem.File).Data()) + } + if flag&os.O_APPEND > 0 { + _, err = file.Seek(0, os.SEEK_END) + if err != nil { + file.Close() + return nil, err + } + } + if flag&os.O_TRUNC > 0 && flag&(os.O_RDWR|os.O_WRONLY) > 0 { + err = file.Truncate(0) + if err != nil { + file.Close() + return nil, err + } + } + if chmod { + m.Chmod(name, perm) + } + return file, nil +} + +func (m *MemMapFs) Remove(name string) error { + name = normalizePath(name) + + m.mu.Lock() + defer m.mu.Unlock() + + if _, ok := m.getData()[name]; ok { + err := m.unRegisterWithParent(name) + if err != nil { + return &os.PathError{Op: "remove", Path: name, Err: err} + } + delete(m.getData(), name) + } else { + return &os.PathError{Op: "remove", Path: name, Err: os.ErrNotExist} + } + return nil +} + +func (m *MemMapFs) RemoveAll(path string) error { + path = normalizePath(path) + m.mu.Lock() + m.unRegisterWithParent(path) + m.mu.Unlock() + + m.mu.RLock() + defer m.mu.RUnlock() + + for p, _ := range m.getData() { + if strings.HasPrefix(p, path) { + m.mu.RUnlock() + m.mu.Lock() + delete(m.getData(), p) + m.mu.Unlock() + m.mu.RLock() + } + } + return nil +} + +func (m *MemMapFs) Rename(oldname, newname string) error { + oldname = normalizePath(oldname) + newname = normalizePath(newname) + + if oldname == newname { + return nil + } + + m.mu.RLock() + defer m.mu.RUnlock() + if _, ok := m.getData()[oldname]; ok { + m.mu.RUnlock() + m.mu.Lock() + m.unRegisterWithParent(oldname) + fileData := m.getData()[oldname] + delete(m.getData(), oldname) + mem.ChangeFileName(fileData, newname) + m.getData()[newname] = fileData + m.registerWithParent(fileData) + m.mu.Unlock() + m.mu.RLock() + } else { + return &os.PathError{Op: "rename", Path: oldname, Err: ErrFileNotFound} + } + return nil +} + +func (m *MemMapFs) Stat(name string) (os.FileInfo, error) { + f, err := m.Open(name) + if err != nil { + return nil, err + } + fi := mem.GetFileInfo(f.(*mem.File).Data()) + return fi, nil +} + +func (m *MemMapFs) Chmod(name string, mode os.FileMode) error { + name = normalizePath(name) + + m.mu.RLock() + f, ok := m.getData()[name] + m.mu.RUnlock() + if !ok { + return &os.PathError{Op: "chmod", Path: name, Err: ErrFileNotFound} + } + + m.mu.Lock() + mem.SetMode(f, mode) + m.mu.Unlock() + + return nil +} + +func (m *MemMapFs) Chtimes(name string, atime time.Time, mtime time.Time) error { + name = normalizePath(name) + + m.mu.RLock() + f, ok := m.getData()[name] + m.mu.RUnlock() + if !ok { + return &os.PathError{Op: "chtimes", Path: name, Err: ErrFileNotFound} + } + + m.mu.Lock() + mem.SetModTime(f, mtime) + m.mu.Unlock() + + return nil +} + +func (m *MemMapFs) List() { + for _, x := range m.data { + y := mem.FileInfo{FileData: x} + fmt.Println(x.Name(), y.Size()) + } +} + +// func debugMemMapList(fs Fs) { +// if x, ok := fs.(*MemMapFs); ok { +// x.List() +// } +// } diff --git a/vendor/github.com/spf13/afero/os.go b/vendor/github.com/spf13/afero/os.go new file mode 100644 index 000000000..6b8bce1c5 --- /dev/null +++ b/vendor/github.com/spf13/afero/os.go @@ -0,0 +1,94 @@ +// Copyright © 2014 Steve Francia . +// Copyright 2013 tsuru authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "os" + "time" +) + +// OsFs is a Fs implementation that uses functions provided by the os package. +// +// For details in any method, check the documentation of the os package +// (http://golang.org/pkg/os/). +type OsFs struct{} + +func NewOsFs() Fs { + return &OsFs{} +} + +func (OsFs) Name() string { return "OsFs" } + +func (OsFs) Create(name string) (File, error) { + f, e := os.Create(name) + if f == nil { + // while this looks strange, we need to return a bare nil (of type nil) not + // a nil value of type *os.File or nil won't be nil + return nil, e + } + return f, e +} + +func (OsFs) Mkdir(name string, perm os.FileMode) error { + return os.Mkdir(name, perm) +} + +func (OsFs) MkdirAll(path string, perm os.FileMode) error { + return os.MkdirAll(path, perm) +} + +func (OsFs) Open(name string) (File, error) { + f, e := os.Open(name) + if f == nil { + // while this looks strange, we need to return a bare nil (of type nil) not + // a nil value of type *os.File or nil won't be nil + return nil, e + } + return f, e +} + +func (OsFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + f, e := os.OpenFile(name, flag, perm) + if f == nil { + // while this looks strange, we need to return a bare nil (of type nil) not + // a nil value of type *os.File or nil won't be nil + return nil, e + } + return f, e +} + +func (OsFs) Remove(name string) error { + return os.Remove(name) +} + +func (OsFs) RemoveAll(path string) error { + return os.RemoveAll(path) +} + +func (OsFs) Rename(oldname, newname string) error { + return os.Rename(oldname, newname) +} + +func (OsFs) Stat(name string) (os.FileInfo, error) { + return os.Stat(name) +} + +func (OsFs) Chmod(name string, mode os.FileMode) error { + return os.Chmod(name, mode) +} + +func (OsFs) Chtimes(name string, atime time.Time, mtime time.Time) error { + return os.Chtimes(name, atime, mtime) +} diff --git a/vendor/github.com/spf13/afero/path.go b/vendor/github.com/spf13/afero/path.go new file mode 100644 index 000000000..1d90e46dd --- /dev/null +++ b/vendor/github.com/spf13/afero/path.go @@ -0,0 +1,108 @@ +// Copyright ©2015 The Go Authors +// Copyright ©2015 Steve Francia +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "os" + "path/filepath" + "sort" +) + +// readDirNames reads the directory named by dirname and returns +// a sorted list of directory entries. +// adapted from https://golang.org/src/path/filepath/path.go +func readDirNames(fs Fs, dirname string) ([]string, error) { + f, err := fs.Open(dirname) + if err != nil { + return nil, err + } + names, err := f.Readdirnames(-1) + f.Close() + if err != nil { + return nil, err + } + sort.Strings(names) + return names, nil +} + +// walk recursively descends path, calling walkFn +// adapted from https://golang.org/src/path/filepath/path.go +func walk(fs Fs, path string, info os.FileInfo, walkFn filepath.WalkFunc) error { + err := walkFn(path, info, nil) + if err != nil { + if info.IsDir() && err == filepath.SkipDir { + return nil + } + return err + } + + if !info.IsDir() { + return nil + } + + names, err := readDirNames(fs, path) + if err != nil { + return walkFn(path, info, err) + } + + for _, name := range names { + filename := filepath.Join(path, name) + fileInfo, err := lstatIfOs(fs, filename) + if err != nil { + if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { + return err + } + } else { + err = walk(fs, filename, fileInfo, walkFn) + if err != nil { + if !fileInfo.IsDir() || err != filepath.SkipDir { + return err + } + } + } + } + return nil +} + +// if the filesystem is OsFs use Lstat, else use fs.Stat +func lstatIfOs(fs Fs, path string) (info os.FileInfo, err error) { + _, ok := fs.(*OsFs) + if ok { + info, err = os.Lstat(path) + } else { + info, err = fs.Stat(path) + } + return +} + +// Walk walks the file tree rooted at root, calling walkFn for each file or +// directory in the tree, including root. All errors that arise visiting files +// and directories are filtered by walkFn. The files are walked in lexical +// order, which makes the output deterministic but means that for very +// large directories Walk can be inefficient. +// Walk does not follow symbolic links. + +func (a Afero) Walk(root string, walkFn filepath.WalkFunc) error { + return Walk(a.Fs, root, walkFn) +} + +func Walk(fs Fs, root string, walkFn filepath.WalkFunc) error { + info, err := lstatIfOs(fs, root) + if err != nil { + return walkFn(root, nil, err) + } + return walk(fs, root, info, walkFn) +} diff --git a/vendor/github.com/spf13/afero/readonlyfs.go b/vendor/github.com/spf13/afero/readonlyfs.go new file mode 100644 index 000000000..f1fa55bcf --- /dev/null +++ b/vendor/github.com/spf13/afero/readonlyfs.go @@ -0,0 +1,70 @@ +package afero + +import ( + "os" + "syscall" + "time" +) + +type ReadOnlyFs struct { + source Fs +} + +func NewReadOnlyFs(source Fs) Fs { + return &ReadOnlyFs{source: source} +} + +func (r *ReadOnlyFs) ReadDir(name string) ([]os.FileInfo, error) { + return ReadDir(r.source, name) +} + +func (r *ReadOnlyFs) Chtimes(n string, a, m time.Time) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) Chmod(n string, m os.FileMode) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) Name() string { + return "ReadOnlyFilter" +} + +func (r *ReadOnlyFs) Stat(name string) (os.FileInfo, error) { + return r.source.Stat(name) +} + +func (r *ReadOnlyFs) Rename(o, n string) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) RemoveAll(p string) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) Remove(n string) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + if flag&(os.O_WRONLY|syscall.O_RDWR|os.O_APPEND|os.O_CREATE|os.O_TRUNC) != 0 { + return nil, syscall.EPERM + } + return r.source.OpenFile(name, flag, perm) +} + +func (r *ReadOnlyFs) Open(n string) (File, error) { + return r.source.Open(n) +} + +func (r *ReadOnlyFs) Mkdir(n string, p os.FileMode) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) MkdirAll(n string, p os.FileMode) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) Create(n string) (File, error) { + return nil, syscall.EPERM +} diff --git a/vendor/github.com/spf13/afero/regexpfs.go b/vendor/github.com/spf13/afero/regexpfs.go new file mode 100644 index 000000000..9d92dbc05 --- /dev/null +++ b/vendor/github.com/spf13/afero/regexpfs.go @@ -0,0 +1,214 @@ +package afero + +import ( + "os" + "regexp" + "syscall" + "time" +) + +// The RegexpFs filters files (not directories) by regular expression. Only +// files matching the given regexp will be allowed, all others get a ENOENT error ( +// "No such file or directory"). +// +type RegexpFs struct { + re *regexp.Regexp + source Fs +} + +func NewRegexpFs(source Fs, re *regexp.Regexp) Fs { + return &RegexpFs{source: source, re: re} +} + +type RegexpFile struct { + f File + re *regexp.Regexp +} + +func (r *RegexpFs) matchesName(name string) error { + if r.re == nil { + return nil + } + if r.re.MatchString(name) { + return nil + } + return syscall.ENOENT +} + +func (r *RegexpFs) dirOrMatches(name string) error { + dir, err := IsDir(r.source, name) + if err != nil { + return err + } + if dir { + return nil + } + return r.matchesName(name) +} + +func (r *RegexpFs) Chtimes(name string, a, m time.Time) error { + if err := r.dirOrMatches(name); err != nil { + return err + } + return r.source.Chtimes(name, a, m) +} + +func (r *RegexpFs) Chmod(name string, mode os.FileMode) error { + if err := r.dirOrMatches(name); err != nil { + return err + } + return r.source.Chmod(name, mode) +} + +func (r *RegexpFs) Name() string { + return "RegexpFs" +} + +func (r *RegexpFs) Stat(name string) (os.FileInfo, error) { + if err := r.dirOrMatches(name); err != nil { + return nil, err + } + return r.source.Stat(name) +} + +func (r *RegexpFs) Rename(oldname, newname string) error { + dir, err := IsDir(r.source, oldname) + if err != nil { + return err + } + if dir { + return nil + } + if err := r.matchesName(oldname); err != nil { + return err + } + if err := r.matchesName(newname); err != nil { + return err + } + return r.source.Rename(oldname, newname) +} + +func (r *RegexpFs) RemoveAll(p string) error { + dir, err := IsDir(r.source, p) + if err != nil { + return err + } + if !dir { + if err := r.matchesName(p); err != nil { + return err + } + } + return r.source.RemoveAll(p) +} + +func (r *RegexpFs) Remove(name string) error { + if err := r.dirOrMatches(name); err != nil { + return err + } + return r.source.Remove(name) +} + +func (r *RegexpFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + if err := r.dirOrMatches(name); err != nil { + return nil, err + } + return r.source.OpenFile(name, flag, perm) +} + +func (r *RegexpFs) Open(name string) (File, error) { + dir, err := IsDir(r.source, name) + if err != nil { + return nil, err + } + if !dir { + if err := r.matchesName(name); err != nil { + return nil, err + } + } + f, err := r.source.Open(name) + return &RegexpFile{f: f, re: r.re}, nil +} + +func (r *RegexpFs) Mkdir(n string, p os.FileMode) error { + return r.source.Mkdir(n, p) +} + +func (r *RegexpFs) MkdirAll(n string, p os.FileMode) error { + return r.source.MkdirAll(n, p) +} + +func (r *RegexpFs) Create(name string) (File, error) { + if err := r.matchesName(name); err != nil { + return nil, err + } + return r.source.Create(name) +} + +func (f *RegexpFile) Close() error { + return f.f.Close() +} + +func (f *RegexpFile) Read(s []byte) (int, error) { + return f.f.Read(s) +} + +func (f *RegexpFile) ReadAt(s []byte, o int64) (int, error) { + return f.f.ReadAt(s, o) +} + +func (f *RegexpFile) Seek(o int64, w int) (int64, error) { + return f.f.Seek(o, w) +} + +func (f *RegexpFile) Write(s []byte) (int, error) { + return f.f.Write(s) +} + +func (f *RegexpFile) WriteAt(s []byte, o int64) (int, error) { + return f.f.WriteAt(s, o) +} + +func (f *RegexpFile) Name() string { + return f.f.Name() +} + +func (f *RegexpFile) Readdir(c int) (fi []os.FileInfo, err error) { + var rfi []os.FileInfo + rfi, err = f.f.Readdir(c) + if err != nil { + return nil, err + } + for _, i := range rfi { + if i.IsDir() || f.re.MatchString(i.Name()) { + fi = append(fi, i) + } + } + return fi, nil +} + +func (f *RegexpFile) Readdirnames(c int) (n []string, err error) { + fi, err := f.Readdir(c) + if err != nil { + return nil, err + } + for _, s := range fi { + n = append(n, s.Name()) + } + return n, nil +} + +func (f *RegexpFile) Stat() (os.FileInfo, error) { + return f.f.Stat() +} + +func (f *RegexpFile) Sync() error { + return f.f.Sync() +} + +func (f *RegexpFile) Truncate(s int64) error { + return f.f.Truncate(s) +} + +func (f *RegexpFile) WriteString(s string) (int, error) { + return f.f.WriteString(s) +} diff --git a/vendor/github.com/spf13/afero/unionFile.go b/vendor/github.com/spf13/afero/unionFile.go new file mode 100644 index 000000000..99f9e5db2 --- /dev/null +++ b/vendor/github.com/spf13/afero/unionFile.go @@ -0,0 +1,274 @@ +package afero + +import ( + "io" + "os" + "path/filepath" + "syscall" +) + +// The UnionFile implements the afero.File interface and will be returned +// when reading a directory present at least in the overlay or opening a file +// for writing. +// +// The calls to +// Readdir() and Readdirnames() merge the file os.FileInfo / names from the +// base and the overlay - for files present in both layers, only those +// from the overlay will be used. +// +// When opening files for writing (Create() / OpenFile() with the right flags) +// the operations will be done in both layers, starting with the overlay. A +// successful read in the overlay will move the cursor position in the base layer +// by the number of bytes read. +type UnionFile struct { + base File + layer File + off int + files []os.FileInfo +} + +func (f *UnionFile) Close() error { + // first close base, so we have a newer timestamp in the overlay. If we'd close + // the overlay first, we'd get a cacheStale the next time we access this file + // -> cache would be useless ;-) + if f.base != nil { + f.base.Close() + } + if f.layer != nil { + return f.layer.Close() + } + return BADFD +} + +func (f *UnionFile) Read(s []byte) (int, error) { + if f.layer != nil { + n, err := f.layer.Read(s) + if (err == nil || err == io.EOF) && f.base != nil { + // advance the file position also in the base file, the next + // call may be a write at this position (or a seek with SEEK_CUR) + if _, seekErr := f.base.Seek(int64(n), os.SEEK_CUR); seekErr != nil { + // only overwrite err in case the seek fails: we need to + // report an eventual io.EOF to the caller + err = seekErr + } + } + return n, err + } + if f.base != nil { + return f.base.Read(s) + } + return 0, BADFD +} + +func (f *UnionFile) ReadAt(s []byte, o int64) (int, error) { + if f.layer != nil { + n, err := f.layer.ReadAt(s, o) + if (err == nil || err == io.EOF) && f.base != nil { + _, err = f.base.Seek(o+int64(n), os.SEEK_SET) + } + return n, err + } + if f.base != nil { + return f.base.ReadAt(s, o) + } + return 0, BADFD +} + +func (f *UnionFile) Seek(o int64, w int) (pos int64, err error) { + if f.layer != nil { + pos, err = f.layer.Seek(o, w) + if (err == nil || err == io.EOF) && f.base != nil { + _, err = f.base.Seek(o, w) + } + return pos, err + } + if f.base != nil { + return f.base.Seek(o, w) + } + return 0, BADFD +} + +func (f *UnionFile) Write(s []byte) (n int, err error) { + if f.layer != nil { + n, err = f.layer.Write(s) + if err == nil && f.base != nil { // hmm, do we have fixed size files where a write may hit the EOF mark? + _, err = f.base.Write(s) + } + return n, err + } + if f.base != nil { + return f.base.Write(s) + } + return 0, BADFD +} + +func (f *UnionFile) WriteAt(s []byte, o int64) (n int, err error) { + if f.layer != nil { + n, err = f.layer.WriteAt(s, o) + if err == nil && f.base != nil { + _, err = f.base.WriteAt(s, o) + } + return n, err + } + if f.base != nil { + return f.base.WriteAt(s, o) + } + return 0, BADFD +} + +func (f *UnionFile) Name() string { + if f.layer != nil { + return f.layer.Name() + } + return f.base.Name() +} + +// Readdir will weave the two directories together and +// return a single view of the overlayed directories +func (f *UnionFile) Readdir(c int) (ofi []os.FileInfo, err error) { + if f.off == 0 { + var files = make(map[string]os.FileInfo) + var rfi []os.FileInfo + if f.layer != nil { + rfi, err = f.layer.Readdir(-1) + if err != nil { + return nil, err + } + for _, fi := range rfi { + files[fi.Name()] = fi + } + } + + if f.base != nil { + rfi, err = f.base.Readdir(-1) + if err != nil { + return nil, err + } + for _, fi := range rfi { + if _, exists := files[fi.Name()]; !exists { + files[fi.Name()] = fi + } + } + } + for _, fi := range files { + f.files = append(f.files, fi) + } + } + if c == -1 { + return f.files[f.off:], nil + } + defer func() { f.off += c }() + return f.files[f.off:c], nil +} + +func (f *UnionFile) Readdirnames(c int) ([]string, error) { + rfi, err := f.Readdir(c) + if err != nil { + return nil, err + } + var names []string + for _, fi := range rfi { + names = append(names, fi.Name()) + } + return names, nil +} + +func (f *UnionFile) Stat() (os.FileInfo, error) { + if f.layer != nil { + return f.layer.Stat() + } + if f.base != nil { + return f.base.Stat() + } + return nil, BADFD +} + +func (f *UnionFile) Sync() (err error) { + if f.layer != nil { + err = f.layer.Sync() + if err == nil && f.base != nil { + err = f.base.Sync() + } + return err + } + if f.base != nil { + return f.base.Sync() + } + return BADFD +} + +func (f *UnionFile) Truncate(s int64) (err error) { + if f.layer != nil { + err = f.layer.Truncate(s) + if err == nil && f.base != nil { + err = f.base.Truncate(s) + } + return err + } + if f.base != nil { + return f.base.Truncate(s) + } + return BADFD +} + +func (f *UnionFile) WriteString(s string) (n int, err error) { + if f.layer != nil { + n, err = f.layer.WriteString(s) + if err == nil && f.base != nil { + _, err = f.base.WriteString(s) + } + return n, err + } + if f.base != nil { + return f.base.WriteString(s) + } + return 0, BADFD +} + +func copyToLayer(base Fs, layer Fs, name string) error { + bfh, err := base.Open(name) + if err != nil { + return err + } + defer bfh.Close() + + // First make sure the directory exists + exists, err := Exists(layer, filepath.Dir(name)) + if err != nil { + return err + } + if !exists { + err = layer.MkdirAll(filepath.Dir(name), 0777) // FIXME? + if err != nil { + return err + } + } + + // Create the file on the overlay + lfh, err := layer.Create(name) + if err != nil { + return err + } + n, err := io.Copy(lfh, bfh) + if err != nil { + // If anything fails, clean up the file + layer.Remove(name) + lfh.Close() + return err + } + + bfi, err := bfh.Stat() + if err != nil || bfi.Size() != n { + layer.Remove(name) + lfh.Close() + return syscall.EIO + } + + err = lfh.Close() + if err != nil { + layer.Remove(name) + lfh.Close() + return err + } + return layer.Chtimes(name, bfi.ModTime(), bfi.ModTime()) +} diff --git a/vendor/github.com/spf13/afero/util.go b/vendor/github.com/spf13/afero/util.go new file mode 100644 index 000000000..4f253f481 --- /dev/null +++ b/vendor/github.com/spf13/afero/util.go @@ -0,0 +1,330 @@ +// Copyright ©2015 Steve Francia +// Portions Copyright ©2015 The Hugo Authors +// Portions Copyright 2016-present Bjørn Erik Pedersen +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "unicode" + + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" +) + +// Filepath separator defined by os.Separator. +const FilePathSeparator = string(filepath.Separator) + +// Takes a reader and a path and writes the content +func (a Afero) WriteReader(path string, r io.Reader) (err error) { + return WriteReader(a.Fs, path, r) +} + +func WriteReader(fs Fs, path string, r io.Reader) (err error) { + dir, _ := filepath.Split(path) + ospath := filepath.FromSlash(dir) + + if ospath != "" { + err = fs.MkdirAll(ospath, 0777) // rwx, rw, r + if err != nil { + if err != os.ErrExist { + return err + } + } + } + + file, err := fs.Create(path) + if err != nil { + return + } + defer file.Close() + + _, err = io.Copy(file, r) + return +} + +// Same as WriteReader but checks to see if file/directory already exists. +func (a Afero) SafeWriteReader(path string, r io.Reader) (err error) { + return SafeWriteReader(a.Fs, path, r) +} + +func SafeWriteReader(fs Fs, path string, r io.Reader) (err error) { + dir, _ := filepath.Split(path) + ospath := filepath.FromSlash(dir) + + if ospath != "" { + err = fs.MkdirAll(ospath, 0777) // rwx, rw, r + if err != nil { + return + } + } + + exists, err := Exists(fs, path) + if err != nil { + return + } + if exists { + return fmt.Errorf("%v already exists", path) + } + + file, err := fs.Create(path) + if err != nil { + return + } + defer file.Close() + + _, err = io.Copy(file, r) + return +} + +func (a Afero) GetTempDir(subPath string) string { + return GetTempDir(a.Fs, subPath) +} + +// GetTempDir returns the default temp directory with trailing slash +// if subPath is not empty then it will be created recursively with mode 777 rwx rwx rwx +func GetTempDir(fs Fs, subPath string) string { + addSlash := func(p string) string { + if FilePathSeparator != p[len(p)-1:] { + p = p + FilePathSeparator + } + return p + } + dir := addSlash(os.TempDir()) + + if subPath != "" { + // preserve windows backslash :-( + if FilePathSeparator == "\\" { + subPath = strings.Replace(subPath, "\\", "____", -1) + } + dir = dir + UnicodeSanitize((subPath)) + if FilePathSeparator == "\\" { + dir = strings.Replace(dir, "____", "\\", -1) + } + + if exists, _ := Exists(fs, dir); exists { + return addSlash(dir) + } + + err := fs.MkdirAll(dir, 0777) + if err != nil { + panic(err) + } + dir = addSlash(dir) + } + return dir +} + +// Rewrite string to remove non-standard path characters +func UnicodeSanitize(s string) string { + source := []rune(s) + target := make([]rune, 0, len(source)) + + for _, r := range source { + if unicode.IsLetter(r) || + unicode.IsDigit(r) || + unicode.IsMark(r) || + r == '.' || + r == '/' || + r == '\\' || + r == '_' || + r == '-' || + r == '%' || + r == ' ' || + r == '#' { + target = append(target, r) + } + } + + return string(target) +} + +// Transform characters with accents into plain forms. +func NeuterAccents(s string) string { + t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC) + result, _, _ := transform.String(t, string(s)) + + return result +} + +func isMn(r rune) bool { + return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks +} + +func (a Afero) FileContainsBytes(filename string, subslice []byte) (bool, error) { + return FileContainsBytes(a.Fs, filename, subslice) +} + +// Check if a file contains a specified byte slice. +func FileContainsBytes(fs Fs, filename string, subslice []byte) (bool, error) { + f, err := fs.Open(filename) + if err != nil { + return false, err + } + defer f.Close() + + return readerContainsAny(f, subslice), nil +} + +func (a Afero) FileContainsAnyBytes(filename string, subslices [][]byte) (bool, error) { + return FileContainsAnyBytes(a.Fs, filename, subslices) +} + +// Check if a file contains any of the specified byte slices. +func FileContainsAnyBytes(fs Fs, filename string, subslices [][]byte) (bool, error) { + f, err := fs.Open(filename) + if err != nil { + return false, err + } + defer f.Close() + + return readerContainsAny(f, subslices...), nil +} + +// readerContains reports whether any of the subslices is within r. +func readerContainsAny(r io.Reader, subslices ...[]byte) bool { + + if r == nil || len(subslices) == 0 { + return false + } + + largestSlice := 0 + + for _, sl := range subslices { + if len(sl) > largestSlice { + largestSlice = len(sl) + } + } + + if largestSlice == 0 { + return false + } + + bufflen := largestSlice * 4 + halflen := bufflen / 2 + buff := make([]byte, bufflen) + var err error + var n, i int + + for { + i++ + if i == 1 { + n, err = io.ReadAtLeast(r, buff[:halflen], halflen) + } else { + if i != 2 { + // shift left to catch overlapping matches + copy(buff[:], buff[halflen:]) + } + n, err = io.ReadAtLeast(r, buff[halflen:], halflen) + } + + if n > 0 { + for _, sl := range subslices { + if bytes.Contains(buff, sl) { + return true + } + } + } + + if err != nil { + break + } + } + return false +} + +func (a Afero) DirExists(path string) (bool, error) { + return DirExists(a.Fs, path) +} + +// DirExists checks if a path exists and is a directory. +func DirExists(fs Fs, path string) (bool, error) { + fi, err := fs.Stat(path) + if err == nil && fi.IsDir() { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +func (a Afero) IsDir(path string) (bool, error) { + return IsDir(a.Fs, path) +} + +// IsDir checks if a given path is a directory. +func IsDir(fs Fs, path string) (bool, error) { + fi, err := fs.Stat(path) + if err != nil { + return false, err + } + return fi.IsDir(), nil +} + +func (a Afero) IsEmpty(path string) (bool, error) { + return IsEmpty(a.Fs, path) +} + +// IsEmpty checks if a given file or directory is empty. +func IsEmpty(fs Fs, path string) (bool, error) { + if b, _ := Exists(fs, path); !b { + return false, fmt.Errorf("%q path does not exist", path) + } + fi, err := fs.Stat(path) + if err != nil { + return false, err + } + if fi.IsDir() { + f, err := fs.Open(path) + if err != nil { + return false, err + } + defer f.Close() + list, err := f.Readdir(-1) + return len(list) == 0, nil + } + return fi.Size() == 0, nil +} + +func (a Afero) Exists(path string) (bool, error) { + return Exists(a.Fs, path) +} + +// Check if a file or directory exists. +func Exists(fs Fs, path string) (bool, error) { + _, err := fs.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +func FullBaseFsPath(basePathFs *BasePathFs, relativePath string) string { + combinedPath := filepath.Join(basePathFs.path, relativePath) + if parent, ok := basePathFs.source.(*BasePathFs); ok { + return FullBaseFsPath(parent, combinedPath) + } + + return combinedPath +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 4da4a0692..0a9fdf263 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -2127,6 +2127,18 @@ "revision": "b061729afc07e77a8aa4fad0a2fd840958f1942a", "revisionTime": "2016-09-27T10:08:44Z" }, + { + "checksumSHA1": "F3JU4T4XXvZTRAcv6rhTao4QGos=", + "path": "github.com/spf13/afero", + "revision": "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c", + "revisionTime": "2018-01-15T19:27:20Z" + }, + { + "checksumSHA1": "X6RueW0rO55PbOQ0sMWSQOxVl4I=", + "path": "github.com/spf13/afero/mem", + "revision": "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c", + "revisionTime": "2018-01-15T19:27:20Z" + }, { "checksumSHA1": "5XyfTtyMIFp/q/Xz3y0LgbhCnfQ=", "path": "github.com/terraform-providers/terraform-provider-aws", From 7987a2fdb25c892b7ec7eaee3f62efd7bd05fa77 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 1 Feb 2018 19:03:54 -0800 Subject: [PATCH 02/25] configs: new package for HCL2-based configuration There's a lot of complexity in our existing "config" package that results from our approach to handling configuration with HCL and HIL. A lot of that functionality is no longer needed -- or must work in a significantly different way -- for HCL2. The new package "configs", which is named following the convention of some Go standard library packages like "strings", is a re-imagination of some of the functionality from the "config" package for an HCL2-only world. The scope of this package will be slightly smaller than "config", since it only deals with config loading and not with expression evaluation. Another package "lang" (mentioned in the docstring here but not yet added) will deal with the more dynamic portions of of configuration handling, including populating an hcl.EvalContext to evaluate expressions. --- configs/doc.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 configs/doc.go diff --git a/configs/doc.go b/configs/doc.go new file mode 100644 index 000000000..9688b36cd --- /dev/null +++ b/configs/doc.go @@ -0,0 +1,14 @@ +// Package configs contains types that represent Terraform configurations and +// the different elements thereof. +// +// The functionality in this package can be used for some static analyses of +// Terraform configurations, but this package generally exposes representations +// of the configuration source code rather than the result of evaluating these +// objects. The sibling package "lang" deals with evaluation of structures +// and expressions in the configuration. +// +// Due to its close relationship with HCL, this package makes frequent use +// of types from the HCL API, including raw HCL diagnostic messages. Such +// diagnostics can be converted into Terraform-flavored diagnostics, if needed, +// using functions in the sibling package tfdiags. +package configs From a0f4a313ef9d9f6cbab300cc72a03ad4d4b21941 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 1 Feb 2018 19:07:02 -0800 Subject: [PATCH 03/25] configs: Parser type configs.Parser is the entry-point for this package, providing functions to load and parse HCL-based configuration files. We use the library "afero" to decouple the parser from the physical OS filesystem, which here allows us to easily use an in-memory filesystem for testing and will, in future, allow us to read files from more unusual places, such as configuration embedded in a plan file. --- configs/parser.go | 85 ++++++++++++++++++++++++++++++++++++++++++ configs/parser_test.go | 31 +++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 configs/parser.go create mode 100644 configs/parser_test.go diff --git a/configs/parser.go b/configs/parser.go new file mode 100644 index 000000000..3d3f07ff1 --- /dev/null +++ b/configs/parser.go @@ -0,0 +1,85 @@ +package configs + +import ( + "fmt" + "strings" + + "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hclparse" + "github.com/spf13/afero" +) + +// Parser is the main interface to read configuration files and other related +// files from disk. +// +// It retains a cache of all files that are loaded so that they can be used +// to create source code snippets in diagnostics, etc. +type Parser struct { + fs afero.Afero + p *hclparse.Parser +} + +// NewParser creates and returns a new Parser that reads files from the given +// filesystem. If a nil filesystem is passed then the system's "real" filesystem +// will be used, via afero.OsFs. +func NewParser(fs afero.Fs) *Parser { + if fs == nil { + fs = afero.OsFs{} + } + + return &Parser{ + fs: afero.Afero{Fs: fs}, + p: hclparse.NewParser(), + } +} + +// LoadHCLFile is a low-level method that reads the file at the given path, +// parses it, and returns the hcl.Body representing its root. In many cases +// it is better to use one of the other Load*File methods on this type, +// which additionally decode the root body in some way and return a higher-level +// construct. +// +// If the file cannot be read at all -- e.g. because it does not exist -- then +// this method will return a nil body and error diagnostics. In this case +// callers may wish to ignore the provided error diagnostics and produce +// a more context-sensitive error instead. +// +// The file will be parsed using the HCL native syntax unless the filename +// ends with ".json", in which case the HCL JSON syntax will be used. +func (p *Parser) LoadHCLFile(path string) (hcl.Body, hcl.Diagnostics) { + src, err := p.fs.ReadFile(path) + + if err != nil { + return nil, hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: "Failed to read file", + Detail: fmt.Sprintf("The file %q could not be read.", path), + }, + } + } + + var file *hcl.File + var diags hcl.Diagnostics + switch { + case strings.HasSuffix(path, ".json"): + file, diags = p.p.ParseJSON(src, path) + default: + file, diags = p.p.ParseHCL(src, path) + } + + // If the returned file or body is nil, then we'll return a non-nil empty + // body so we'll meet our contract that nil means an error reading the file. + if file == nil || file.Body == nil { + return hcl.EmptyBody(), diags + } + + return file.Body, diags +} + +// Sources returns a map of the cached source buffers for all files that +// have been loaded through this parser, with source filenames (as requested +// when each file was opened) as the keys. +func (p *Parser) Sources() map[string][]byte { + return p.p.Sources() +} diff --git a/configs/parser_test.go b/configs/parser_test.go new file mode 100644 index 000000000..316a09e5d --- /dev/null +++ b/configs/parser_test.go @@ -0,0 +1,31 @@ +package configs + +import ( + "os" + "path" + + "github.com/spf13/afero" +) + +// testParser returns a parser that reads files from the given map, which +// is from paths to file contents. +// +// Since this function uses only in-memory objects, it should never fail. +// If any errors are encountered in practice, this function will panic. +func testParser(files map[string]string) *Parser { + fs := afero.Afero{Fs: afero.NewMemMapFs()} + + for filePath, contents := range files { + dirPath := path.Dir(filePath) + err := fs.MkdirAll(dirPath, os.ModePerm) + if err != nil { + panic(err) + } + err = fs.WriteFile(filePath, []byte(contents), os.ModePerm) + if err != nil { + panic(err) + } + } + + return NewParser(fs) +} From 05e3b47ba1b6a7b18b7facf5a0fa4e8274bd9f51 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 1 Feb 2018 19:12:56 -0800 Subject: [PATCH 04/25] configs: Parser.LoadValuesFile This method loads a "values file" -- also known as a "tfvars file" -- and returns the values found inside. A values file is an HCL file (in either native or JSON syntax) whose top-level body is treated as a set of arbitrary key/value pairs whose values may not depend on any variables or functions. We will load values files through a configs.Parser -- even though values files are not strictly-speaking part of configuration -- because this causes them to be registered in our source code cache so that we can generate source code snippets if we need to report any diagnostics. --- configs/parser_values.go | 43 +++++++++++++ configs/parser_values_test.go | 113 ++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 configs/parser_values.go create mode 100644 configs/parser_values_test.go diff --git a/configs/parser_values.go b/configs/parser_values.go new file mode 100644 index 000000000..b7f1c1c5d --- /dev/null +++ b/configs/parser_values.go @@ -0,0 +1,43 @@ +package configs + +import ( + "github.com/hashicorp/hcl2/hcl" + "github.com/zclconf/go-cty/cty" +) + +// LoadValuesFile reads the file at the given path and parses it as a "values +// file", which is an HCL config file whose top-level attributes are treated +// as arbitrary key.value pairs. +// +// If the file cannot be read -- for example, if it does not exist -- then +// a nil map will be returned along with error diagnostics. Callers may wish +// to disregard the returned diagnostics in this case and instead generate +// their own error message(s) with additional context. +// +// If the returned diagnostics has errors when a non-nil map is returned +// then the map may be incomplete but should be valid enough for careful +// static analysis. +// +// This method wraps LoadHCLFile, and so it inherits the syntax selection +// behaviors documented for that method. +func (p *Parser) LoadValuesFile(path string) (map[string]cty.Value, hcl.Diagnostics) { + body, diags := p.LoadHCLFile(path) + if body == nil { + return nil, diags + } + + vals := make(map[string]cty.Value) + attrs, attrDiags := body.JustAttributes() + diags = append(diags, attrDiags...) + if attrs == nil { + return vals, diags + } + + for name, attr := range attrs { + val, valDiags := attr.Expr.Value(nil) + diags = append(diags, valDiags...) + vals[name] = val + } + + return vals, diags +} diff --git a/configs/parser_values_test.go b/configs/parser_values_test.go new file mode 100644 index 000000000..9c95b65f2 --- /dev/null +++ b/configs/parser_values_test.go @@ -0,0 +1,113 @@ +package configs + +import ( + "testing" + + "github.com/zclconf/go-cty/cty" +) + +func TestParserLoadValuesFile(t *testing.T) { + tests := map[string]struct { + Source string + Want map[string]cty.Value + DiagCount int + }{ + "empty.tfvars": { + "", + map[string]cty.Value{}, + 0, + }, + "empty.json": { + "{}", + map[string]cty.Value{}, + 0, + }, + "zerolen.json": { + "", + map[string]cty.Value{}, + 2, // syntax error and missing root object + }, + "one-number.tfvars": { + "foo = 1\n", + map[string]cty.Value{ + "foo": cty.NumberIntVal(1), + }, + 0, + }, + "one-number.tfvars.json": { + `{"foo": 1}`, + map[string]cty.Value{ + "foo": cty.NumberIntVal(1), + }, + 0, + }, + "two-bools.tfvars": { + "foo = true\nbar = false\n", + map[string]cty.Value{ + "foo": cty.True, + "bar": cty.False, + }, + 0, + }, + "two-bools.tfvars.json": { + `{"foo": true, "bar": false}`, + map[string]cty.Value{ + "foo": cty.True, + "bar": cty.False, + }, + 0, + }, + "invalid-syntax.tfvars": { + "foo bar baz\n", + map[string]cty.Value{}, + 1, // attribute or block definition required + }, + "block.tfvars": { + "foo = true\ninvalid {\n}\n", + map[string]cty.Value{ + "foo": cty.True, + }, + 1, // blocks are not allowed + }, + "variables.tfvars": { + "baz = true\nfoo = var.baz\n", + map[string]cty.Value{ + "baz": cty.True, + "foo": cty.DynamicVal, + }, + 1, // variables cannot be referenced here + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + p := testParser(map[string]string{ + name: test.Source, + }) + got, diags := p.LoadValuesFile(name) + if len(diags) != test.DiagCount { + t.Errorf("wrong number of diagnostics %d; want %d", len(diags), test.DiagCount) + for _, diag := range diags { + t.Logf("- %s", diag) + } + } + + if len(got) != len(test.Want) { + t.Errorf("wrong number of result keys %d; want %d", len(got), len(test.Want)) + } + + for name, gotVal := range got { + wantVal := test.Want[name] + if wantVal == cty.NilVal { + t.Errorf("unexpected result key %q", name) + continue + } + + if !gotVal.RawEquals(wantVal) { + t.Errorf("wrong value for %q\ngot: %#v\nwant: %#v", name, gotVal, wantVal) + continue + } + } + }) + } +} From 13fa73c63e36b5fb22867d094c7d3e1c958c6dae Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 1 Feb 2018 20:33:06 -0800 Subject: [PATCH 05/25] configs: stub out main configuration structs These types represent the individual elements within configuration, the modules a configuration is made of, and the configuration (static module tree) itself. --- configs/backend.go | 14 +++++ configs/config.go | 30 +++++++++ configs/module.go | 66 ++++++++++++++++++++ configs/module_call.go | 18 ++++++ configs/named_values.go | 37 +++++++++++ configs/provider.go | 28 +++++++++ configs/provisioneronfailure_string.go | 16 +++++ configs/provisionerwhen_string.go | 16 +++++ configs/resource.go | 86 ++++++++++++++++++++++++++ configs/variable_type_hint.go | 45 ++++++++++++++ configs/variabletypehint_string.go | 29 +++++++++ configs/version_constraint.go | 15 +++++ 12 files changed, 400 insertions(+) create mode 100644 configs/backend.go create mode 100644 configs/config.go create mode 100644 configs/module.go create mode 100644 configs/module_call.go create mode 100644 configs/named_values.go create mode 100644 configs/provider.go create mode 100644 configs/provisioneronfailure_string.go create mode 100644 configs/provisionerwhen_string.go create mode 100644 configs/resource.go create mode 100644 configs/variable_type_hint.go create mode 100644 configs/variabletypehint_string.go create mode 100644 configs/version_constraint.go diff --git a/configs/backend.go b/configs/backend.go new file mode 100644 index 000000000..43e872bed --- /dev/null +++ b/configs/backend.go @@ -0,0 +1,14 @@ +package configs + +import ( + "github.com/hashicorp/hcl2/hcl" +) + +// Backend represents a "backend" block inside a "terraform" block in a module +// or file. +type Backend struct { + Type string + Config hcl.Body + + DeclRange hcl.Range +} diff --git a/configs/config.go b/configs/config.go new file mode 100644 index 000000000..07f6ed0a5 --- /dev/null +++ b/configs/config.go @@ -0,0 +1,30 @@ +package configs + +// A Config is a node in the tree of modules within a configuration. +// +// The module tree is constructed by following ModuleCall instances recursively +// through the root module transitively into descendent modules. +// +// A module tree described in *this* package represents the static tree +// represented by configuration. During evaluation a static ModuleNode may +// expand into zero or more module instances depending on the use of count and +// for_each configuration attributes within each call. +type Config struct { + // RootModule points to the Config for the root module within the same + // module tree as this module. If this module _is_ the root module then + // this is self-referential. + Root *Config + + // ParentModule points to the Config for the module that directly calls + // this module. If this is the root module then this field is nil. + Parent *Config + + // ChildModules points to the Config for each of the direct child modules + // called from this module. The keys in this map match the keys in + // Module.ModuleCalls. + Children map[string]*Config + + // Elements points to the object describing the configuration for the + // various elements (variables, resources, etc) defined by this module. + Elements *Module +} diff --git a/configs/module.go b/configs/module.go new file mode 100644 index 000000000..5b9f51041 --- /dev/null +++ b/configs/module.go @@ -0,0 +1,66 @@ +package configs + +import ( + "github.com/hashicorp/hcl2/hcl" +) + +// Module is a container for a set of configuration constructs that are +// evaluated within a common namespace. +type Module struct { + CoreVersionConstraints []VersionConstraint + + Backend *Backend + ProviderConfigs map[string]*Provider + ProviderRequirements map[string][]VersionConstraint + + Variables map[string]*Variable + Locals map[string]*Local + Outputs map[string]*Output + + ModuleCalls map[string]*ModuleCall + + ManagedResources map[string]*ManagedResource + DataResources map[string]*DataResource +} + +// NewModule takes a list of primary files and a list of override files and +// produces a *Module by combining the files together. +// +// If there are any conflicting declarations in the given files -- for example, +// if the same variable name is defined twice -- then the resulting module +// will be incomplete and error diagnostics will be returned. Careful static +// analysis of the returned Module is still possible in this case, but the +// module will probably not be semantically valid. +func NewModule(primaryFiles, overrideFiles []*File) (*Module, hcl.Diagnostics) { + // TODO: process each file in turn, combining and merging as necessary + // to produce a single flat *Module. + panic("NewModule not yet implemented") +} + +// File describes the contents of a single configuration file. +// +// Individual files are not usually used alone, but rather combined together +// with other files (conventionally, those in the same directory) to produce +// a *Module, using NewModule. +// +// At the level of an individual file we represent directly the structural +// elements present in the file, without any attempt to detect conflicting +// declarations. A File object can therefore be used for some basic static +// analysis of individual elements, but must be built into a Module to detect +// duplicate declarations. +type File struct { + CoreVersionConstraints []*VersionConstraint + + Backends []*Backend + ProviderConfigs []*Provider + ProviderRequirements []*ProviderRequirement + + Variables []*Variable + Locals []*Local + Outputs []*Output + + ModuleCalls []*ModuleCall + + ManagedResources []*ManagedResource + DataResources []*DataResource +} diff --git a/configs/module_call.go b/configs/module_call.go new file mode 100644 index 000000000..ca3205cc4 --- /dev/null +++ b/configs/module_call.go @@ -0,0 +1,18 @@ +package configs + +import ( + "github.com/hashicorp/hcl2/hcl" +) + +// ModuleCall represents a "module" block in a module or file. +type ModuleCall struct { + Source string + SourceRange hcl.Range + + Version VersionConstraint + + Count hcl.Expression + ForEach hcl.Expression + + DeclRange hcl.Range +} diff --git a/configs/named_values.go b/configs/named_values.go new file mode 100644 index 000000000..c2d2c8b83 --- /dev/null +++ b/configs/named_values.go @@ -0,0 +1,37 @@ +package configs + +import ( + "github.com/hashicorp/hcl2/hcl" + "github.com/zclconf/go-cty/cty" +) + +// Variable represents a "variable" block in a module or file. +type Variable struct { + Name string + Description string + Default cty.Value + TypeHint VariableTypeHint + + DeclRange hcl.Range +} + +// Output represents an "output" block in a module or file. +type Output struct { + Name string + Description string + Expr hcl.Expression + DependsOn []hcl.Traversal + Sensitive bool + + DeclRange hcl.Range +} + +// Local represents a single entry from a "locals" block in a module or file. +// The "locals" block itself is not represented, because it serves only to +// provide context for us to interpret its contents. +type Local struct { + Name string + Expr hcl.Expression + + DeclRange hcl.Range +} diff --git a/configs/provider.go b/configs/provider.go new file mode 100644 index 000000000..8145a87dc --- /dev/null +++ b/configs/provider.go @@ -0,0 +1,28 @@ +package configs + +import ( + "github.com/hashicorp/hcl2/hcl" +) + +// Provider represents a "provider" block in a module or file. A provider +// block is a provider configuration, and there can be zero or more +// configurations for each actual provider. +type Provider struct { + Name string + Alias string + AliasRange hcl.Range + + Version VersionConstraint + + Config hcl.Body + + DeclRange hcl.Range +} + +// ProviderRequirement represents a declaration of a dependency on a particular +// provider version without actually configuring that provider. This is used in +// child modules that expect a provider to be passed in from their parent. +type ProviderRequirement struct { + Name string + Requirement VersionConstraint +} diff --git a/configs/provisioneronfailure_string.go b/configs/provisioneronfailure_string.go new file mode 100644 index 000000000..8704b0861 --- /dev/null +++ b/configs/provisioneronfailure_string.go @@ -0,0 +1,16 @@ +// Code generated by "stringer -type ProvisionerOnFailure"; DO NOT EDIT. + +package configs + +import "strconv" + +const _ProvisionerOnFailure_name = "ProvisionerOnFailureInvalidProvisionerOnFailureContinueProvisionerOnFailureFail" + +var _ProvisionerOnFailure_index = [...]uint8{0, 27, 55, 79} + +func (i ProvisionerOnFailure) String() string { + if i < 0 || i >= ProvisionerOnFailure(len(_ProvisionerOnFailure_index)-1) { + return "ProvisionerOnFailure(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _ProvisionerOnFailure_name[_ProvisionerOnFailure_index[i]:_ProvisionerOnFailure_index[i+1]] +} diff --git a/configs/provisionerwhen_string.go b/configs/provisionerwhen_string.go new file mode 100644 index 000000000..cbecb20ae --- /dev/null +++ b/configs/provisionerwhen_string.go @@ -0,0 +1,16 @@ +// Code generated by "stringer -type ProvisionerWhen"; DO NOT EDIT. + +package configs + +import "strconv" + +const _ProvisionerWhen_name = "ProvisionerWhenInvalidProvisionerWhenCreateProvisionerWhenDestroy" + +var _ProvisionerWhen_index = [...]uint8{0, 22, 43, 65} + +func (i ProvisionerWhen) String() string { + if i < 0 || i >= ProvisionerWhen(len(_ProvisionerWhen_index)-1) { + return "ProvisionerWhen(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _ProvisionerWhen_name[_ProvisionerWhen_index[i]:_ProvisionerWhen_index[i+1]] +} diff --git a/configs/resource.go b/configs/resource.go new file mode 100644 index 000000000..459ce0d3d --- /dev/null +++ b/configs/resource.go @@ -0,0 +1,86 @@ +package configs + +import ( + "github.com/hashicorp/hcl2/hcl" +) + +// ManagedResource represents a "resource" block in a module or file. +type ManagedResource struct { + Name string + Type string + Config hcl.Body + Count hcl.Expression + ForEach hcl.Expression + + ProviderConfigAddr hcl.Traversal + + DependsOn []hcl.Traversal + + Connection *Connection + Provisioners []*Provisioner + + CreateBeforeDestroy bool + PreventDestroy bool + IgnoreChanges []hcl.Traversal + + DeclRange hcl.Range +} + +// DataResource represents a "data" block in a module or file. +type DataResource struct { + Name string + Type string + Config hcl.Body + Count hcl.Expression + ForEach hcl.Expression + + ProviderConfigAddr hcl.Traversal + + DependsOn []hcl.Traversal + + DeclRange hcl.Range +} + +// Provisioner represents a "provisioner" block when used within a +// "resource" block in a module or file. +type Provisioner struct { + Type string + Config hcl.Body + Connection *Connection + When ProvisionerWhen + OnFailure ProvisionerOnFailure + + DeclRange hcl.Range +} + +// Connection represents a "connection" block when used within either a +// "resource" or "provisioner" block in a module or file. +type Connection struct { + Type string + Config hcl.Body + + DeclRange hcl.Range +} + +// ProvisionerWhen is an enum for valid values for when to run provisioners. +type ProvisionerWhen int + +//go:generate stringer -type ProvisionerWhen + +const ( + ProvisionerWhenInvalid ProvisionerWhen = iota + ProvisionerWhenCreate + ProvisionerWhenDestroy +) + +// ProvisionerOnFailure is an enum for valid values for on_failure options +// for provisioners. +type ProvisionerOnFailure int + +//go:generate stringer -type ProvisionerOnFailure + +const ( + ProvisionerOnFailureInvalid ProvisionerOnFailure = iota + ProvisionerOnFailureContinue + ProvisionerOnFailureFail +) diff --git a/configs/variable_type_hint.go b/configs/variable_type_hint.go new file mode 100644 index 000000000..204efd120 --- /dev/null +++ b/configs/variable_type_hint.go @@ -0,0 +1,45 @@ +package configs + +// VariableTypeHint is an enumeration used for the Variable.TypeHint field, +// which is an incompletely-specified type for the variable which is used +// as a hint for whether a value provided in an ambiguous context (on the +// command line or in an environment variable) should be taken literally as a +// string or parsed as an HCL expression to produce a data structure. +// +// The type hint is applied to runtime values as well, but since it does not +// accurately describe a precise type it is not fully-sufficient to infer +// the dynamic type of a value passed through a variable. +// +// These hints use inaccurate terminology for historical reasons. Full details +// are in the documentation for each constant in this enumeration, but in +// summary: +// +// TypeHintString requires a primitive type +// TypeHintList requires a type that could be converted to a tuple +// TypeHintMap requires a type that could be converted to an object +type VariableTypeHint rune + +//go:generate stringer -type VariableTypeHint + +// TypeHintNone indicates the absense of a type hint. Values specified in +// ambiguous contexts will be treated as literal strings, as if TypeHintString +// were selected, but no runtime value checks will be applied. This is reasonable +// type hint for a module that is never intended to be used at the top-level +// of a configuration, since descendent modules never recieve values from +// ambiguous contexts. +const TypeHintNone VariableTypeHint = 0 + +// TypeHintString spec indicates that a value provided in an ambiguous context +// should be treated as a literal string, and additionally requires that the +// runtime value for the variable is of a primitive type (string, number, bool). +const TypeHintString VariableTypeHint = 'S' + +// TypeHintList indicates that a value provided in an ambiguous context should +// be treated as an HCL expression, and additionally requires that the +// runtime value for the variable is of an tuple, list, or set type. +const TypeHintList VariableTypeHint = 'L' + +// TypeHintMap indicates that a value provided in an ambiguous context should +// be treated as an HCL expression, and additionally requires that the +// runtime value for the variable is of an object or map type. +const TypeHintMap VariableTypeHint = 'M' diff --git a/configs/variabletypehint_string.go b/configs/variabletypehint_string.go new file mode 100644 index 000000000..4447cf556 --- /dev/null +++ b/configs/variabletypehint_string.go @@ -0,0 +1,29 @@ +// Code generated by "stringer -type VariableTypeHint"; DO NOT EDIT. + +package configs + +import "strconv" + +const ( + _VariableTypeHint_name_0 = "TypeHintNone" + _VariableTypeHint_name_1 = "TypeHintListTypeHintMap" + _VariableTypeHint_name_2 = "TypeHintString" +) + +var ( + _VariableTypeHint_index_1 = [...]uint8{0, 12, 23} +) + +func (i VariableTypeHint) String() string { + switch { + case i == 0: + return _VariableTypeHint_name_0 + case 76 <= i && i <= 77: + i -= 76 + return _VariableTypeHint_name_1[_VariableTypeHint_index_1[i]:_VariableTypeHint_index_1[i+1]] + case i == 83: + return _VariableTypeHint_name_2 + default: + return "VariableTypeHint(" + strconv.FormatInt(int64(i), 10) + ")" + } +} diff --git a/configs/version_constraint.go b/configs/version_constraint.go new file mode 100644 index 000000000..6b2882cee --- /dev/null +++ b/configs/version_constraint.go @@ -0,0 +1,15 @@ +package configs + +import ( + version "github.com/hashicorp/go-version" + "github.com/hashicorp/hcl2/hcl" +) + +// VersionConstraint represents a version constraint on some resource +// (e.g. Terraform Core, a provider, a module, ...) that carries with it +// a source range so that a helpful diagnostic can be printed in the event +// that a particular constraint does not match. +type VersionConstraint struct { + Required []version.Constraints + DeclRange hcl.Range +} From b865d62bb843621aa0e2bd1fa21269959b1aaaf0 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 2 Feb 2018 11:15:48 -0800 Subject: [PATCH 06/25] govendor fetch github.com/hashicorp/hcl2/... This just catches us up with the latest fixes and improvements in upstream HCL2. --- .../hashicorp/hcl2/hcl/diagnostic_text.go | 85 +- .../hashicorp/hcl2/hcl/expr_list.go | 37 + .../hashicorp/hcl2/hcl/expr_unwrap.go | 68 + .../hashicorp/hcl2/hcl/hclsyntax/doc.go | 4 +- .../hcl2/hcl/hclsyntax/expression.go | 16 +- .../hashicorp/hcl2/hcl/hclsyntax/generate.go | 2 + .../hcl2/hcl/hclsyntax/navigation.go | 2 +- .../hashicorp/hcl2/hcl/hclsyntax/parser.go | 247 +- .../hcl2/hcl/hclsyntax/parser_template.go | 4 +- .../hashicorp/hcl2/hcl/hclsyntax/public.go | 30 +- .../hcl2/hcl/hclsyntax/scan_string_lit.go | 301 +++ .../hcl2/hcl/hclsyntax/scan_string_lit.rl | 105 + .../hcl2/hcl/hclsyntax/scan_tokens.go | 2329 ++++++++++++++--- .../hcl2/hcl/hclsyntax/scan_tokens.rl | 36 +- .../hashicorp/hcl2/hcl/hclsyntax/structure.go | 2 +- .../hashicorp/hcl2/hcl/hclsyntax/token.go | 1 + .../hcl2/hcl/hclsyntax/token_type_string.go | 4 +- .../hashicorp/hcl2/hcl/json/navigation.go | 2 +- .../hashicorp/hcl2/hcl/json/parser.go | 2 +- .../hashicorp/hcl2/hcl/json/public.go | 29 +- .../hashicorp/hcl2/hcl/json/spec.md | 2 +- .../hashicorp/hcl2/hcl/json/structure.go | 74 +- vendor/github.com/hashicorp/hcl2/hcl/ops.go | 2 +- vendor/github.com/hashicorp/hcl2/hcl/pos.go | 166 ++ .../hashicorp/hcl2/hcl/pos_scanner.go | 148 ++ vendor/github.com/hashicorp/hcl2/hcl/spec.md | 4 +- .../hashicorp/hcl2/hcl/traversal.go | 28 + .../hashicorp/hcl2/hcl/traversal_for_expr.go | 121 + .../hashicorp/hcl2/hcldec/public.go | 25 + .../github.com/hashicorp/hcl2/hcldec/spec.go | 139 + .../github.com/hashicorp/hcl2/hcltest/mock.go | 124 +- .../github.com/hashicorp/hcl2/hclwrite/ast.go | 2 +- .../hashicorp/hcl2/hclwrite/format.go | 2 +- .../hashicorp/hcl2/hclwrite/parser.go | 14 +- .../hashicorp/hcl2/hclwrite/public.go | 2 +- .../hashicorp/hcl2/hclwrite/tokens.go | 4 +- vendor/vendor.json | 48 +- 37 files changed, 3532 insertions(+), 679 deletions(-) create mode 100644 vendor/github.com/hashicorp/hcl2/hcl/expr_list.go create mode 100644 vendor/github.com/hashicorp/hcl2/hcl/expr_unwrap.go create mode 100644 vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_string_lit.go create mode 100644 vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_string_lit.rl create mode 100644 vendor/github.com/hashicorp/hcl2/hcl/pos_scanner.go create mode 100644 vendor/github.com/hashicorp/hcl2/hcl/traversal_for_expr.go diff --git a/vendor/github.com/hashicorp/hcl2/hcl/diagnostic_text.go b/vendor/github.com/hashicorp/hcl2/hcl/diagnostic_text.go index 9776f04de..dfa473add 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/diagnostic_text.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/diagnostic_text.go @@ -2,7 +2,6 @@ package hcl import ( "bufio" - "bytes" "errors" "fmt" "io" @@ -43,7 +42,7 @@ func (w *diagnosticTextWriter) WriteDiagnostic(diag *Diagnostic) error { return errors.New("nil diagnostic") } - var colorCode, resetCode string + var colorCode, highlightCode, resetCode string if w.color { switch diag.Severity { case DiagError: @@ -52,6 +51,7 @@ func (w *diagnosticTextWriter) WriteDiagnostic(diag *Diagnostic) error { colorCode = "\x1b[33m" } resetCode = "\x1b[0m" + highlightCode = "\x1b[1;4m" } var severityStr string @@ -68,24 +68,31 @@ func (w *diagnosticTextWriter) WriteDiagnostic(diag *Diagnostic) error { fmt.Fprintf(w.wr, "%s%s%s: %s\n\n", colorCode, severityStr, resetCode, diag.Summary) if diag.Subject != nil { + snipRange := *diag.Subject + highlightRange := snipRange + if diag.Context != nil { + // Show enough of the source code to include both the subject + // and context ranges, which overlap in all reasonable + // situations. + snipRange = RangeOver(snipRange, *diag.Context) + } + // We can't illustrate an empty range, so we'll turn such ranges into + // single-character ranges, which might not be totally valid (may point + // off the end of a line, or off the end of the file) but are good + // enough for the bounds checks we do below. + if snipRange.Empty() { + snipRange.End.Byte++ + snipRange.End.Column++ + } + if highlightRange.Empty() { + highlightRange.End.Byte++ + highlightRange.End.Column++ + } file := w.files[diag.Subject.Filename] if file == nil || file.Bytes == nil { fmt.Fprintf(w.wr, " on %s line %d:\n (source code not available)\n\n", diag.Subject.Filename, diag.Subject.Start.Line) } else { - src := file.Bytes - r := bytes.NewReader(src) - sc := bufio.NewScanner(r) - sc.Split(bufio.ScanLines) - - var startLine, endLine int - if diag.Context != nil { - startLine = diag.Context.Start.Line - endLine = diag.Context.End.Line - } else { - startLine = diag.Subject.Start.Line - endLine = diag.Subject.End.Line - } var contextLine string if diag.Subject != nil { @@ -95,35 +102,33 @@ func (w *diagnosticTextWriter) WriteDiagnostic(diag *Diagnostic) error { } } - li := 1 - var ls string - for sc.Scan() { - ls = sc.Text() - - if li == startLine { - break - } - li++ - } - fmt.Fprintf(w.wr, " on %s line %d%s:\n", diag.Subject.Filename, diag.Subject.Start.Line, contextLine) - // TODO: Generate markers for the specific characters that are in the Context and Subject ranges. - // For now, we just print out the lines. + src := file.Bytes + sc := NewRangeScanner(src, diag.Subject.Filename, bufio.ScanLines) - fmt.Fprintf(w.wr, "%4d: %s\n", li, ls) - - if endLine > li { - for sc.Scan() { - ls = sc.Text() - li++ - - fmt.Fprintf(w.wr, "%4d: %s\n", li, ls) - - if li == endLine { - break - } + for sc.Scan() { + lineRange := sc.Range() + if !lineRange.Overlaps(snipRange) { + continue } + + beforeRange, highlightedRange, afterRange := lineRange.PartitionAround(highlightRange) + if highlightedRange.Empty() { + fmt.Fprintf(w.wr, "%4d: %s\n", lineRange.Start.Line, sc.Bytes()) + } else { + before := beforeRange.SliceBytes(src) + highlighted := highlightedRange.SliceBytes(src) + after := afterRange.SliceBytes(src) + fmt.Fprintf( + w.wr, "%4d: %s%s%s%s%s\n", + lineRange.Start.Line, + before, + highlightCode, highlighted, resetCode, + after, + ) + } + } w.wr.Write([]byte{'\n'}) diff --git a/vendor/github.com/hashicorp/hcl2/hcl/expr_list.go b/vendor/github.com/hashicorp/hcl2/hcl/expr_list.go new file mode 100644 index 000000000..d05cca0b9 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl2/hcl/expr_list.go @@ -0,0 +1,37 @@ +package hcl + +// ExprList tests if the given expression is a static list construct and, +// if so, extracts the expressions that represent the list elements. +// If the given expression is not a static list, error diagnostics are +// returned. +// +// A particular Expression implementation can support this function by +// offering a method called ExprList that takes no arguments and returns +// []Expression. This method should return nil if a static list cannot +// be extracted. Alternatively, an implementation can support +// UnwrapExpression to delegate handling of this function to a wrapped +// Expression object. +func ExprList(expr Expression) ([]Expression, Diagnostics) { + type exprList interface { + ExprList() []Expression + } + + physExpr := UnwrapExpressionUntil(expr, func(expr Expression) bool { + _, supported := expr.(exprList) + return supported + }) + + if exL, supported := physExpr.(exprList); supported { + if list := exL.ExprList(); list != nil { + return list, nil + } + } + return nil, Diagnostics{ + &Diagnostic{ + Severity: DiagError, + Summary: "Invalid expression", + Detail: "A static list expression is required.", + Subject: expr.StartRange().Ptr(), + }, + } +} diff --git a/vendor/github.com/hashicorp/hcl2/hcl/expr_unwrap.go b/vendor/github.com/hashicorp/hcl2/hcl/expr_unwrap.go new file mode 100644 index 000000000..6d5d205c4 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl2/hcl/expr_unwrap.go @@ -0,0 +1,68 @@ +package hcl + +type unwrapExpression interface { + UnwrapExpression() Expression +} + +// UnwrapExpression removes any "wrapper" expressions from the given expression, +// to recover the representation of the physical expression given in source +// code. +// +// Sometimes wrapping expressions are used to modify expression behavior, e.g. +// in extensions that need to make some local variables available to certain +// sub-trees of the configuration. This can make it difficult to reliably +// type-assert on the physical AST types used by the underlying syntax. +// +// Unwrapping an expression may modify its behavior by stripping away any +// additional constraints or capabilities being applied to the Value and +// Variables methods, so this function should generally only be used prior +// to operations that concern themselves with the static syntax of the input +// configuration, and not with the effective value of the expression. +// +// Wrapper expression types must support unwrapping by implementing a method +// called UnwrapExpression that takes no arguments and returns the embedded +// Expression. Implementations of this method should peel away only one level +// of wrapping, if multiple are present. This method may return nil to +// indicate _dynamically_ that no wrapped expression is available, for +// expression types that might only behave as wrappers in certain cases. +func UnwrapExpression(expr Expression) Expression { + for { + unwrap, wrapped := expr.(unwrapExpression) + if !wrapped { + return expr + } + innerExpr := unwrap.UnwrapExpression() + if innerExpr == nil { + return expr + } + expr = innerExpr + } +} + +// UnwrapExpressionUntil is similar to UnwrapExpression except it gives the +// caller an opportunity to test each level of unwrapping to see each a +// particular expression is accepted. +// +// This could be used, for example, to unwrap until a particular other +// interface is satisfied, regardless of wrap wrapping level it is satisfied +// at. +// +// The given callback function must return false to continue wrapping, or +// true to accept and return the proposed expression given. If the callback +// function rejects even the final, physical expression then the result of +// this function is nil. +func UnwrapExpressionUntil(expr Expression, until func(Expression) bool) Expression { + for { + if until(expr) { + return expr + } + unwrap, wrapped := expr.(unwrapExpression) + if !wrapped { + return nil + } + expr = unwrap.UnwrapExpression() + if expr == nil { + return nil + } + } +} diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/doc.go b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/doc.go index 8db11573f..617bc29dc 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/doc.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/doc.go @@ -1,7 +1,7 @@ -// Package hclsyntax contains the parser, AST, etc for zcl's native language, +// Package hclsyntax contains the parser, AST, etc for HCL's native language, // as opposed to the JSON variant. // // In normal use applications should rarely depend on this package directly, // instead preferring the higher-level interface of the main hcl package and -// its companion hclparse. +// its companion package hclparse. package hclsyntax diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/expression.go b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/expression.go index e90ac2be6..4fa1988bf 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/expression.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/expression.go @@ -9,7 +9,7 @@ import ( "github.com/zclconf/go-cty/cty/function" ) -// Expression is the abstract type for nodes that behave as zcl expressions. +// Expression is the abstract type for nodes that behave as HCL expressions. type Expression interface { Node @@ -70,6 +70,11 @@ func (e *ScopeTraversalExpr) StartRange() hcl.Range { return e.SrcRange } +// Implementation for hcl.AbsTraversalForExpr. +func (e *ScopeTraversalExpr) AsTraversal() hcl.Traversal { + return e.Traversal +} + // RelativeTraversalExpr is an Expression that retrieves a value from another // value using a _relative_ traversal. type RelativeTraversalExpr struct { @@ -539,6 +544,15 @@ func (e *TupleConsExpr) StartRange() hcl.Range { return e.OpenRange } +// Implementation for hcl.ExprList +func (e *TupleConsExpr) ExprList() []hcl.Expression { + ret := make([]hcl.Expression, len(e.Exprs)) + for i, expr := range e.Exprs { + ret[i] = expr + } + return ret +} + type ObjectConsExpr struct { Items []ObjectConsItem diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/generate.go b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/generate.go index ecc389f11..841656a6a 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/generate.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/generate.go @@ -4,4 +4,6 @@ package hclsyntax //go:generate ruby unicode2ragel.rb --url=http://www.unicode.org/Public/9.0.0/ucd/DerivedCoreProperties.txt -m UnicodeDerived -p ID_Start,ID_Continue -o unicode_derived.rl //go:generate ragel -Z scan_tokens.rl //go:generate gofmt -w scan_tokens.go +//go:generate ragel -Z scan_string_lit.rl +//go:generate gofmt -w scan_string_lit.go //go:generate stringer -type TokenType -output token_type_string.go diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/navigation.go b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/navigation.go index 8a04c2081..4d41b6b66 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/navigation.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/navigation.go @@ -9,7 +9,7 @@ type navigation struct { root *Body } -// Implementation of zcled.ContextString +// Implementation of hcled.ContextString func (n navigation) ContextString(offset int) string { // We will walk our top-level blocks until we find one that contains // the given offset, and then construct a representation of the header diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/parser.go b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/parser.go index 0f81ddfb1..28c6a7b19 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/parser.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/parser.go @@ -1,9 +1,10 @@ package hclsyntax import ( - "bufio" "bytes" "fmt" + "strconv" + "unicode/utf8" "github.com/apparentlymart/go-textseg/textseg" "github.com/hashicorp/hcl2/hcl" @@ -627,7 +628,7 @@ Traversal: if lit, isLit := keyExpr.(*LiteralValueExpr); isLit { litKey, _ := lit.Value(nil) rng := hcl.RangeBetween(open.Range, close.Range) - step := &hcl.TraverseIndex{ + step := hcl.TraverseIndex{ Key: litKey, SrcRange: rng, } @@ -1476,139 +1477,149 @@ func (p *parser) decodeStringLit(tok Token) (string, hcl.Diagnostics) { var diags hcl.Diagnostics ret := make([]byte, 0, len(tok.Bytes)) - var esc []byte + slices := scanStringLit(tok.Bytes, quoted) - sc := bufio.NewScanner(bytes.NewReader(tok.Bytes)) - sc.Split(textseg.ScanGraphemeClusters) + // We will mutate rng constantly as we walk through our token slices below. + // Any diagnostics must take a copy of this rng rather than simply pointing + // to it, e.g. by using rng.Ptr() rather than &rng. + rng := tok.Range + rng.End = rng.Start - pos := tok.Range.Start - newPos := pos -Character: - for sc.Scan() { - pos = newPos - ch := sc.Bytes() - - // Adjust position based on our new character. - // \r\n is considered to be a single character in text segmentation, - if (len(ch) == 1 && ch[0] == '\n') || (len(ch) == 2 && ch[1] == '\n') { - newPos.Line++ - newPos.Column = 0 - } else { - newPos.Column++ +Slices: + for _, slice := range slices { + if len(slice) == 0 { + continue } - newPos.Byte += len(ch) - if len(esc) > 0 { - switch esc[0] { - case '\\': - if len(ch) == 1 { - switch ch[0] { + // Advance the start of our range to where the previous token ended + rng.Start = rng.End - // TODO: numeric character escapes with \uXXXX - - case 'n': - ret = append(ret, '\n') - esc = esc[:0] - continue Character - case 'r': - ret = append(ret, '\r') - esc = esc[:0] - continue Character - case 't': - ret = append(ret, '\t') - esc = esc[:0] - continue Character - case '"': - ret = append(ret, '"') - esc = esc[:0] - continue Character - case '\\': - ret = append(ret, '\\') - esc = esc[:0] - continue Character - } - } - - var detail string - switch { - case len(ch) == 1 && (ch[0] == '$' || ch[0] == '!'): - detail = fmt.Sprintf( - "The characters \"\\%s\" do not form a recognized escape sequence. To escape a \"%s{\" template sequence, use \"%s%s{\".", - ch, ch, ch, ch, - ) - default: - detail = fmt.Sprintf("The characters \"\\%s\" do not form a recognized escape sequence.", ch) - } + // Advance the end of our range to after our token. + b := slice + for len(b) > 0 { + adv, ch, _ := textseg.ScanGraphemeClusters(b, true) + rng.End.Byte += adv + switch ch[0] { + case '\r', '\n': + rng.End.Line++ + rng.End.Column = 1 + default: + rng.End.Column++ + } + b = b[adv:] + } + TokenType: + switch slice[0] { + case '\\': + if !quoted { + // If we're not in quoted mode then just treat this token as + // normal. (Slices can still start with backslash even if we're + // not specifically looking for backslash sequences.) + break TokenType + } + if len(slice) < 2 { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "Invalid escape sequence", - Detail: detail, - Subject: &hcl.Range{ - Filename: tok.Range.Filename, - Start: hcl.Pos{ - Line: pos.Line, - Column: pos.Column - 1, // safe because we know the previous character must be a backslash - Byte: pos.Byte - 1, - }, - End: hcl.Pos{ - Line: pos.Line, - Column: pos.Column + 1, // safe because we know the previous character must be a backslash - Byte: pos.Byte + len(ch), - }, - }, + Detail: "Backslash must be followed by an escape sequence selector character.", + Subject: rng.Ptr(), }) - ret = append(ret, ch...) - esc = esc[:0] - continue Character + break TokenType + } - case '$', '!': - switch len(esc) { - case 1: - if len(ch) == 1 && ch[0] == esc[0] { - esc = append(esc, ch[0]) - continue Character - } + switch slice[1] { - // Any other character means this wasn't an escape sequence - // after all. - ret = append(ret, esc...) - ret = append(ret, ch...) - esc = esc[:0] - case 2: - if len(ch) == 1 && ch[0] == '{' { - // successful escape sequence - ret = append(ret, esc[0]) - } else { - // not an escape sequence, so just output literal - ret = append(ret, esc...) - } - ret = append(ret, ch...) - esc = esc[:0] - default: - // should never happen - panic("have invalid escape sequence >2 characters") + case 'n': + ret = append(ret, '\n') + continue Slices + case 'r': + ret = append(ret, '\r') + continue Slices + case 't': + ret = append(ret, '\t') + continue Slices + case '"': + ret = append(ret, '"') + continue Slices + case '\\': + ret = append(ret, '\\') + continue Slices + case 'u', 'U': + if slice[1] == 'u' && len(slice) != 6 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid escape sequence", + Detail: "The \\u escape sequence must be followed by four hexadecimal digits.", + Subject: rng.Ptr(), + }) + break TokenType + } else if slice[1] == 'U' && len(slice) != 10 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid escape sequence", + Detail: "The \\U escape sequence must be followed by eight hexadecimal digits.", + Subject: rng.Ptr(), + }) + break TokenType } - } - } else { - if len(ch) == 1 { - switch ch[0] { - case '\\': - if quoted { // ignore backslashes in unquoted mode - esc = append(esc, '\\') - continue Character - } - case '$': - esc = append(esc, '$') - continue Character - case '!': - esc = append(esc, '!') - continue Character + numHex := string(slice[2:]) + num, err := strconv.ParseUint(numHex, 16, 32) + if err != nil { + // Should never happen because the scanner won't match + // a sequence of digits that isn't valid. + panic(err) } + + r := rune(num) + l := utf8.RuneLen(r) + if l == -1 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid escape sequence", + Detail: fmt.Sprintf("Cannot encode character U+%04x in UTF-8.", num), + Subject: rng.Ptr(), + }) + break TokenType + } + for i := 0; i < l; i++ { + ret = append(ret, 0) + } + rb := ret[len(ret)-l:] + utf8.EncodeRune(rb, r) + + continue Slices + + default: + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid escape sequence", + Detail: fmt.Sprintf("The symbol %q is not a valid escape sequence selector.", slice[1:]), + Subject: rng.Ptr(), + }) + ret = append(ret, slice[1:]...) + continue Slices } - ret = append(ret, ch...) + + case '$', '%': + if len(slice) != 3 { + // Not long enough to be our escape sequence, so it's literal. + break TokenType + } + + if slice[1] == slice[0] && slice[2] == '{' { + ret = append(ret, slice[0]) + ret = append(ret, '{') + continue Slices + } + + break TokenType } + + // If we fall out here or break out of here from the switch above + // then this slice is just a literal. + ret = append(ret, slice...) } return string(ret), diags diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/parser_template.go b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/parser_template.go index e04c8e0f3..3711067ee 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/parser_template.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/parser_template.go @@ -435,7 +435,7 @@ Token: }) case TokenTemplateControl: - // if the opener is !{~ then we want to eat any trailing whitespace + // if the opener is %{~ then we want to eat any trailing whitespace // in the preceding literal token, assuming it is indeed a literal // token. if canTrimPrev && len(next.Bytes) == 3 && next.Bytes[2] == '~' && len(parts) > 0 { @@ -452,7 +452,7 @@ Token: diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "Invalid template directive", - Detail: "A template directive keyword (\"if\", \"for\", etc) is expected at the beginning of a !{ sequence.", + Detail: "A template directive keyword (\"if\", \"for\", etc) is expected at the beginning of a %{ sequence.", Subject: &kw.Range, Context: hcl.RangeBetween(next.Range, kw.Range).Ptr(), }) diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/public.go b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/public.go index 49d8ab182..e527d63f4 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/public.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/public.go @@ -4,13 +4,13 @@ import ( "github.com/hashicorp/hcl2/hcl" ) -// ParseConfig parses the given buffer as a whole zcl config file, returning +// ParseConfig parses the given buffer as a whole HCL config file, returning // a *hcl.File representing its contents. If HasErrors called on the returned // diagnostics returns true, the returned body is likely to be incomplete // and should therefore be used with care. // -// The body in the returned file has dynamic type *zclsyntax.Body, so callers -// may freely type-assert this to get access to the full zclsyntax API in +// The body in the returned file has dynamic type *hclsyntax.Body, so callers +// may freely type-assert this to get access to the full hclsyntax API in // situations where detailed access is required. However, most common use-cases // should be served using the hcl.Body interface to ensure compatibility with // other configurationg syntaxes, such as JSON. @@ -30,7 +30,7 @@ func ParseConfig(src []byte, filename string, start hcl.Pos) (*hcl.File, hcl.Dia }, diags } -// ParseExpression parses the given buffer as a standalone zcl expression, +// ParseExpression parses the given buffer as a standalone HCL expression, // returning it as an instance of Expression. func ParseExpression(src []byte, filename string, start hcl.Pos) (Expression, hcl.Diagnostics) { tokens, diags := LexExpression(src, filename, start) @@ -57,7 +57,7 @@ func ParseExpression(src []byte, filename string, start hcl.Pos) (Expression, hc return expr, diags } -// ParseTemplate parses the given buffer as a standalone zcl template, +// ParseTemplate parses the given buffer as a standalone HCL template, // returning it as an instance of Expression. func ParseTemplate(src []byte, filename string, start hcl.Pos) (Expression, hcl.Diagnostics) { tokens, diags := LexTemplate(src, filename, start) @@ -89,7 +89,7 @@ func ParseTraversalAbs(src []byte, filename string, start hcl.Pos) (hcl.Traversa } // LexConfig performs lexical analysis on the given buffer, treating it as a -// whole zcl config file, and returns the resulting tokens. +// whole HCL config file, and returns the resulting tokens. // // Only minimal validation is done during lexical analysis, so the returned // diagnostics may include errors about lexical issues such as bad character @@ -102,7 +102,7 @@ func LexConfig(src []byte, filename string, start hcl.Pos) (Tokens, hcl.Diagnost } // LexExpression performs lexical analysis on the given buffer, treating it as -// a standalone zcl expression, and returns the resulting tokens. +// a standalone HCL expression, and returns the resulting tokens. // // Only minimal validation is done during lexical analysis, so the returned // diagnostics may include errors about lexical issues such as bad character @@ -117,7 +117,7 @@ func LexExpression(src []byte, filename string, start hcl.Pos) (Tokens, hcl.Diag } // LexTemplate performs lexical analysis on the given buffer, treating it as a -// standalone zcl template, and returns the resulting tokens. +// standalone HCL template, and returns the resulting tokens. // // Only minimal validation is done during lexical analysis, so the returned // diagnostics may include errors about lexical issues such as bad character @@ -128,3 +128,17 @@ func LexTemplate(src []byte, filename string, start hcl.Pos) (Tokens, hcl.Diagno diags := checkInvalidTokens(tokens) return tokens, diags } + +// ValidIdentifier tests if the given string could be a valid identifier in +// a native syntax expression. +// +// This is useful when accepting names from the user that will be used as +// variable or attribute names in the scope, to ensure that any name chosen +// will be traversable using the variable or attribute traversal syntax. +func ValidIdentifier(s string) bool { + // This is a kinda-expensive way to do something pretty simple, but it + // is easiest to do with our existing scanner-related infrastructure here + // and nobody should be validating identifiers in a tight loop. + tokens := scanTokens([]byte(s), "", hcl.Pos{}, scanIdentOnly) + return len(tokens) == 2 && tokens[0].Type == TokenIdent && tokens[1].Type == TokenEOF +} diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_string_lit.go b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_string_lit.go new file mode 100644 index 000000000..de1f524ca --- /dev/null +++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_string_lit.go @@ -0,0 +1,301 @@ +// line 1 "scan_string_lit.rl" + +package hclsyntax + +// This file is generated from scan_string_lit.rl. DO NOT EDIT. + +// line 9 "scan_string_lit.go" +var _hclstrtok_actions []byte = []byte{ + 0, 1, 0, 1, 1, 2, 1, 0, +} + +var _hclstrtok_key_offsets []byte = []byte{ + 0, 0, 2, 4, 6, 10, 14, 18, + 22, 27, 31, 36, 41, 46, 51, 57, + 62, 74, 85, 96, 107, 118, 129, 140, + 151, +} + +var _hclstrtok_trans_keys []byte = []byte{ + 128, 191, 128, 191, 128, 191, 10, 13, + 36, 37, 10, 13, 36, 37, 10, 13, + 36, 37, 10, 13, 36, 37, 10, 13, + 36, 37, 123, 10, 13, 36, 37, 10, + 13, 36, 37, 92, 10, 13, 36, 37, + 92, 10, 13, 36, 37, 92, 10, 13, + 36, 37, 92, 10, 13, 36, 37, 92, + 123, 10, 13, 36, 37, 92, 85, 117, + 128, 191, 192, 223, 224, 239, 240, 247, + 248, 255, 10, 13, 36, 37, 92, 48, + 57, 65, 70, 97, 102, 10, 13, 36, + 37, 92, 48, 57, 65, 70, 97, 102, + 10, 13, 36, 37, 92, 48, 57, 65, + 70, 97, 102, 10, 13, 36, 37, 92, + 48, 57, 65, 70, 97, 102, 10, 13, + 36, 37, 92, 48, 57, 65, 70, 97, + 102, 10, 13, 36, 37, 92, 48, 57, + 65, 70, 97, 102, 10, 13, 36, 37, + 92, 48, 57, 65, 70, 97, 102, 10, + 13, 36, 37, 92, 48, 57, 65, 70, + 97, 102, +} + +var _hclstrtok_single_lengths []byte = []byte{ + 0, 0, 0, 0, 4, 4, 4, 4, + 5, 4, 5, 5, 5, 5, 6, 5, + 2, 5, 5, 5, 5, 5, 5, 5, + 5, +} + +var _hclstrtok_range_lengths []byte = []byte{ + 0, 1, 1, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 5, 3, 3, 3, 3, 3, 3, 3, + 3, +} + +var _hclstrtok_index_offsets []byte = []byte{ + 0, 0, 2, 4, 6, 11, 16, 21, + 26, 32, 37, 43, 49, 55, 61, 68, + 74, 82, 91, 100, 109, 118, 127, 136, + 145, +} + +var _hclstrtok_indicies []byte = []byte{ + 0, 1, 2, 1, 3, 1, 5, 6, + 7, 8, 4, 10, 11, 12, 13, 9, + 14, 11, 12, 13, 9, 10, 11, 15, + 13, 9, 10, 11, 12, 13, 14, 9, + 10, 11, 12, 15, 9, 17, 18, 19, + 20, 21, 16, 23, 24, 25, 26, 27, + 22, 0, 24, 25, 26, 27, 22, 23, + 24, 28, 26, 27, 22, 23, 24, 25, + 26, 27, 0, 22, 23, 24, 25, 28, + 27, 22, 29, 30, 22, 2, 3, 31, + 22, 0, 23, 24, 25, 26, 27, 32, + 32, 32, 22, 23, 24, 25, 26, 27, + 33, 33, 33, 22, 23, 24, 25, 26, + 27, 34, 34, 34, 22, 23, 24, 25, + 26, 27, 30, 30, 30, 22, 23, 24, + 25, 26, 27, 35, 35, 35, 22, 23, + 24, 25, 26, 27, 36, 36, 36, 22, + 23, 24, 25, 26, 27, 37, 37, 37, + 22, 23, 24, 25, 26, 27, 0, 0, + 0, 22, +} + +var _hclstrtok_trans_targs []byte = []byte{ + 11, 0, 1, 2, 4, 5, 6, 7, + 9, 4, 5, 6, 7, 9, 5, 8, + 10, 11, 12, 13, 15, 16, 10, 11, + 12, 13, 15, 16, 14, 17, 21, 3, + 18, 19, 20, 22, 23, 24, +} + +var _hclstrtok_trans_actions []byte = []byte{ + 0, 0, 0, 0, 0, 1, 1, 1, + 1, 3, 5, 5, 5, 5, 0, 0, + 0, 1, 1, 1, 1, 1, 3, 5, + 5, 5, 5, 5, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, +} + +var _hclstrtok_eof_actions []byte = []byte{ + 0, 0, 0, 0, 0, 3, 3, 3, + 3, 3, 0, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, +} + +const hclstrtok_start int = 4 +const hclstrtok_first_final int = 4 +const hclstrtok_error int = 0 + +const hclstrtok_en_quoted int = 10 +const hclstrtok_en_unquoted int = 4 + +// line 10 "scan_string_lit.rl" + +func scanStringLit(data []byte, quoted bool) [][]byte { + var ret [][]byte + + // line 61 "scan_string_lit.rl" + + // Ragel state + p := 0 // "Pointer" into data + pe := len(data) // End-of-data "pointer" + ts := 0 + te := 0 + eof := pe + + var cs int // current state + switch { + case quoted: + cs = hclstrtok_en_quoted + default: + cs = hclstrtok_en_unquoted + } + + // Make Go compiler happy + _ = ts + _ = eof + + /*token := func () { + ret = append(ret, data[ts:te]) + }*/ + + // line 154 "scan_string_lit.go" + { + } + + // line 158 "scan_string_lit.go" + { + var _klen int + var _trans int + var _acts int + var _nacts uint + var _keys int + if p == pe { + goto _test_eof + } + if cs == 0 { + goto _out + } + _resume: + _keys = int(_hclstrtok_key_offsets[cs]) + _trans = int(_hclstrtok_index_offsets[cs]) + + _klen = int(_hclstrtok_single_lengths[cs]) + if _klen > 0 { + _lower := int(_keys) + var _mid int + _upper := int(_keys + _klen - 1) + for { + if _upper < _lower { + break + } + + _mid = _lower + ((_upper - _lower) >> 1) + switch { + case data[p] < _hclstrtok_trans_keys[_mid]: + _upper = _mid - 1 + case data[p] > _hclstrtok_trans_keys[_mid]: + _lower = _mid + 1 + default: + _trans += int(_mid - int(_keys)) + goto _match + } + } + _keys += _klen + _trans += _klen + } + + _klen = int(_hclstrtok_range_lengths[cs]) + if _klen > 0 { + _lower := int(_keys) + var _mid int + _upper := int(_keys + (_klen << 1) - 2) + for { + if _upper < _lower { + break + } + + _mid = _lower + (((_upper - _lower) >> 1) & ^1) + switch { + case data[p] < _hclstrtok_trans_keys[_mid]: + _upper = _mid - 2 + case data[p] > _hclstrtok_trans_keys[_mid+1]: + _lower = _mid + 2 + default: + _trans += int((_mid - int(_keys)) >> 1) + goto _match + } + } + _trans += _klen + } + + _match: + _trans = int(_hclstrtok_indicies[_trans]) + cs = int(_hclstrtok_trans_targs[_trans]) + + if _hclstrtok_trans_actions[_trans] == 0 { + goto _again + } + + _acts = int(_hclstrtok_trans_actions[_trans]) + _nacts = uint(_hclstrtok_actions[_acts]) + _acts++ + for ; _nacts > 0; _nacts-- { + _acts++ + switch _hclstrtok_actions[_acts-1] { + case 0: + // line 40 "scan_string_lit.rl" + + // If te is behind p then we've skipped over some literal + // characters which we must now return. + if te < p { + ret = append(ret, data[te:p]) + } + ts = p + + case 1: + // line 48 "scan_string_lit.rl" + + te = p + ret = append(ret, data[ts:te]) + + // line 255 "scan_string_lit.go" + } + } + + _again: + if cs == 0 { + goto _out + } + p++ + if p != pe { + goto _resume + } + _test_eof: + { + } + if p == eof { + __acts := _hclstrtok_eof_actions[cs] + __nacts := uint(_hclstrtok_actions[__acts]) + __acts++ + for ; __nacts > 0; __nacts-- { + __acts++ + switch _hclstrtok_actions[__acts-1] { + case 1: + // line 48 "scan_string_lit.rl" + + te = p + ret = append(ret, data[ts:te]) + + // line 281 "scan_string_lit.go" + } + } + } + + _out: + { + } + } + + // line 89 "scan_string_lit.rl" + + if te < p { + // Collect any leftover literal characters at the end of the input + ret = append(ret, data[te:p]) + } + + // If we fall out here without being in a final state then we've + // encountered something that the scanner can't match, which should + // be impossible (the scanner matches all bytes _somehow_) but we'll + // tolerate it and let the caller deal with it. + if cs < hclstrtok_first_final { + ret = append(ret, data[p:len(data)]) + } + + return ret +} diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_string_lit.rl b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_string_lit.rl new file mode 100644 index 000000000..f8ac11751 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_string_lit.rl @@ -0,0 +1,105 @@ + +package hclsyntax + +// This file is generated from scan_string_lit.rl. DO NOT EDIT. +%%{ + # (except you are actually in scan_string_lit.rl here, so edit away!) + + machine hclstrtok; + write data; +}%% + +func scanStringLit(data []byte, quoted bool) [][]byte { + var ret [][]byte + + %%{ + include UnicodeDerived "unicode_derived.rl"; + + UTF8Cont = 0x80 .. 0xBF; + AnyUTF8 = ( + 0x00..0x7F | + 0xC0..0xDF . UTF8Cont | + 0xE0..0xEF . UTF8Cont . UTF8Cont | + 0xF0..0xF7 . UTF8Cont . UTF8Cont . UTF8Cont + ); + BadUTF8 = any - AnyUTF8; + + Hex = ('0'..'9' | 'a'..'f' | 'A'..'F'); + + # Our goal with this patterns is to capture user intent as best as + # possible, even if the input is invalid. The caller will then verify + # whether each token is valid and generate suitable error messages + # if not. + UnicodeEscapeShort = "\\u" . Hex{0,4}; + UnicodeEscapeLong = "\\U" . Hex{0,8}; + UnicodeEscape = (UnicodeEscapeShort | UnicodeEscapeLong); + SimpleEscape = "\\" . (AnyUTF8 - ('U'|'u'))?; + TemplateEscape = ("$" . ("$" . ("{"?))?) | ("%" . ("%" . ("{"?))?); + Newline = ("\r\n" | "\r" | "\n"); + + action Begin { + // If te is behind p then we've skipped over some literal + // characters which we must now return. + if te < p { + ret = append(ret, data[te:p]) + } + ts = p; + } + action End { + te = p; + ret = append(ret, data[ts:te]); + } + + QuotedToken = (UnicodeEscape | SimpleEscape | TemplateEscape | Newline) >Begin %End; + UnquotedToken = (TemplateEscape | Newline) >Begin %End; + QuotedLiteral = (any - ("\\" | "$" | "%" | "\r" | "\n")); + UnquotedLiteral = (any - ("$" | "%" | "\r" | "\n")); + + quoted := (QuotedToken | QuotedLiteral)**; + unquoted := (UnquotedToken | UnquotedLiteral)**; + + }%% + + // Ragel state + p := 0 // "Pointer" into data + pe := len(data) // End-of-data "pointer" + ts := 0 + te := 0 + eof := pe + + var cs int // current state + switch { + case quoted: + cs = hclstrtok_en_quoted + default: + cs = hclstrtok_en_unquoted + } + + // Make Go compiler happy + _ = ts + _ = eof + + /*token := func () { + ret = append(ret, data[ts:te]) + }*/ + + %%{ + write init nocs; + write exec; + }%% + + if te < p { + // Collect any leftover literal characters at the end of the input + ret = append(ret, data[te:p]) + } + + // If we fall out here without being in a final state then we've + // encountered something that the scanner can't match, which should + // be impossible (the scanner matches all bytes _somehow_) but we'll + // tolerate it and let the caller deal with it. + if cs < hclstrtok_first_final { + ret = append(ret, data[p:len(data)]) + } + + return ret +} diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_tokens.go b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_tokens.go index a8ab57c3e..7d557c08d 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_tokens.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_tokens.go @@ -1,4 +1,5 @@ // line 1 "scan_tokens.rl" + package hclsyntax import ( @@ -9,8 +10,8 @@ import ( // This file is generated from scan_tokens.rl. DO NOT EDIT. -// line 14 "scan_tokens.go" -var _zcltok_actions []byte = []byte{ +// line 15 "scan_tokens.go" +var _hcltok_actions []byte = []byte{ 0, 1, 0, 1, 2, 1, 3, 1, 4, 1, 5, 1, 6, 1, 7, 1, 8, 1, 9, 1, 10, 1, 11, 1, 12, @@ -19,22 +20,24 @@ var _zcltok_actions []byte = []byte{ 1, 23, 1, 24, 1, 25, 1, 26, 1, 27, 1, 30, 1, 31, 1, 32, 1, 33, 1, 34, 1, 35, 1, 36, - 1, 37, 1, 38, 1, 39, 1, 45, - 1, 46, 1, 47, 1, 48, 1, 49, - 1, 50, 1, 51, 1, 52, 1, 53, - 1, 54, 1, 55, 1, 56, 1, 57, - 1, 58, 1, 59, 1, 60, 1, 61, - 1, 62, 1, 63, 1, 64, 1, 65, - 1, 66, 1, 67, 1, 68, 1, 69, - 1, 70, 1, 71, 1, 72, 1, 73, - 1, 74, 1, 75, 2, 0, 1, 2, - 3, 16, 2, 3, 17, 2, 3, 28, - 2, 3, 29, 2, 3, 40, 2, 3, - 41, 2, 3, 42, 2, 3, 43, 2, - 3, 44, + 1, 37, 1, 38, 1, 39, 1, 42, + 1, 43, 1, 44, 1, 45, 1, 46, + 1, 47, 1, 48, 1, 54, 1, 55, + 1, 56, 1, 57, 1, 58, 1, 59, + 1, 60, 1, 61, 1, 62, 1, 63, + 1, 64, 1, 65, 1, 66, 1, 67, + 1, 68, 1, 69, 1, 70, 1, 71, + 1, 72, 1, 73, 1, 74, 1, 75, + 1, 76, 1, 77, 1, 78, 1, 79, + 1, 80, 1, 81, 1, 82, 1, 83, + 2, 0, 1, 2, 3, 16, 2, 3, + 17, 2, 3, 28, 2, 3, 29, 2, + 3, 40, 2, 3, 41, 2, 3, 49, + 2, 3, 50, 2, 3, 51, 2, 3, + 52, 2, 3, 53, } -var _zcltok_key_offsets []int16 = []int16{ +var _hcltok_key_offsets []int16 = []int16{ 0, 0, 1, 2, 3, 5, 10, 14, 16, 57, 97, 143, 144, 148, 154, 154, 156, 158, 167, 173, 180, 181, 184, 185, @@ -153,20 +156,89 @@ var _zcltok_key_offsets []int16 = []int16{ 5909, 5913, 5917, 5922, 5926, 5928, 5935, 5939, 5947, 5951, 5952, 5954, 5956, 5958, 5960, 5962, 5963, 5964, 5966, 5968, 5970, 5971, 5972, 5973, - 5974, 5976, 5978, 5980, 5981, 5982, 6057, 6058, - 6059, 6060, 6061, 6062, 6063, 6064, 6066, 6067, - 6072, 6074, 6076, 6077, 6121, 6122, 6123, 6125, - 6130, 6134, 6134, 6136, 6138, 6149, 6159, 6167, - 6168, 6170, 6171, 6175, 6179, 6189, 6193, 6200, - 6211, 6218, 6222, 6228, 6239, 6271, 6320, 6335, - 6350, 6355, 6357, 6362, 6394, 6402, 6404, 6426, - 6448, 6450, 6466, 6482, 6497, 6506, 6520, 6534, - 6550, 6551, 6552, 6553, 6554, 6556, 6558, 6560, - 6574, 6588, 6589, 6590, 6592, 6594, 6596, 6610, - 6624, 6625, 6626, 6628, 6630, + 5974, 5976, 5978, 5980, 5981, 5982, 5986, 5992, + 5992, 5994, 5996, 6005, 6011, 6018, 6019, 6022, + 6023, 6027, 6032, 6041, 6045, 6049, 6057, 6059, + 6061, 6063, 6066, 6098, 6100, 6102, 6106, 6110, + 6113, 6124, 6137, 6156, 6169, 6185, 6197, 6213, + 6228, 6249, 6259, 6271, 6282, 6296, 6311, 6321, + 6333, 6342, 6354, 6356, 6360, 6381, 6390, 6400, + 6406, 6412, 6413, 6462, 6464, 6468, 6470, 6476, + 6483, 6491, 6498, 6501, 6507, 6511, 6515, 6517, + 6521, 6525, 6529, 6535, 6543, 6551, 6557, 6559, + 6563, 6565, 6571, 6575, 6579, 6583, 6587, 6592, + 6599, 6605, 6607, 6609, 6613, 6615, 6621, 6625, + 6629, 6639, 6644, 6658, 6673, 6675, 6683, 6685, + 6690, 6704, 6709, 6711, 6715, 6716, 6720, 6726, + 6732, 6742, 6752, 6763, 6771, 6774, 6777, 6781, + 6785, 6787, 6790, 6790, 6793, 6795, 6825, 6827, + 6829, 6833, 6838, 6842, 6847, 6849, 6851, 6853, + 6862, 6866, 6870, 6876, 6878, 6886, 6894, 6906, + 6909, 6915, 6919, 6921, 6925, 6945, 6947, 6949, + 6960, 6966, 6968, 6970, 6972, 6976, 6982, 6988, + 6990, 6995, 6999, 7001, 7009, 7027, 7067, 7077, + 7081, 7083, 7085, 7086, 7090, 7094, 7098, 7102, + 7106, 7111, 7115, 7119, 7123, 7125, 7127, 7131, + 7141, 7145, 7147, 7151, 7155, 7159, 7172, 7174, + 7176, 7180, 7182, 7186, 7188, 7190, 7220, 7224, + 7228, 7232, 7235, 7242, 7247, 7258, 7262, 7278, + 7292, 7296, 7301, 7305, 7309, 7315, 7317, 7323, + 7325, 7329, 7331, 7337, 7342, 7347, 7357, 7359, + 7361, 7365, 7369, 7371, 7384, 7386, 7390, 7394, + 7402, 7404, 7408, 7410, 7411, 7414, 7419, 7421, + 7423, 7427, 7429, 7433, 7439, 7459, 7465, 7471, + 7473, 7474, 7484, 7485, 7493, 7500, 7502, 7505, + 7507, 7509, 7511, 7516, 7520, 7524, 7529, 7539, + 7549, 7553, 7557, 7571, 7597, 7607, 7609, 7611, + 7614, 7616, 7619, 7621, 7625, 7627, 7628, 7632, + 7634, 7636, 7643, 7647, 7654, 7661, 7670, 7686, + 7698, 7716, 7727, 7739, 7747, 7765, 7773, 7803, + 7806, 7816, 7826, 7838, 7849, 7858, 7871, 7883, + 7887, 7893, 7920, 7929, 7932, 7937, 7943, 7948, + 7969, 7973, 7979, 7979, 7986, 7995, 8003, 8006, + 8010, 8016, 8022, 8025, 8029, 8036, 8042, 8051, + 8060, 8064, 8068, 8072, 8076, 8083, 8087, 8091, + 8101, 8107, 8111, 8117, 8121, 8124, 8130, 8136, + 8148, 8152, 8156, 8166, 8170, 8181, 8183, 8185, + 8189, 8201, 8206, 8230, 8234, 8240, 8262, 8271, + 8275, 8278, 8279, 8287, 8295, 8301, 8311, 8318, + 8336, 8339, 8342, 8350, 8356, 8360, 8364, 8368, + 8374, 8382, 8387, 8393, 8397, 8405, 8412, 8416, + 8423, 8429, 8437, 8445, 8451, 8457, 8468, 8472, + 8484, 8493, 8510, 8527, 8530, 8534, 8536, 8542, + 8544, 8548, 8563, 8567, 8571, 8575, 8579, 8583, + 8585, 8591, 8596, 8600, 8606, 8613, 8616, 8634, + 8636, 8681, 8687, 8693, 8697, 8701, 8707, 8711, + 8717, 8723, 8730, 8732, 8738, 8744, 8748, 8752, + 8760, 8773, 8779, 8786, 8794, 8800, 8809, 8815, + 8819, 8824, 8828, 8836, 8840, 8844, 8874, 8880, + 8886, 8892, 8898, 8905, 8911, 8918, 8923, 8933, + 8937, 8944, 8950, 8954, 8961, 8965, 8971, 8974, + 8978, 8982, 8986, 8990, 8995, 9000, 9004, 9015, + 9019, 9023, 9029, 9037, 9041, 9058, 9062, 9068, + 9078, 9084, 9090, 9093, 9098, 9107, 9111, 9115, + 9121, 9125, 9131, 9139, 9157, 9158, 9168, 9169, + 9178, 9186, 9188, 9191, 9193, 9195, 9197, 9202, + 9215, 9219, 9234, 9263, 9274, 9276, 9280, 9284, + 9289, 9293, 9295, 9302, 9306, 9314, 9318, 9393, + 9395, 9396, 9397, 9398, 9399, 9400, 9402, 9403, + 9408, 9410, 9412, 9413, 9457, 9458, 9459, 9461, + 9466, 9470, 9470, 9472, 9474, 9485, 9495, 9503, + 9504, 9506, 9507, 9511, 9515, 9525, 9529, 9536, + 9547, 9554, 9558, 9564, 9575, 9607, 9656, 9671, + 9686, 9691, 9693, 9698, 9730, 9738, 9740, 9762, + 9784, 9786, 9802, 9818, 9833, 9842, 9856, 9870, + 9886, 9887, 9888, 9889, 9890, 9892, 9894, 9896, + 9910, 9924, 9925, 9926, 9928, 9930, 9932, 9946, + 9960, 9961, 9962, 9964, 9966, 9968, 10016, 10060, + 10062, 10067, 10071, 10071, 10073, 10075, 10086, 10096, + 10104, 10105, 10107, 10108, 10112, 10116, 10126, 10130, + 10137, 10148, 10155, 10159, 10165, 10176, 10208, 10257, + 10272, 10287, 10292, 10294, 10299, 10331, 10339, 10341, + 10363, 10385, } -var _zcltok_trans_keys []byte = []byte{ +var _hcltok_trans_keys []byte = []byte{ 10, 46, 42, 42, 47, 46, 69, 101, 48, 57, 43, 45, 48, 57, 48, 57, 45, 194, 195, 198, 199, 203, 205, 206, @@ -186,7 +258,7 @@ var _zcltok_trans_keys []byte = []byte{ 234, 237, 239, 240, 243, 48, 57, 65, 90, 97, 122, 196, 218, 229, 236, 10, 170, 181, 183, 186, 128, 150, 152, 182, - 184, 255, 192, 255, 0, 127, 173, 130, + 184, 255, 192, 255, 128, 255, 173, 130, 133, 146, 159, 165, 171, 175, 255, 181, 190, 184, 185, 192, 255, 140, 134, 138, 142, 161, 163, 255, 182, 130, 136, 137, @@ -914,7 +986,424 @@ var _zcltok_trans_keys []byte = []byte{ 128, 191, 128, 191, 128, 191, 128, 191, 128, 191, 10, 123, 128, 191, 128, 191, 128, 191, 123, 123, 10, 123, 128, 191, - 128, 191, 128, 191, 123, 123, 9, 10, + 128, 191, 128, 191, 123, 123, 170, 181, + 183, 186, 128, 150, 152, 182, 184, 255, + 192, 255, 128, 255, 173, 130, 133, 146, + 159, 165, 171, 175, 255, 181, 190, 184, + 185, 192, 255, 140, 134, 138, 142, 161, + 163, 255, 182, 130, 136, 137, 176, 151, + 152, 154, 160, 190, 136, 144, 192, 255, + 135, 129, 130, 132, 133, 144, 170, 176, + 178, 144, 154, 160, 191, 128, 169, 174, + 255, 148, 169, 157, 158, 189, 190, 192, + 255, 144, 255, 139, 140, 178, 255, 186, + 128, 181, 160, 161, 162, 163, 164, 165, + 166, 167, 168, 169, 170, 171, 172, 173, + 174, 175, 176, 177, 178, 179, 180, 181, + 182, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 128, 173, 128, 155, 160, 180, + 182, 189, 148, 161, 163, 255, 176, 164, + 165, 132, 169, 177, 141, 142, 145, 146, + 179, 181, 186, 187, 158, 133, 134, 137, + 138, 143, 150, 152, 155, 164, 165, 178, + 255, 188, 129, 131, 133, 138, 143, 144, + 147, 168, 170, 176, 178, 179, 181, 182, + 184, 185, 190, 255, 157, 131, 134, 137, + 138, 142, 144, 146, 152, 159, 165, 182, + 255, 129, 131, 133, 141, 143, 145, 147, + 168, 170, 176, 178, 179, 181, 185, 188, + 255, 134, 138, 142, 143, 145, 159, 164, + 165, 176, 184, 186, 255, 129, 131, 133, + 140, 143, 144, 147, 168, 170, 176, 178, + 179, 181, 185, 188, 191, 177, 128, 132, + 135, 136, 139, 141, 150, 151, 156, 157, + 159, 163, 166, 175, 156, 130, 131, 133, + 138, 142, 144, 146, 149, 153, 154, 158, + 159, 163, 164, 168, 170, 174, 185, 190, + 191, 144, 151, 128, 130, 134, 136, 138, + 141, 166, 175, 128, 131, 133, 140, 142, + 144, 146, 168, 170, 185, 189, 255, 133, + 137, 151, 142, 148, 155, 159, 164, 165, + 176, 255, 128, 131, 133, 140, 142, 144, + 146, 168, 170, 179, 181, 185, 188, 191, + 158, 128, 132, 134, 136, 138, 141, 149, + 150, 160, 163, 166, 175, 177, 178, 129, + 131, 133, 140, 142, 144, 146, 186, 189, + 255, 133, 137, 143, 147, 152, 158, 164, + 165, 176, 185, 192, 255, 189, 130, 131, + 133, 150, 154, 177, 179, 187, 138, 150, + 128, 134, 143, 148, 152, 159, 166, 175, + 178, 179, 129, 186, 128, 142, 144, 153, + 132, 138, 141, 165, 167, 129, 130, 135, + 136, 148, 151, 153, 159, 161, 163, 170, + 171, 173, 185, 187, 189, 134, 128, 132, + 136, 141, 144, 153, 156, 159, 128, 181, + 183, 185, 152, 153, 160, 169, 190, 191, + 128, 135, 137, 172, 177, 191, 128, 132, + 134, 151, 153, 188, 134, 128, 129, 130, + 131, 137, 138, 139, 140, 141, 142, 143, + 144, 153, 154, 155, 156, 157, 158, 159, + 160, 161, 162, 163, 164, 165, 166, 167, + 168, 169, 170, 173, 175, 176, 177, 178, + 179, 181, 182, 183, 188, 189, 190, 191, + 132, 152, 172, 184, 185, 187, 128, 191, + 128, 137, 144, 255, 158, 159, 134, 187, + 136, 140, 142, 143, 137, 151, 153, 142, + 143, 158, 159, 137, 177, 142, 143, 182, + 183, 191, 255, 128, 130, 133, 136, 150, + 152, 255, 145, 150, 151, 155, 156, 160, + 168, 178, 255, 128, 143, 160, 255, 182, + 183, 190, 255, 129, 255, 173, 174, 192, + 255, 129, 154, 160, 255, 171, 173, 185, + 255, 128, 140, 142, 148, 160, 180, 128, + 147, 160, 172, 174, 176, 178, 179, 148, + 150, 152, 155, 158, 159, 170, 255, 139, + 141, 144, 153, 160, 255, 184, 255, 128, + 170, 176, 255, 182, 255, 128, 158, 160, + 171, 176, 187, 134, 173, 176, 180, 128, + 171, 176, 255, 138, 143, 155, 255, 128, + 155, 160, 255, 159, 189, 190, 192, 255, + 167, 128, 137, 144, 153, 176, 189, 140, + 143, 154, 170, 180, 255, 180, 255, 128, + 183, 128, 137, 141, 189, 128, 136, 144, + 146, 148, 182, 184, 185, 128, 181, 187, + 191, 150, 151, 158, 159, 152, 154, 156, + 158, 134, 135, 142, 143, 190, 255, 190, + 128, 180, 182, 188, 130, 132, 134, 140, + 144, 147, 150, 155, 160, 172, 178, 180, + 182, 188, 128, 129, 130, 131, 132, 133, + 134, 176, 177, 178, 179, 180, 181, 182, + 183, 191, 255, 129, 147, 149, 176, 178, + 190, 192, 255, 144, 156, 161, 144, 156, + 165, 176, 130, 135, 149, 164, 166, 168, + 138, 147, 152, 157, 170, 185, 188, 191, + 142, 133, 137, 160, 255, 137, 255, 128, + 174, 176, 255, 159, 165, 170, 180, 255, + 167, 173, 128, 165, 176, 255, 168, 174, + 176, 190, 192, 255, 128, 150, 160, 166, + 168, 174, 176, 182, 184, 190, 128, 134, + 136, 142, 144, 150, 152, 158, 160, 191, + 128, 129, 130, 131, 132, 133, 134, 135, + 144, 145, 255, 133, 135, 161, 175, 177, + 181, 184, 188, 160, 151, 152, 187, 192, + 255, 133, 173, 177, 255, 143, 159, 187, + 255, 176, 191, 182, 183, 184, 191, 192, + 255, 150, 255, 128, 146, 147, 148, 152, + 153, 154, 155, 156, 158, 159, 160, 161, + 162, 163, 164, 165, 166, 167, 168, 169, + 170, 171, 172, 173, 174, 175, 176, 129, + 255, 141, 255, 144, 189, 141, 143, 172, + 255, 191, 128, 175, 180, 189, 151, 159, + 162, 255, 175, 137, 138, 184, 255, 183, + 255, 168, 255, 128, 179, 188, 134, 143, + 154, 159, 184, 186, 190, 255, 128, 173, + 176, 255, 148, 159, 189, 255, 129, 142, + 154, 159, 191, 255, 128, 182, 128, 141, + 144, 153, 160, 182, 186, 255, 128, 130, + 155, 157, 160, 175, 178, 182, 129, 134, + 137, 142, 145, 150, 160, 166, 168, 174, + 176, 255, 155, 166, 175, 128, 170, 172, + 173, 176, 185, 158, 159, 160, 255, 164, + 175, 135, 138, 188, 255, 164, 169, 171, + 172, 173, 174, 175, 180, 181, 182, 183, + 184, 185, 187, 188, 189, 190, 191, 165, + 186, 174, 175, 154, 255, 190, 128, 134, + 147, 151, 157, 168, 170, 182, 184, 188, + 128, 129, 131, 132, 134, 255, 147, 255, + 190, 255, 144, 145, 136, 175, 188, 255, + 128, 143, 160, 175, 179, 180, 141, 143, + 176, 180, 182, 255, 189, 255, 191, 144, + 153, 161, 186, 129, 154, 166, 255, 191, + 255, 130, 135, 138, 143, 146, 151, 154, + 156, 144, 145, 146, 147, 148, 150, 151, + 152, 155, 157, 158, 160, 170, 171, 172, + 175, 161, 169, 128, 129, 130, 131, 133, + 135, 138, 139, 140, 141, 142, 143, 144, + 145, 146, 147, 148, 149, 152, 156, 157, + 160, 161, 162, 163, 164, 166, 168, 169, + 170, 171, 172, 173, 174, 176, 177, 153, + 155, 178, 179, 128, 139, 141, 166, 168, + 186, 188, 189, 191, 255, 142, 143, 158, + 255, 187, 255, 128, 180, 189, 128, 156, + 160, 255, 145, 159, 161, 255, 128, 159, + 176, 255, 139, 143, 187, 255, 128, 157, + 160, 255, 144, 132, 135, 150, 255, 158, + 159, 170, 175, 148, 151, 188, 255, 128, + 167, 176, 255, 164, 255, 183, 255, 128, + 149, 160, 167, 136, 188, 128, 133, 138, + 181, 183, 184, 191, 255, 150, 159, 183, + 255, 128, 158, 160, 178, 180, 181, 128, + 149, 160, 185, 128, 183, 190, 191, 191, + 128, 131, 133, 134, 140, 147, 149, 151, + 153, 179, 184, 186, 160, 188, 128, 156, + 128, 135, 137, 166, 128, 181, 128, 149, + 160, 178, 128, 145, 128, 178, 129, 130, + 131, 132, 133, 135, 136, 138, 139, 140, + 141, 144, 145, 146, 147, 150, 151, 152, + 153, 154, 155, 156, 162, 163, 171, 176, + 177, 178, 128, 134, 135, 165, 176, 190, + 144, 168, 176, 185, 128, 180, 182, 191, + 182, 144, 179, 155, 133, 137, 141, 143, + 157, 255, 190, 128, 145, 147, 183, 136, + 128, 134, 138, 141, 143, 157, 159, 168, + 176, 255, 171, 175, 186, 255, 128, 131, + 133, 140, 143, 144, 147, 168, 170, 176, + 178, 179, 181, 185, 188, 191, 144, 151, + 128, 132, 135, 136, 139, 141, 157, 163, + 166, 172, 176, 180, 128, 138, 144, 153, + 134, 136, 143, 154, 255, 128, 181, 184, + 255, 129, 151, 158, 255, 129, 131, 133, + 143, 154, 255, 128, 137, 128, 153, 157, + 171, 176, 185, 160, 255, 170, 190, 192, + 255, 128, 184, 128, 136, 138, 182, 184, + 191, 128, 144, 153, 178, 255, 168, 144, + 145, 183, 255, 128, 142, 145, 149, 129, + 141, 144, 146, 147, 148, 175, 255, 132, + 255, 128, 144, 129, 143, 144, 153, 145, + 152, 135, 255, 160, 168, 169, 171, 172, + 173, 174, 188, 189, 190, 191, 161, 167, + 185, 255, 128, 158, 160, 169, 144, 173, + 176, 180, 128, 131, 144, 153, 163, 183, + 189, 255, 144, 255, 133, 143, 191, 255, + 143, 159, 160, 128, 129, 255, 159, 160, + 171, 172, 255, 173, 255, 179, 255, 128, + 176, 177, 178, 128, 129, 171, 175, 189, + 255, 128, 136, 144, 153, 157, 158, 133, + 134, 137, 144, 145, 146, 147, 148, 149, + 154, 155, 156, 157, 158, 159, 168, 169, + 170, 150, 153, 165, 169, 173, 178, 187, + 255, 131, 132, 140, 169, 174, 255, 130, + 132, 149, 157, 173, 186, 188, 160, 161, + 163, 164, 167, 168, 132, 134, 149, 157, + 186, 139, 140, 191, 255, 134, 128, 132, + 138, 144, 146, 255, 166, 167, 129, 155, + 187, 149, 181, 143, 175, 137, 169, 131, + 140, 141, 192, 255, 128, 182, 187, 255, + 173, 180, 182, 255, 132, 155, 159, 161, + 175, 128, 160, 163, 164, 165, 184, 185, + 186, 161, 162, 128, 134, 136, 152, 155, + 161, 163, 164, 166, 170, 133, 143, 151, + 255, 139, 143, 154, 255, 164, 167, 185, + 187, 128, 131, 133, 159, 161, 162, 169, + 178, 180, 183, 130, 135, 137, 139, 148, + 151, 153, 155, 157, 159, 164, 190, 141, + 143, 145, 146, 161, 162, 167, 170, 172, + 178, 180, 183, 185, 188, 128, 137, 139, + 155, 161, 163, 165, 169, 171, 187, 155, + 156, 151, 255, 156, 157, 160, 181, 255, + 186, 187, 255, 162, 255, 160, 168, 161, + 167, 158, 255, 160, 132, 135, 133, 134, + 176, 255, 128, 191, 154, 164, 168, 128, + 149, 150, 191, 128, 152, 153, 191, 181, + 128, 159, 160, 189, 190, 191, 189, 128, + 131, 132, 185, 186, 191, 144, 128, 151, + 152, 161, 162, 176, 177, 255, 169, 177, + 129, 132, 141, 142, 145, 146, 179, 181, + 186, 188, 190, 191, 192, 255, 142, 158, + 128, 155, 156, 161, 162, 175, 176, 177, + 178, 191, 169, 177, 180, 183, 128, 132, + 133, 138, 139, 142, 143, 144, 145, 146, + 147, 185, 186, 191, 157, 128, 152, 153, + 158, 159, 177, 178, 180, 181, 191, 142, + 146, 169, 177, 180, 189, 128, 132, 133, + 185, 186, 191, 144, 185, 128, 159, 160, + 161, 162, 191, 169, 177, 180, 189, 128, + 132, 133, 140, 141, 142, 143, 144, 145, + 146, 147, 185, 186, 191, 158, 177, 128, + 155, 156, 161, 162, 191, 131, 145, 155, + 157, 128, 132, 133, 138, 139, 141, 142, + 149, 150, 152, 153, 159, 160, 162, 163, + 164, 165, 167, 168, 170, 171, 173, 174, + 185, 186, 191, 144, 128, 191, 141, 145, + 169, 189, 128, 132, 133, 185, 186, 191, + 128, 151, 152, 154, 155, 159, 160, 161, + 162, 191, 128, 141, 145, 169, 180, 189, + 129, 132, 133, 185, 186, 191, 158, 128, + 159, 160, 161, 162, 176, 177, 178, 179, + 191, 141, 145, 189, 128, 132, 133, 186, + 187, 191, 142, 128, 147, 148, 150, 151, + 158, 159, 161, 162, 185, 186, 191, 178, + 188, 128, 132, 133, 150, 151, 153, 154, + 189, 190, 191, 128, 134, 135, 191, 128, + 177, 129, 179, 180, 191, 128, 131, 137, + 141, 152, 160, 164, 166, 172, 177, 189, + 129, 132, 133, 134, 135, 138, 139, 147, + 148, 167, 168, 169, 170, 179, 180, 191, + 133, 128, 134, 135, 155, 156, 159, 160, + 191, 128, 129, 191, 136, 128, 172, 173, + 191, 128, 135, 136, 140, 141, 191, 191, + 128, 170, 171, 190, 161, 128, 143, 144, + 149, 150, 153, 154, 157, 158, 164, 165, + 166, 167, 173, 174, 176, 177, 180, 181, + 255, 130, 141, 143, 159, 134, 187, 136, + 140, 142, 143, 137, 151, 153, 142, 143, + 158, 159, 137, 177, 191, 142, 143, 182, + 183, 192, 255, 129, 151, 128, 133, 134, + 135, 136, 255, 145, 150, 151, 155, 191, + 192, 255, 128, 143, 144, 159, 160, 255, + 182, 183, 190, 191, 192, 255, 128, 129, + 255, 173, 174, 192, 255, 128, 129, 154, + 155, 159, 160, 255, 171, 173, 185, 191, + 192, 255, 141, 128, 145, 146, 159, 160, + 177, 178, 191, 173, 128, 145, 146, 159, + 160, 176, 177, 191, 128, 179, 180, 191, + 151, 156, 128, 191, 128, 159, 160, 255, + 184, 191, 192, 255, 169, 128, 170, 171, + 175, 176, 255, 182, 191, 192, 255, 128, + 158, 159, 191, 128, 143, 144, 173, 174, + 175, 176, 180, 181, 191, 128, 171, 172, + 175, 176, 255, 138, 191, 192, 255, 128, + 150, 151, 159, 160, 255, 149, 191, 192, + 255, 167, 128, 191, 128, 132, 133, 179, + 180, 191, 128, 132, 133, 139, 140, 191, + 128, 130, 131, 160, 161, 173, 174, 175, + 176, 185, 186, 255, 166, 191, 192, 255, + 128, 163, 164, 191, 128, 140, 141, 143, + 144, 153, 154, 189, 190, 191, 128, 136, + 137, 191, 173, 128, 168, 169, 177, 178, + 180, 181, 182, 183, 191, 0, 127, 192, + 255, 150, 151, 158, 159, 152, 154, 156, + 158, 134, 135, 142, 143, 190, 191, 192, + 255, 181, 189, 191, 128, 190, 133, 181, + 128, 129, 130, 140, 141, 143, 144, 147, + 148, 149, 150, 155, 156, 159, 160, 172, + 173, 177, 178, 188, 189, 191, 177, 191, + 128, 190, 128, 143, 144, 156, 157, 191, + 130, 135, 148, 164, 166, 168, 128, 137, + 138, 149, 150, 151, 152, 157, 158, 169, + 170, 185, 186, 187, 188, 191, 142, 128, + 132, 133, 137, 138, 159, 160, 255, 137, + 191, 192, 255, 175, 128, 255, 159, 165, + 170, 175, 177, 180, 191, 192, 255, 166, + 173, 128, 167, 168, 175, 176, 255, 168, + 174, 176, 191, 192, 255, 167, 175, 183, + 191, 128, 150, 151, 159, 160, 190, 135, + 143, 151, 128, 158, 159, 191, 128, 132, + 133, 135, 136, 160, 161, 169, 170, 176, + 177, 181, 182, 183, 184, 188, 189, 191, + 160, 151, 154, 187, 192, 255, 128, 132, + 133, 173, 174, 176, 177, 255, 143, 159, + 187, 191, 192, 255, 128, 175, 176, 191, + 150, 191, 192, 255, 141, 191, 192, 255, + 128, 143, 144, 189, 190, 191, 141, 143, + 160, 169, 172, 191, 192, 255, 191, 128, + 174, 175, 190, 128, 157, 158, 159, 160, + 255, 176, 191, 192, 255, 128, 150, 151, + 159, 160, 161, 162, 255, 175, 137, 138, + 184, 191, 192, 255, 128, 182, 183, 255, + 130, 134, 139, 163, 191, 192, 255, 128, + 129, 130, 179, 180, 191, 187, 189, 128, + 177, 178, 183, 184, 191, 128, 137, 138, + 165, 166, 175, 176, 255, 135, 159, 189, + 191, 192, 255, 128, 131, 132, 178, 179, + 191, 143, 165, 191, 128, 159, 160, 175, + 176, 185, 186, 190, 128, 168, 169, 191, + 131, 186, 128, 139, 140, 159, 160, 182, + 183, 189, 190, 255, 176, 178, 180, 183, + 184, 190, 191, 192, 255, 129, 128, 130, + 131, 154, 155, 157, 158, 159, 160, 170, + 171, 177, 178, 180, 181, 191, 128, 167, + 175, 129, 134, 135, 136, 137, 142, 143, + 144, 145, 150, 151, 159, 160, 255, 155, + 166, 175, 128, 162, 163, 191, 164, 175, + 135, 138, 188, 191, 192, 255, 174, 175, + 154, 191, 192, 255, 157, 169, 183, 189, + 191, 128, 134, 135, 146, 147, 151, 152, + 158, 159, 190, 130, 133, 128, 255, 178, + 191, 192, 255, 128, 146, 147, 255, 190, + 191, 192, 255, 128, 143, 144, 255, 144, + 145, 136, 175, 188, 191, 192, 255, 181, + 128, 175, 176, 255, 189, 191, 192, 255, + 128, 160, 161, 186, 187, 191, 128, 129, + 154, 155, 165, 166, 255, 191, 192, 255, + 128, 129, 130, 135, 136, 137, 138, 143, + 144, 145, 146, 151, 152, 153, 154, 156, + 157, 191, 128, 191, 128, 129, 130, 131, + 133, 138, 139, 140, 141, 142, 143, 144, + 145, 146, 147, 148, 149, 152, 156, 157, + 160, 161, 162, 163, 164, 166, 168, 169, + 170, 171, 172, 173, 174, 176, 177, 132, + 151, 153, 155, 158, 175, 178, 179, 180, + 191, 140, 167, 187, 190, 128, 255, 142, + 143, 158, 191, 192, 255, 187, 191, 192, + 255, 128, 180, 181, 191, 128, 156, 157, + 159, 160, 255, 145, 191, 192, 255, 128, + 159, 160, 175, 176, 255, 139, 143, 182, + 191, 192, 255, 144, 132, 135, 150, 191, + 192, 255, 158, 175, 148, 151, 188, 191, + 192, 255, 128, 167, 168, 175, 176, 255, + 164, 191, 192, 255, 183, 191, 192, 255, + 128, 149, 150, 159, 160, 167, 168, 191, + 136, 182, 188, 128, 133, 134, 137, 138, + 184, 185, 190, 191, 255, 150, 159, 183, + 191, 192, 255, 179, 128, 159, 160, 181, + 182, 191, 128, 149, 150, 159, 160, 185, + 186, 191, 128, 183, 184, 189, 190, 191, + 128, 148, 152, 129, 143, 144, 179, 180, + 191, 128, 159, 160, 188, 189, 191, 128, + 156, 157, 191, 136, 128, 164, 165, 191, + 128, 181, 182, 191, 128, 149, 150, 159, + 160, 178, 179, 191, 128, 145, 146, 191, + 128, 178, 179, 191, 128, 130, 131, 132, + 133, 134, 135, 136, 138, 139, 140, 141, + 144, 145, 146, 147, 150, 151, 152, 153, + 154, 156, 162, 163, 171, 176, 177, 178, + 129, 191, 128, 130, 131, 183, 184, 191, + 128, 130, 131, 175, 176, 191, 128, 143, + 144, 168, 169, 191, 128, 130, 131, 166, + 167, 191, 182, 128, 143, 144, 178, 179, + 191, 128, 130, 131, 178, 179, 191, 128, + 154, 156, 129, 132, 133, 191, 146, 128, + 171, 172, 191, 135, 137, 142, 158, 128, + 168, 169, 175, 176, 255, 159, 191, 192, + 255, 144, 128, 156, 157, 161, 162, 191, + 128, 134, 135, 138, 139, 191, 128, 175, + 176, 191, 134, 128, 131, 132, 135, 136, + 191, 128, 174, 175, 191, 128, 151, 152, + 155, 156, 191, 132, 128, 191, 128, 170, + 171, 191, 128, 153, 154, 191, 160, 190, + 192, 255, 128, 184, 185, 191, 137, 128, + 174, 175, 191, 128, 129, 177, 178, 255, + 144, 191, 192, 255, 128, 142, 143, 144, + 145, 146, 149, 129, 148, 150, 191, 175, + 191, 192, 255, 132, 191, 192, 255, 128, + 144, 129, 143, 145, 191, 144, 153, 128, + 143, 145, 152, 154, 191, 135, 191, 192, + 255, 160, 168, 169, 171, 172, 173, 174, + 188, 189, 190, 191, 128, 159, 161, 167, + 170, 187, 185, 191, 192, 255, 128, 143, + 144, 173, 174, 191, 128, 131, 132, 162, + 163, 183, 184, 188, 189, 255, 133, 143, + 145, 191, 192, 255, 128, 146, 147, 159, + 160, 191, 160, 128, 191, 128, 129, 191, + 192, 255, 159, 160, 171, 128, 170, 172, + 191, 192, 255, 173, 191, 192, 255, 179, + 191, 192, 255, 128, 176, 177, 178, 129, + 191, 128, 129, 130, 191, 171, 175, 189, + 191, 192, 255, 128, 136, 137, 143, 144, + 153, 154, 191, 144, 145, 146, 147, 148, + 149, 154, 155, 156, 157, 158, 159, 128, + 143, 150, 153, 160, 191, 149, 157, 173, + 186, 188, 160, 161, 163, 164, 167, 168, + 132, 134, 149, 157, 186, 191, 139, 140, + 192, 255, 133, 145, 128, 134, 135, 137, + 138, 255, 166, 167, 129, 155, 187, 149, + 181, 143, 175, 137, 169, 131, 140, 191, + 192, 255, 160, 163, 164, 165, 184, 185, + 186, 128, 159, 161, 162, 166, 191, 133, + 191, 192, 255, 132, 160, 163, 167, 179, + 184, 186, 128, 164, 165, 168, 169, 187, + 188, 191, 130, 135, 137, 139, 144, 147, + 151, 153, 155, 157, 159, 163, 171, 179, + 184, 189, 191, 128, 140, 141, 148, 149, + 160, 161, 164, 165, 166, 167, 190, 138, + 164, 170, 128, 155, 156, 160, 161, 187, + 188, 191, 128, 191, 155, 156, 128, 191, + 151, 191, 192, 255, 156, 157, 160, 128, + 191, 181, 191, 192, 255, 158, 159, 186, + 128, 185, 187, 191, 192, 255, 162, 191, + 192, 255, 160, 168, 128, 159, 161, 167, + 169, 191, 158, 191, 192, 255, 9, 10, 13, 32, 33, 34, 35, 38, 46, 47, 60, 61, 62, 64, 92, 95, 123, 124, 125, 126, 127, 194, 195, 198, 199, 203, @@ -924,7 +1413,7 @@ var _zcltok_trans_keys []byte = []byte{ 238, 239, 240, 0, 39, 40, 45, 48, 57, 58, 63, 65, 90, 91, 96, 97, 122, 192, 193, 196, 218, 229, 236, 241, - 247, 9, 10, 32, 61, 10, 38, 46, + 247, 9, 32, 10, 61, 10, 38, 46, 42, 47, 42, 46, 69, 101, 48, 57, 60, 61, 61, 62, 61, 45, 95, 194, 195, 198, 199, 203, 204, 205, 206, 207, @@ -996,9 +1485,62 @@ var _zcltok_trans_keys []byte = []byte{ 248, 255, 10, 13, 36, 37, 128, 191, 192, 223, 224, 239, 240, 247, 248, 255, 126, 126, 128, 191, 128, 191, 128, 191, + 194, 195, 198, 199, 203, 204, 205, 206, + 207, 210, 212, 213, 214, 215, 216, 217, + 219, 220, 221, 222, 223, 224, 225, 226, + 227, 228, 233, 234, 237, 238, 239, 240, + 65, 90, 97, 122, 128, 191, 192, 193, + 196, 218, 229, 236, 241, 247, 248, 255, + 45, 95, 194, 195, 198, 199, 203, 204, + 205, 206, 207, 210, 212, 213, 214, 215, + 216, 217, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 233, 234, 237, 239, + 240, 243, 48, 57, 65, 90, 97, 122, + 196, 218, 229, 236, 128, 191, 170, 181, + 186, 128, 191, 151, 183, 128, 255, 192, + 255, 0, 127, 173, 130, 133, 146, 159, + 165, 171, 175, 191, 192, 255, 181, 190, + 128, 175, 176, 183, 184, 185, 186, 191, + 134, 139, 141, 162, 128, 135, 136, 255, + 182, 130, 137, 176, 151, 152, 154, 160, + 136, 191, 192, 255, 128, 143, 144, 170, + 171, 175, 176, 178, 179, 191, 128, 159, + 160, 191, 176, 128, 138, 139, 173, 174, + 255, 148, 150, 164, 167, 173, 176, 185, + 189, 190, 192, 255, 144, 128, 145, 146, + 175, 176, 191, 128, 140, 141, 255, 166, + 176, 178, 191, 192, 255, 186, 128, 137, + 138, 170, 171, 179, 180, 181, 182, 191, + 160, 161, 162, 164, 165, 166, 167, 168, + 169, 170, 171, 172, 173, 174, 175, 176, + 177, 178, 179, 180, 181, 182, 183, 184, + 185, 186, 187, 188, 189, 190, 128, 191, + 128, 129, 130, 131, 137, 138, 139, 140, + 141, 142, 143, 144, 153, 154, 155, 156, + 157, 158, 159, 160, 161, 162, 163, 164, + 165, 166, 167, 168, 169, 170, 171, 172, + 173, 174, 175, 176, 177, 178, 179, 180, + 182, 183, 184, 188, 189, 190, 191, 132, + 187, 129, 130, 132, 133, 134, 176, 177, + 178, 179, 180, 181, 182, 183, 128, 191, + 128, 129, 130, 131, 132, 133, 134, 135, + 144, 136, 143, 145, 191, 192, 255, 182, + 183, 184, 128, 191, 128, 191, 191, 128, + 190, 192, 255, 128, 146, 147, 148, 152, + 153, 154, 155, 156, 158, 159, 160, 161, + 162, 163, 164, 165, 166, 167, 168, 169, + 170, 171, 172, 173, 174, 175, 176, 129, + 191, 192, 255, 158, 159, 128, 157, 160, + 191, 192, 255, 128, 191, 164, 169, 171, + 172, 173, 174, 175, 180, 181, 182, 183, + 184, 185, 187, 188, 189, 190, 191, 128, + 163, 165, 186, 144, 145, 146, 147, 148, + 150, 151, 152, 155, 157, 158, 160, 170, + 171, 172, 175, 128, 159, 161, 169, 173, + 191, 128, 191, } -var _zcltok_single_lengths []byte = []byte{ +var _hcltok_single_lengths []byte = []byte{ 0, 1, 1, 1, 2, 3, 2, 0, 31, 30, 36, 1, 4, 0, 0, 0, 0, 1, 2, 1, 1, 1, 1, 0, @@ -1117,7 +1659,71 @@ var _zcltok_single_lengths []byte = []byte{ 2, 0, 3, 0, 0, 1, 0, 2, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, - 0, 0, 0, 1, 1, 53, 1, 1, + 0, 0, 0, 1, 1, 4, 0, 0, + 0, 0, 1, 2, 1, 1, 1, 1, + 0, 1, 1, 0, 0, 2, 0, 0, + 0, 1, 32, 0, 0, 0, 0, 1, + 3, 1, 1, 1, 0, 2, 0, 1, + 1, 2, 0, 3, 0, 1, 0, 2, + 1, 2, 0, 0, 5, 1, 4, 0, + 0, 1, 43, 0, 0, 0, 2, 3, + 2, 1, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 1, 0, 15, 0, 0, 0, 1, + 6, 1, 0, 0, 1, 0, 2, 0, + 0, 0, 9, 0, 1, 1, 0, 0, + 0, 3, 0, 1, 0, 28, 0, 0, + 0, 1, 0, 1, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 1, + 0, 2, 0, 0, 18, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 16, 36, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 2, + 0, 0, 0, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 28, 0, 0, + 0, 1, 1, 1, 1, 0, 0, 2, + 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 1, 4, 0, 0, + 2, 2, 0, 11, 0, 0, 0, 0, + 0, 0, 0, 1, 1, 3, 0, 0, + 4, 0, 0, 0, 18, 0, 0, 0, + 1, 4, 1, 4, 1, 0, 3, 2, + 2, 2, 1, 0, 0, 1, 8, 0, + 0, 0, 4, 12, 0, 2, 0, 3, + 0, 1, 0, 2, 0, 1, 2, 0, + 0, 3, 0, 1, 1, 1, 2, 2, + 4, 1, 6, 2, 4, 2, 4, 1, + 4, 0, 6, 1, 3, 1, 2, 0, + 2, 11, 1, 1, 1, 0, 1, 1, + 0, 2, 0, 3, 3, 2, 1, 0, + 0, 0, 1, 0, 1, 0, 1, 1, + 0, 2, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 0, + 4, 3, 2, 2, 0, 6, 1, 0, + 1, 1, 0, 2, 0, 4, 3, 0, + 1, 1, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 0, 1, 0, 3, + 0, 2, 0, 0, 0, 3, 0, 2, + 1, 1, 3, 1, 0, 0, 0, 0, + 0, 5, 2, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 1, 1, 0, 0, + 35, 4, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 0, 0, 0, 0, + 3, 0, 1, 0, 0, 3, 0, 0, + 1, 0, 0, 0, 0, 28, 0, 0, + 0, 0, 1, 0, 3, 1, 4, 0, + 1, 0, 0, 1, 0, 0, 1, 0, + 0, 0, 0, 1, 1, 0, 7, 0, + 0, 2, 2, 0, 11, 0, 0, 0, + 0, 0, 1, 1, 3, 0, 0, 4, + 0, 0, 0, 12, 1, 4, 1, 5, + 2, 0, 3, 2, 2, 2, 1, 7, + 0, 7, 17, 3, 0, 2, 0, 3, + 0, 0, 1, 0, 2, 0, 53, 2, 1, 1, 1, 1, 1, 2, 1, 3, 2, 2, 1, 34, 1, 1, 0, 3, 2, 0, 0, 0, 1, 2, 4, 1, @@ -1127,10 +1733,15 @@ var _zcltok_single_lengths []byte = []byte{ 0, 6, 4, 3, 1, 4, 4, 4, 1, 1, 1, 1, 0, 0, 0, 4, 2, 1, 1, 0, 0, 0, 4, 2, - 1, 1, 0, 0, 0, + 1, 1, 0, 0, 0, 32, 34, 0, + 3, 2, 0, 0, 0, 1, 2, 4, + 1, 0, 1, 0, 0, 0, 0, 1, + 1, 1, 0, 0, 1, 30, 47, 13, + 9, 3, 0, 1, 28, 2, 0, 18, + 16, 0, } -var _zcltok_range_lengths []byte = []byte{ +var _hcltok_range_lengths []byte = []byte{ 0, 0, 0, 0, 0, 1, 1, 1, 5, 5, 5, 0, 0, 3, 0, 1, 1, 4, 2, 3, 0, 1, 0, 2, @@ -1249,7 +1860,71 @@ var _zcltok_range_lengths []byte = []byte{ 1, 2, 1, 2, 1, 3, 2, 3, 2, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, - 1, 1, 1, 0, 0, 11, 0, 0, + 1, 1, 1, 0, 0, 0, 3, 0, + 1, 1, 4, 2, 3, 0, 1, 0, + 2, 2, 4, 2, 2, 3, 1, 1, + 1, 1, 0, 1, 1, 2, 2, 1, + 4, 6, 9, 6, 8, 5, 8, 7, + 10, 4, 6, 4, 7, 7, 5, 5, + 4, 5, 1, 2, 8, 4, 3, 3, + 3, 0, 3, 1, 2, 1, 2, 2, + 3, 3, 1, 3, 2, 2, 1, 2, + 2, 2, 3, 4, 4, 3, 1, 2, + 1, 3, 2, 2, 2, 2, 2, 3, + 3, 1, 1, 2, 1, 3, 2, 2, + 3, 2, 7, 0, 1, 4, 1, 2, + 4, 2, 1, 2, 0, 2, 2, 3, + 5, 5, 1, 4, 1, 1, 2, 2, + 1, 0, 0, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 1, 1, 1, 4, + 2, 2, 3, 1, 4, 4, 6, 1, + 3, 1, 1, 2, 1, 1, 1, 5, + 3, 1, 1, 1, 2, 3, 3, 1, + 2, 2, 1, 4, 1, 2, 5, 2, + 1, 1, 0, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 1, 1, 2, 4, + 2, 1, 2, 2, 2, 6, 1, 1, + 2, 1, 2, 1, 1, 1, 2, 2, + 2, 1, 3, 2, 5, 2, 8, 6, + 2, 2, 2, 2, 3, 1, 3, 1, + 2, 1, 3, 2, 2, 3, 1, 1, + 1, 1, 1, 1, 1, 2, 2, 4, + 1, 2, 1, 0, 1, 1, 1, 1, + 0, 1, 2, 3, 1, 3, 3, 1, + 0, 3, 0, 2, 3, 1, 0, 0, + 0, 0, 2, 2, 2, 2, 1, 5, + 2, 2, 5, 7, 5, 0, 1, 0, + 1, 1, 1, 1, 1, 0, 1, 1, + 1, 2, 2, 3, 3, 4, 7, 5, + 7, 5, 3, 3, 7, 3, 13, 1, + 3, 5, 3, 5, 3, 6, 5, 2, + 2, 8, 4, 1, 2, 3, 2, 10, + 2, 2, 0, 2, 3, 3, 1, 2, + 3, 3, 1, 2, 3, 3, 4, 4, + 2, 1, 2, 2, 3, 2, 2, 5, + 3, 2, 3, 2, 1, 3, 3, 6, + 2, 2, 5, 2, 5, 1, 1, 2, + 4, 1, 11, 1, 3, 8, 4, 2, + 1, 0, 4, 3, 3, 3, 2, 9, + 1, 1, 4, 3, 2, 2, 2, 3, + 4, 2, 3, 2, 4, 3, 2, 2, + 3, 3, 4, 3, 3, 4, 2, 5, + 4, 8, 7, 1, 2, 1, 3, 1, + 2, 5, 1, 2, 2, 2, 2, 1, + 3, 2, 2, 3, 3, 1, 9, 1, + 5, 1, 3, 2, 2, 3, 2, 3, + 3, 3, 1, 3, 3, 2, 2, 4, + 5, 3, 3, 4, 3, 3, 3, 2, + 2, 2, 4, 2, 2, 1, 3, 3, + 3, 3, 3, 3, 2, 2, 3, 2, + 3, 3, 2, 3, 2, 3, 1, 2, + 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 3, 2, 3, 2, 3, 5, + 3, 3, 1, 2, 3, 2, 2, 1, + 2, 3, 4, 3, 0, 3, 0, 2, + 3, 1, 0, 0, 0, 0, 2, 3, + 2, 4, 6, 4, 1, 1, 2, 1, + 2, 1, 3, 2, 3, 2, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 1, 1, 1, 0, 1, 1, 5, 4, 2, 0, @@ -1259,10 +1934,15 @@ var _zcltok_range_lengths []byte = []byte{ 1, 5, 6, 6, 4, 5, 5, 6, 0, 0, 0, 0, 1, 1, 1, 5, 6, 0, 0, 1, 1, 1, 5, 6, - 0, 0, 1, 1, 1, + 0, 0, 1, 1, 1, 8, 5, 1, + 1, 1, 0, 1, 1, 5, 4, 2, + 0, 1, 0, 2, 2, 5, 2, 3, + 5, 3, 2, 3, 5, 1, 1, 1, + 3, 1, 1, 2, 2, 3, 1, 2, + 3, 1, } -var _zcltok_index_offsets []int16 = []int16{ +var _hcltok_index_offsets []int16 = []int16{ 0, 0, 2, 4, 6, 9, 14, 18, 20, 57, 93, 135, 137, 142, 146, 147, 149, 151, 157, 162, 167, 169, 172, 174, @@ -1381,20 +2061,89 @@ var _zcltok_index_offsets []int16 = []int16{ 4692, 4696, 4699, 4704, 4707, 4709, 4714, 4717, 4723, 4726, 4728, 4730, 4732, 4734, 4736, 4738, 4740, 4742, 4744, 4746, 4748, 4750, 4752, 4754, - 4756, 4758, 4760, 4762, 4764, 4766, 4831, 4833, - 4835, 4837, 4839, 4841, 4843, 4845, 4848, 4850, - 4855, 4858, 4861, 4863, 4903, 4905, 4907, 4909, - 4914, 4918, 4919, 4921, 4923, 4930, 4937, 4944, - 4946, 4948, 4950, 4953, 4956, 4962, 4965, 4970, - 4977, 4982, 4985, 4989, 4996, 5028, 5077, 5092, - 5105, 5110, 5112, 5116, 5147, 5153, 5155, 5176, - 5196, 5198, 5210, 5221, 5231, 5237, 5247, 5257, - 5268, 5270, 5272, 5274, 5276, 5278, 5280, 5282, - 5292, 5301, 5303, 5305, 5307, 5309, 5311, 5321, - 5330, 5332, 5334, 5336, 5338, + 4756, 4758, 4760, 4762, 4764, 4766, 4771, 4775, + 4776, 4778, 4780, 4786, 4791, 4796, 4798, 4801, + 4803, 4806, 4810, 4816, 4819, 4822, 4828, 4830, + 4832, 4834, 4837, 4870, 4872, 4874, 4877, 4880, + 4883, 4891, 4899, 4910, 4918, 4927, 4935, 4944, + 4953, 4965, 4972, 4979, 4987, 4995, 5004, 5010, + 5018, 5024, 5032, 5034, 5037, 5051, 5057, 5065, + 5069, 5073, 5075, 5122, 5124, 5127, 5129, 5134, + 5140, 5146, 5151, 5154, 5158, 5161, 5164, 5166, + 5169, 5172, 5175, 5179, 5184, 5189, 5193, 5195, + 5198, 5200, 5204, 5207, 5210, 5213, 5216, 5220, + 5225, 5229, 5231, 5233, 5236, 5238, 5242, 5245, + 5248, 5256, 5260, 5268, 5284, 5286, 5291, 5293, + 5297, 5308, 5312, 5314, 5317, 5319, 5322, 5327, + 5331, 5337, 5343, 5354, 5359, 5362, 5365, 5368, + 5371, 5373, 5377, 5378, 5381, 5383, 5413, 5415, + 5417, 5420, 5424, 5427, 5431, 5433, 5435, 5437, + 5443, 5446, 5449, 5453, 5455, 5460, 5465, 5472, + 5475, 5479, 5483, 5485, 5488, 5508, 5510, 5512, + 5519, 5523, 5525, 5527, 5529, 5532, 5536, 5540, + 5542, 5546, 5549, 5551, 5556, 5574, 5613, 5619, + 5622, 5624, 5626, 5628, 5631, 5634, 5637, 5640, + 5643, 5647, 5650, 5653, 5656, 5658, 5660, 5663, + 5670, 5673, 5675, 5678, 5681, 5684, 5692, 5694, + 5696, 5699, 5701, 5704, 5706, 5708, 5738, 5741, + 5744, 5747, 5750, 5755, 5759, 5766, 5769, 5778, + 5787, 5790, 5794, 5797, 5800, 5804, 5806, 5810, + 5812, 5815, 5817, 5821, 5825, 5829, 5837, 5839, + 5841, 5845, 5849, 5851, 5864, 5866, 5869, 5872, + 5877, 5879, 5882, 5884, 5886, 5889, 5894, 5896, + 5898, 5903, 5905, 5908, 5912, 5932, 5936, 5940, + 5942, 5944, 5952, 5954, 5961, 5966, 5968, 5972, + 5975, 5978, 5981, 5985, 5988, 5991, 5995, 6005, + 6011, 6014, 6017, 6027, 6047, 6053, 6056, 6058, + 6062, 6064, 6067, 6069, 6073, 6075, 6077, 6081, + 6083, 6085, 6091, 6094, 6099, 6104, 6110, 6120, + 6128, 6140, 6147, 6157, 6163, 6175, 6181, 6199, + 6202, 6210, 6216, 6226, 6233, 6240, 6248, 6256, + 6259, 6264, 6284, 6290, 6293, 6297, 6301, 6305, + 6317, 6320, 6325, 6326, 6332, 6339, 6345, 6348, + 6351, 6355, 6359, 6362, 6365, 6370, 6374, 6380, + 6386, 6389, 6393, 6396, 6399, 6404, 6407, 6410, + 6416, 6420, 6423, 6427, 6430, 6433, 6437, 6441, + 6448, 6451, 6454, 6460, 6463, 6470, 6472, 6474, + 6477, 6486, 6491, 6505, 6509, 6513, 6528, 6534, + 6537, 6540, 6542, 6547, 6553, 6557, 6565, 6571, + 6581, 6584, 6587, 6592, 6596, 6599, 6602, 6605, + 6609, 6614, 6618, 6622, 6625, 6630, 6635, 6638, + 6644, 6648, 6654, 6659, 6663, 6667, 6675, 6678, + 6686, 6692, 6702, 6713, 6716, 6719, 6721, 6725, + 6727, 6730, 6741, 6745, 6748, 6751, 6754, 6757, + 6759, 6763, 6767, 6770, 6774, 6779, 6782, 6792, + 6794, 6835, 6841, 6845, 6848, 6851, 6855, 6858, + 6862, 6866, 6871, 6873, 6877, 6881, 6884, 6887, + 6892, 6901, 6905, 6910, 6915, 6919, 6926, 6930, + 6933, 6937, 6940, 6945, 6948, 6951, 6981, 6985, + 6989, 6993, 6997, 7002, 7006, 7012, 7016, 7024, + 7027, 7032, 7036, 7039, 7044, 7047, 7051, 7054, + 7057, 7060, 7063, 7066, 7070, 7074, 7077, 7087, + 7090, 7093, 7098, 7104, 7107, 7122, 7125, 7129, + 7135, 7139, 7143, 7146, 7150, 7157, 7160, 7163, + 7169, 7172, 7176, 7181, 7197, 7199, 7207, 7209, + 7217, 7223, 7225, 7229, 7232, 7235, 7238, 7242, + 7253, 7256, 7268, 7292, 7300, 7302, 7306, 7309, + 7314, 7317, 7319, 7324, 7327, 7333, 7336, 7401, + 7404, 7406, 7408, 7410, 7412, 7414, 7417, 7419, + 7424, 7427, 7430, 7432, 7472, 7474, 7476, 7478, + 7483, 7487, 7488, 7490, 7492, 7499, 7506, 7513, + 7515, 7517, 7519, 7522, 7525, 7531, 7534, 7539, + 7546, 7551, 7554, 7558, 7565, 7597, 7646, 7661, + 7674, 7679, 7681, 7685, 7716, 7722, 7724, 7745, + 7765, 7767, 7779, 7790, 7800, 7806, 7816, 7826, + 7837, 7839, 7841, 7843, 7845, 7847, 7849, 7851, + 7861, 7870, 7872, 7874, 7876, 7878, 7880, 7890, + 7899, 7901, 7903, 7905, 7907, 7909, 7950, 7990, + 7992, 7997, 8001, 8002, 8004, 8006, 8013, 8020, + 8027, 8029, 8031, 8033, 8036, 8039, 8045, 8048, + 8053, 8060, 8065, 8068, 8072, 8079, 8111, 8160, + 8175, 8188, 8193, 8195, 8199, 8230, 8236, 8238, + 8259, 8279, } -var _zcltok_indicies []int16 = []int16{ +var _hcltok_indicies []int16 = []int16{ 2, 1, 4, 3, 6, 5, 6, 7, 5, 9, 11, 11, 10, 8, 12, 12, 10, 8, 10, 8, 13, 15, 16, 18, @@ -1413,7 +2162,7 @@ var _zcltok_indicies []int16 = []int16{ 64, 65, 40, 42, 66, 44, 67, 68, 69, 14, 14, 14, 17, 41, 3, 47, 3, 14, 14, 14, 14, 3, 14, 14, - 14, 3, 14, 3, 14, 3, 14, 3, + 14, 3, 14, 3, 14, 14, 3, 3, 3, 3, 3, 3, 14, 3, 3, 3, 3, 14, 14, 14, 14, 14, 3, 3, 14, 3, 3, 14, 3, 14, 3, 3, @@ -1990,88 +2739,456 @@ var _zcltok_indicies []int16 = []int16{ 773, 772, 775, 774, 776, 777, 777, 774, 778, 774, 779, 776, 780, 777, 781, 777, 783, 782, 784, 785, 785, 782, 786, 782, - 787, 784, 788, 785, 789, 785, 790, 791, - 792, 793, 794, 795, 796, 797, 799, 800, - 801, 802, 803, 672, 672, 672, 804, 805, - 806, 807, 672, 810, 811, 813, 814, 815, - 809, 816, 817, 818, 819, 820, 821, 822, - 823, 824, 825, 826, 827, 828, 829, 830, - 831, 832, 833, 834, 835, 837, 838, 839, - 840, 841, 842, 672, 798, 10, 798, 422, - 798, 422, 809, 812, 836, 843, 808, 790, - 844, 791, 845, 793, 846, 848, 847, 2, - 1, 849, 847, 850, 847, 5, 1, 847, - 6, 5, 9, 11, 11, 10, 852, 853, - 854, 847, 855, 856, 847, 857, 847, 422, - 422, 859, 860, 491, 472, 861, 472, 862, - 863, 864, 865, 866, 867, 868, 869, 870, - 871, 872, 546, 873, 522, 874, 875, 876, - 877, 878, 879, 880, 881, 882, 883, 884, - 885, 422, 422, 422, 427, 567, 858, 886, - 847, 887, 847, 672, 888, 422, 422, 422, - 672, 888, 672, 672, 422, 888, 422, 888, - 422, 888, 422, 672, 672, 672, 672, 672, - 888, 422, 672, 672, 672, 422, 672, 422, - 888, 422, 672, 672, 672, 672, 422, 888, - 672, 422, 672, 422, 672, 422, 672, 672, - 422, 672, 888, 422, 672, 422, 672, 422, - 672, 888, 672, 422, 888, 672, 422, 672, - 422, 888, 672, 672, 672, 672, 672, 888, - 422, 422, 672, 422, 672, 888, 672, 422, - 888, 672, 672, 888, 422, 422, 672, 422, - 672, 422, 672, 888, 889, 890, 891, 892, - 893, 894, 895, 896, 897, 898, 899, 717, - 900, 901, 902, 903, 904, 905, 906, 907, - 908, 909, 910, 911, 910, 912, 913, 914, - 915, 916, 673, 888, 917, 918, 919, 920, - 921, 922, 923, 924, 925, 926, 927, 928, - 929, 930, 931, 932, 933, 934, 935, 727, - 936, 937, 938, 694, 939, 940, 941, 942, - 943, 944, 673, 945, 946, 947, 948, 949, - 950, 951, 952, 676, 953, 673, 676, 954, - 955, 956, 957, 685, 888, 958, 959, 960, - 961, 705, 962, 963, 685, 964, 965, 966, - 967, 968, 673, 888, 969, 928, 970, 971, - 972, 685, 973, 974, 676, 673, 685, 427, - 888, 938, 673, 676, 685, 427, 685, 427, - 975, 685, 888, 427, 676, 976, 977, 676, - 978, 979, 683, 980, 981, 982, 983, 984, - 934, 985, 986, 987, 988, 989, 990, 991, - 992, 993, 994, 995, 996, 953, 997, 676, - 685, 427, 888, 998, 999, 685, 673, 888, - 427, 673, 888, 676, 1000, 733, 1001, 1002, - 1003, 1004, 1005, 1006, 1007, 1008, 673, 1009, - 1010, 1011, 1012, 1013, 1014, 673, 685, 888, - 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, - 1024, 1025, 1026, 1022, 1028, 1029, 1030, 1031, - 1015, 1027, 1015, 888, 1015, 888, 1032, 1032, - 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, - 1037, 769, 1041, 1041, 1041, 1036, 1042, 1041, - 770, 771, 1043, 1041, 769, 1041, 1041, 1036, - 1044, 1041, 770, 771, 1043, 1041, 769, 1036, - 1044, 1045, 1046, 1047, 769, 1041, 1041, 1041, - 1036, 1042, 770, 771, 1043, 1041, 769, 1041, - 1041, 1041, 1036, 1042, 770, 771, 1043, 1041, - 769, 1041, 1041, 1041, 1036, 1042, 771, 770, - 771, 1043, 1041, 769, 1049, 769, 1051, 1050, - 1052, 769, 1054, 1053, 769, 1055, 773, 1055, - 1056, 1055, 775, 1057, 1058, 1059, 1060, 1061, - 1062, 1063, 1060, 777, 775, 1057, 1065, 1064, - 778, 779, 1066, 1064, 777, 1068, 1067, 1070, - 1069, 777, 1071, 778, 1071, 779, 1071, 783, - 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1075, - 785, 783, 1072, 1080, 1079, 786, 787, 1081, - 1079, 785, 1083, 1082, 1085, 1084, 785, 1086, - 786, 1086, 787, 1086, + 787, 784, 788, 785, 789, 785, 791, 791, + 791, 791, 790, 791, 791, 791, 790, 791, + 790, 791, 791, 790, 790, 790, 790, 790, + 790, 791, 790, 790, 790, 790, 791, 791, + 791, 791, 791, 790, 790, 791, 790, 790, + 791, 790, 791, 790, 790, 791, 790, 790, + 790, 791, 791, 791, 791, 791, 791, 790, + 791, 791, 790, 791, 791, 790, 790, 790, + 790, 790, 790, 791, 791, 790, 790, 791, + 790, 791, 791, 791, 790, 793, 794, 795, + 796, 797, 798, 799, 800, 801, 802, 803, + 804, 805, 806, 807, 808, 809, 810, 811, + 812, 813, 814, 815, 816, 817, 818, 819, + 820, 821, 822, 823, 824, 790, 791, 790, + 791, 790, 791, 791, 790, 791, 791, 790, + 790, 790, 791, 790, 790, 790, 790, 790, + 790, 790, 791, 790, 790, 790, 790, 790, + 790, 790, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 790, 790, 790, + 790, 790, 790, 790, 790, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 790, 790, + 790, 790, 790, 790, 790, 790, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 790, + 791, 791, 791, 791, 791, 791, 791, 791, + 790, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 790, 791, 791, 791, + 791, 791, 791, 790, 791, 791, 791, 791, + 791, 791, 790, 790, 790, 790, 790, 790, + 790, 790, 791, 791, 791, 791, 791, 791, + 791, 791, 790, 791, 791, 791, 791, 791, + 791, 791, 791, 790, 791, 791, 791, 791, + 791, 790, 790, 790, 790, 790, 790, 790, + 790, 791, 791, 791, 791, 791, 791, 790, + 791, 791, 791, 791, 791, 791, 791, 790, + 791, 790, 791, 791, 790, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 790, 791, 791, 791, 791, 791, + 790, 791, 791, 791, 791, 791, 791, 791, + 790, 791, 791, 791, 790, 791, 791, 791, + 790, 791, 790, 825, 826, 827, 828, 829, + 830, 831, 832, 833, 834, 835, 836, 837, + 838, 839, 840, 841, 842, 843, 844, 845, + 846, 847, 848, 849, 850, 851, 852, 853, + 854, 855, 856, 857, 858, 859, 860, 797, + 861, 862, 863, 864, 865, 866, 797, 842, + 797, 790, 791, 790, 791, 791, 790, 790, + 791, 790, 790, 790, 790, 791, 790, 790, + 790, 790, 790, 791, 790, 790, 790, 790, + 790, 791, 791, 791, 791, 791, 790, 790, + 790, 791, 790, 790, 790, 791, 791, 791, + 790, 790, 790, 791, 791, 790, 790, 790, + 791, 791, 791, 790, 790, 790, 791, 791, + 791, 791, 790, 791, 791, 791, 791, 790, + 790, 790, 790, 790, 791, 791, 791, 791, + 790, 790, 791, 791, 791, 790, 790, 791, + 791, 791, 791, 790, 791, 791, 790, 791, + 791, 790, 790, 790, 791, 791, 791, 790, + 790, 790, 790, 791, 791, 791, 791, 791, + 790, 790, 790, 790, 791, 790, 791, 791, + 790, 791, 791, 790, 791, 790, 791, 791, + 791, 790, 791, 791, 790, 790, 790, 791, + 790, 790, 790, 790, 790, 790, 790, 791, + 791, 791, 791, 790, 791, 791, 791, 791, + 791, 791, 791, 790, 867, 868, 869, 870, + 871, 872, 873, 874, 875, 797, 876, 877, + 878, 879, 880, 790, 791, 790, 790, 790, + 790, 790, 791, 791, 790, 791, 791, 791, + 790, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 790, 791, 791, 791, 790, + 790, 791, 791, 791, 790, 790, 791, 790, + 790, 791, 791, 791, 791, 791, 790, 790, + 790, 790, 791, 791, 791, 791, 791, 791, + 790, 791, 791, 791, 791, 791, 790, 881, + 836, 882, 883, 884, 797, 885, 886, 842, + 797, 790, 791, 791, 791, 791, 790, 790, + 790, 791, 790, 790, 791, 791, 791, 790, + 790, 790, 791, 791, 790, 847, 790, 842, + 797, 797, 887, 790, 797, 790, 791, 842, + 888, 889, 842, 890, 891, 842, 892, 893, + 894, 895, 896, 897, 842, 898, 899, 900, + 842, 901, 902, 903, 861, 904, 905, 906, + 861, 907, 842, 797, 790, 790, 791, 791, + 790, 790, 790, 791, 791, 791, 791, 790, + 791, 791, 790, 790, 790, 790, 791, 791, + 790, 790, 791, 791, 790, 790, 790, 790, + 790, 790, 791, 791, 791, 790, 790, 790, + 791, 790, 790, 790, 791, 791, 790, 791, + 791, 791, 791, 790, 791, 791, 791, 791, + 790, 791, 791, 791, 791, 791, 791, 790, + 790, 790, 791, 791, 791, 791, 790, 908, + 909, 790, 797, 790, 791, 790, 790, 791, + 842, 910, 911, 912, 913, 892, 914, 915, + 916, 917, 918, 919, 920, 921, 922, 923, + 924, 925, 797, 790, 790, 791, 790, 791, + 791, 791, 791, 791, 791, 791, 790, 791, + 791, 791, 790, 791, 790, 790, 791, 790, + 791, 790, 790, 791, 791, 791, 791, 790, + 791, 791, 791, 790, 790, 791, 791, 791, + 791, 790, 791, 791, 790, 790, 791, 791, + 791, 791, 791, 790, 926, 927, 928, 929, + 930, 931, 932, 933, 934, 935, 936, 932, + 938, 939, 940, 941, 937, 790, 942, 943, + 842, 944, 945, 946, 947, 948, 949, 950, + 951, 952, 842, 797, 953, 954, 955, 956, + 842, 957, 958, 959, 960, 961, 962, 963, + 964, 965, 966, 967, 968, 969, 970, 971, + 842, 873, 797, 972, 790, 791, 791, 791, + 791, 791, 790, 790, 790, 791, 790, 791, + 791, 790, 791, 790, 791, 791, 790, 790, + 790, 791, 791, 791, 790, 790, 790, 791, + 791, 791, 790, 790, 790, 790, 791, 790, + 790, 791, 790, 790, 791, 791, 791, 790, + 790, 791, 790, 791, 791, 791, 790, 791, + 791, 791, 791, 791, 791, 790, 790, 790, + 791, 791, 790, 791, 791, 790, 791, 791, + 790, 791, 791, 790, 791, 791, 791, 791, + 791, 791, 791, 790, 791, 790, 791, 790, + 791, 791, 790, 791, 790, 791, 791, 790, + 791, 790, 791, 790, 973, 944, 974, 975, + 976, 977, 978, 979, 980, 981, 982, 825, + 983, 842, 984, 985, 986, 842, 987, 857, + 988, 989, 990, 991, 992, 993, 994, 995, + 842, 790, 790, 790, 791, 791, 791, 790, + 791, 791, 790, 791, 791, 790, 790, 790, + 790, 790, 791, 791, 791, 791, 790, 791, + 791, 791, 791, 791, 791, 790, 790, 790, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 790, 791, 791, 791, 791, 791, 791, + 791, 791, 790, 791, 791, 790, 790, 790, + 790, 791, 791, 791, 790, 790, 790, 791, + 790, 790, 790, 791, 791, 790, 791, 791, + 791, 790, 791, 790, 790, 790, 791, 791, + 790, 791, 791, 791, 790, 791, 791, 791, + 790, 790, 790, 790, 791, 842, 911, 996, + 997, 797, 842, 797, 790, 790, 791, 790, + 791, 842, 996, 797, 790, 842, 998, 797, + 790, 790, 791, 842, 999, 1000, 1001, 902, + 1002, 1003, 842, 1004, 1005, 1006, 797, 790, + 790, 791, 791, 791, 790, 791, 791, 790, + 791, 791, 791, 791, 790, 790, 791, 790, + 790, 791, 791, 790, 791, 790, 842, 797, + 790, 1007, 842, 1008, 790, 797, 790, 791, + 790, 791, 1009, 842, 1010, 1011, 790, 791, + 790, 790, 790, 791, 791, 791, 791, 790, + 1012, 1013, 1014, 842, 1015, 1016, 1017, 1018, + 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, + 1027, 1028, 797, 790, 791, 791, 791, 790, + 790, 790, 790, 791, 791, 790, 790, 791, + 790, 790, 790, 790, 790, 790, 790, 791, + 790, 791, 790, 790, 790, 790, 790, 790, + 791, 791, 791, 791, 791, 790, 790, 791, + 790, 790, 790, 791, 790, 790, 791, 790, + 790, 791, 790, 790, 791, 790, 790, 790, + 791, 791, 791, 790, 790, 790, 791, 791, + 791, 791, 790, 1029, 842, 1030, 842, 1031, + 1032, 1033, 1034, 797, 790, 791, 791, 791, + 791, 791, 790, 790, 790, 791, 790, 790, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 790, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 790, 791, + 791, 791, 791, 791, 790, 1035, 842, 797, + 790, 791, 1036, 842, 827, 797, 790, 791, + 1037, 790, 797, 790, 791, 842, 1038, 797, + 790, 790, 791, 1039, 790, 842, 1040, 797, + 790, 790, 791, 1042, 1041, 791, 791, 791, + 791, 1042, 1041, 791, 1042, 1041, 1042, 1042, + 791, 1042, 1041, 791, 1042, 791, 1042, 1041, + 791, 1042, 791, 1042, 791, 1041, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1041, 791, + 791, 1042, 1042, 791, 1042, 791, 1042, 1041, + 1042, 1042, 1042, 1042, 1042, 791, 1042, 791, + 1042, 791, 1042, 1041, 1042, 1042, 791, 1042, + 791, 1042, 1041, 1042, 1042, 1042, 1042, 1042, + 791, 1042, 791, 1042, 1041, 791, 791, 1042, + 791, 1042, 1041, 1042, 1042, 1042, 791, 1042, + 791, 1042, 791, 1042, 791, 1042, 1041, 1042, + 791, 1042, 791, 1042, 1041, 791, 1042, 1042, + 1042, 1042, 791, 1042, 791, 1042, 791, 1042, + 791, 1042, 791, 1042, 791, 1042, 1041, 791, + 1042, 1041, 1042, 1042, 1042, 791, 1042, 791, + 1042, 1041, 1042, 791, 1042, 791, 1042, 1041, + 791, 1042, 1042, 1042, 1042, 791, 1042, 791, + 1042, 1041, 791, 1042, 791, 1042, 791, 1042, + 1041, 1042, 1042, 791, 1042, 791, 1042, 1041, + 791, 1042, 791, 1042, 791, 1042, 791, 1041, + 1042, 1042, 1042, 791, 1042, 791, 1042, 1041, + 791, 1042, 1041, 1042, 1042, 791, 1042, 1041, + 1042, 1042, 1042, 791, 1042, 1042, 1042, 1042, + 1042, 1042, 791, 791, 1042, 791, 1042, 791, + 1042, 791, 1042, 1041, 1042, 791, 1042, 791, + 1042, 1041, 791, 1042, 1041, 1042, 791, 1042, + 1041, 1042, 791, 1042, 1041, 791, 791, 1042, + 1041, 791, 1042, 791, 1042, 791, 1042, 791, + 1042, 791, 1042, 791, 1041, 1042, 1042, 791, + 1042, 1042, 1042, 1042, 791, 791, 1042, 1042, + 1042, 1042, 1042, 791, 1042, 1042, 1042, 1042, + 1042, 1041, 791, 1042, 1042, 791, 1042, 791, + 1041, 1042, 1042, 791, 1042, 1041, 791, 791, + 1042, 791, 1041, 1042, 1042, 1041, 791, 1042, + 791, 1041, 1042, 1041, 791, 1042, 791, 1042, + 791, 1041, 1042, 1042, 1041, 791, 1042, 791, + 1042, 791, 1042, 1041, 1042, 791, 1042, 791, + 1042, 1041, 791, 1042, 1041, 791, 791, 1042, + 1041, 1042, 791, 1041, 1042, 1041, 791, 1042, + 791, 1042, 791, 1041, 1042, 1041, 791, 791, + 1042, 1041, 1042, 791, 1042, 791, 1042, 1041, + 791, 1042, 791, 1041, 1042, 1041, 791, 791, + 1042, 791, 1041, 1042, 1041, 791, 791, 1042, + 1041, 1042, 791, 1042, 1041, 1042, 791, 1042, + 1041, 1042, 791, 1042, 791, 1042, 791, 1041, + 1042, 1041, 791, 791, 1042, 1041, 1042, 791, + 1042, 791, 1042, 1041, 791, 1042, 1041, 1042, + 1042, 791, 1042, 791, 1042, 1041, 1041, 791, + 1041, 791, 1042, 1042, 791, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1041, 791, 1042, 1042, + 1042, 791, 1041, 1042, 1042, 1042, 791, 1042, + 791, 1042, 791, 1042, 791, 1042, 791, 1042, + 1041, 791, 791, 1042, 1041, 1042, 791, 1042, + 1041, 791, 791, 1042, 791, 791, 791, 1042, + 791, 1042, 791, 1042, 791, 1042, 791, 1041, + 791, 1042, 791, 1042, 791, 1041, 1042, 1041, + 791, 1042, 791, 1041, 1042, 791, 1042, 1042, + 1042, 1041, 791, 1042, 791, 791, 1042, 791, + 1041, 1042, 1042, 1041, 791, 1042, 1042, 1042, + 1042, 791, 1042, 791, 1041, 1042, 1042, 1042, + 791, 1042, 1041, 1042, 791, 1042, 791, 1042, + 791, 1042, 791, 1042, 1041, 1042, 1042, 791, + 1042, 1041, 791, 1042, 791, 1042, 791, 1041, + 1042, 1042, 1041, 791, 1042, 791, 1041, 1042, + 1041, 791, 1042, 1041, 791, 1042, 791, 1042, + 1041, 1042, 1042, 1042, 1041, 791, 791, 791, + 1042, 1041, 791, 1042, 791, 1041, 1042, 1041, + 791, 1042, 791, 1042, 791, 1041, 1042, 1042, + 1042, 1041, 791, 1042, 791, 1041, 1042, 1042, + 1042, 1042, 1041, 791, 1042, 791, 1042, 1041, + 791, 791, 1042, 791, 1042, 1041, 1042, 791, + 1042, 791, 1041, 1042, 1042, 1041, 791, 1042, + 791, 1042, 1041, 791, 1042, 1042, 1042, 791, + 1042, 791, 1041, 791, 1042, 1041, 1042, 791, + 791, 1042, 791, 1042, 791, 1041, 1042, 1042, + 1042, 1042, 1041, 791, 1042, 791, 1042, 791, + 1042, 791, 1042, 791, 1042, 1041, 1042, 1042, + 1042, 791, 1042, 791, 1042, 791, 1042, 791, + 1041, 1042, 1042, 791, 791, 1042, 1041, 1042, + 791, 1042, 1042, 1041, 791, 1042, 791, 1042, + 1041, 791, 791, 1042, 1042, 1042, 1042, 791, + 1042, 791, 1042, 791, 1041, 1042, 1042, 791, + 1041, 1042, 1041, 791, 1042, 791, 1041, 1042, + 1041, 791, 1042, 791, 1041, 1042, 791, 1042, + 1042, 1041, 791, 1042, 1042, 791, 1041, 1042, + 1041, 791, 1042, 791, 1042, 1041, 1042, 791, + 1042, 791, 1041, 1042, 1041, 791, 1042, 791, + 1042, 791, 1042, 791, 1042, 791, 1042, 1041, + 1043, 1041, 1044, 1045, 1046, 1047, 1048, 1049, + 1050, 1051, 1052, 1053, 1054, 1046, 1055, 1056, + 1057, 1058, 1059, 1046, 1060, 1061, 1062, 1063, + 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, + 1072, 1073, 1074, 1046, 1075, 1043, 1055, 1043, + 1076, 1043, 1041, 1042, 1042, 1042, 1042, 791, + 1041, 1042, 1042, 1041, 791, 1042, 1041, 791, + 791, 1042, 1041, 791, 1042, 791, 1041, 1042, + 1041, 791, 791, 1042, 791, 1041, 1042, 1042, + 1041, 791, 1042, 1042, 1042, 1041, 791, 1042, + 791, 1042, 1042, 1041, 791, 791, 1042, 791, + 1041, 1042, 1041, 791, 1042, 1041, 791, 791, + 1042, 791, 1042, 1041, 791, 1042, 791, 791, + 1042, 791, 1042, 791, 1041, 1042, 1042, 1041, + 791, 1042, 1042, 791, 1042, 1041, 791, 1042, + 791, 1042, 1041, 791, 1042, 791, 1041, 791, + 1042, 1042, 1042, 791, 1042, 1041, 1042, 791, + 1042, 1041, 791, 1042, 1041, 1042, 791, 1042, + 1041, 791, 1042, 1041, 791, 1042, 791, 1042, + 1041, 791, 1042, 1041, 791, 1042, 1041, 1077, + 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, + 1086, 1087, 1088, 1048, 1089, 1090, 1091, 1092, + 1093, 1090, 1094, 1095, 1096, 1097, 1098, 1099, + 1100, 1101, 1102, 1043, 1041, 1042, 791, 1042, + 1041, 1042, 791, 1042, 1041, 1042, 791, 1042, + 1041, 1042, 791, 1042, 1041, 791, 1042, 791, + 1042, 1041, 1042, 791, 1042, 1041, 1042, 791, + 791, 791, 1042, 1041, 1042, 791, 1042, 1041, + 1042, 1042, 1042, 1042, 791, 1042, 791, 1041, + 1042, 1041, 791, 791, 1042, 791, 1042, 1041, + 1042, 791, 1042, 1041, 791, 1042, 1041, 1042, + 1042, 791, 1042, 1041, 791, 1042, 1041, 1042, + 791, 1042, 1041, 791, 1042, 1041, 791, 1042, + 1041, 791, 1042, 1041, 1042, 1041, 791, 791, + 1042, 1041, 1042, 791, 1042, 1041, 791, 1042, + 791, 1041, 1042, 1041, 791, 1046, 1103, 1043, + 1046, 1104, 1046, 1105, 1055, 1043, 1041, 1042, + 1041, 791, 1042, 1041, 791, 1046, 1104, 1055, + 1043, 1041, 1046, 1106, 1043, 1055, 1043, 1041, + 1042, 1041, 791, 1046, 1107, 1064, 1108, 1090, + 1109, 1102, 1046, 1110, 1111, 1112, 1043, 1055, + 1043, 1041, 1042, 1041, 791, 1042, 791, 1042, + 1041, 791, 1042, 791, 1042, 791, 1041, 1042, + 1042, 1041, 791, 1042, 791, 1042, 1041, 791, + 1042, 1041, 1046, 1055, 797, 1041, 1113, 1046, + 1114, 1055, 1043, 1041, 797, 1042, 1041, 791, + 1042, 1041, 791, 1115, 1046, 1116, 1117, 1043, + 1041, 791, 1042, 1041, 1042, 1042, 1041, 791, + 791, 1042, 791, 1042, 1041, 1046, 1118, 1119, + 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, + 1128, 1043, 1055, 1043, 1041, 1042, 791, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 791, 1042, + 791, 1042, 1042, 1042, 1042, 1042, 1042, 1041, + 791, 1042, 1042, 791, 1042, 791, 1041, 1042, + 791, 1042, 1042, 1042, 791, 1042, 1042, 791, + 1042, 1042, 791, 1042, 1042, 791, 1042, 1042, + 1041, 791, 1046, 1129, 1046, 1105, 1130, 1131, + 1132, 1043, 1055, 1043, 1041, 1042, 1041, 791, + 1042, 1042, 1042, 791, 1042, 1042, 1042, 791, + 1042, 791, 1042, 1041, 791, 791, 791, 791, + 1042, 1042, 791, 791, 791, 791, 791, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 791, 1042, + 791, 1042, 791, 1041, 1042, 1042, 1042, 791, + 1042, 791, 1042, 1041, 1055, 797, 1133, 1046, + 1055, 797, 1042, 1041, 791, 1134, 1046, 1135, + 1055, 797, 1042, 1041, 791, 1042, 791, 1136, + 1055, 1043, 1041, 797, 1042, 1041, 791, 1046, + 1137, 1043, 1055, 1043, 1041, 1042, 1041, 791, + 1138, 1139, 1140, 1138, 1141, 1142, 1143, 1144, + 1146, 1147, 1148, 1149, 1150, 672, 672, 672, + 1151, 1152, 1153, 1154, 672, 1157, 1158, 1160, + 1161, 1162, 1156, 1163, 1164, 1165, 1166, 1167, + 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1175, + 1176, 1177, 1178, 1179, 1180, 1181, 1182, 1184, + 1185, 1186, 1187, 1188, 1189, 672, 1145, 10, + 1145, 422, 1145, 422, 1156, 1159, 1183, 1190, + 1155, 1138, 1138, 1191, 1139, 1192, 1194, 1193, + 2, 1, 1195, 1193, 1196, 1193, 5, 1, + 1193, 6, 5, 9, 11, 11, 10, 1198, + 1199, 1200, 1193, 1201, 1202, 1193, 1203, 1193, + 422, 422, 1205, 1206, 491, 472, 1207, 472, + 1208, 1209, 1210, 1211, 1212, 1213, 1214, 1215, + 1216, 1217, 1218, 546, 1219, 522, 1220, 1221, + 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, + 1230, 1231, 422, 422, 422, 427, 567, 1204, + 1232, 1193, 1233, 1193, 672, 1234, 422, 422, + 422, 672, 1234, 672, 672, 422, 1234, 422, + 1234, 422, 1234, 422, 672, 672, 672, 672, + 672, 1234, 422, 672, 672, 672, 422, 672, + 422, 1234, 422, 672, 672, 672, 672, 422, + 1234, 672, 422, 672, 422, 672, 422, 672, + 672, 422, 672, 1234, 422, 672, 422, 672, + 422, 672, 1234, 672, 422, 1234, 672, 422, + 672, 422, 1234, 672, 672, 672, 672, 672, + 1234, 422, 422, 672, 422, 672, 1234, 672, + 422, 1234, 672, 672, 1234, 422, 422, 672, + 422, 672, 422, 672, 1234, 1235, 1236, 1237, + 1238, 1239, 1240, 1241, 1242, 1243, 1244, 1245, + 717, 1246, 1247, 1248, 1249, 1250, 1251, 1252, + 1253, 1254, 1255, 1256, 1257, 1256, 1258, 1259, + 1260, 1261, 1262, 673, 1234, 1263, 1264, 1265, + 1266, 1267, 1268, 1269, 1270, 1271, 1272, 1273, + 1274, 1275, 1276, 1277, 1278, 1279, 1280, 1281, + 727, 1282, 1283, 1284, 694, 1285, 1286, 1287, + 1288, 1289, 1290, 673, 1291, 1292, 1293, 1294, + 1295, 1296, 1297, 1298, 676, 1299, 673, 676, + 1300, 1301, 1302, 1303, 685, 1234, 1304, 1305, + 1306, 1307, 705, 1308, 1309, 685, 1310, 1311, + 1312, 1313, 1314, 673, 1234, 1315, 1274, 1316, + 1317, 1318, 685, 1319, 1320, 676, 673, 685, + 427, 1234, 1284, 673, 676, 685, 427, 685, + 427, 1321, 685, 1234, 427, 676, 1322, 1323, + 676, 1324, 1325, 683, 1326, 1327, 1328, 1329, + 1330, 1280, 1331, 1332, 1333, 1334, 1335, 1336, + 1337, 1338, 1339, 1340, 1341, 1342, 1299, 1343, + 676, 685, 427, 1234, 1344, 1345, 685, 673, + 1234, 427, 673, 1234, 676, 1346, 733, 1347, + 1348, 1349, 1350, 1351, 1352, 1353, 1354, 673, + 1355, 1356, 1357, 1358, 1359, 1360, 673, 685, + 1234, 1362, 1363, 1364, 1365, 1366, 1367, 1368, + 1369, 1370, 1371, 1372, 1368, 1374, 1375, 1376, + 1377, 1361, 1373, 1361, 1234, 1361, 1234, 1378, + 1378, 1379, 1380, 1381, 1382, 1383, 1384, 1385, + 1386, 1383, 769, 1387, 1387, 1387, 1382, 1388, + 1387, 770, 771, 1389, 1387, 769, 1387, 1387, + 1382, 1390, 1387, 770, 771, 1389, 1387, 769, + 1382, 1390, 1391, 1392, 1393, 769, 1387, 1387, + 1387, 1382, 1388, 770, 771, 1389, 1387, 769, + 1387, 1387, 1387, 1382, 1388, 770, 771, 1389, + 1387, 769, 1387, 1387, 1387, 1382, 1388, 771, + 770, 771, 1389, 1387, 769, 1395, 769, 1397, + 1396, 1398, 769, 1400, 1399, 769, 1401, 773, + 1401, 1402, 1401, 775, 1403, 1404, 1405, 1406, + 1407, 1408, 1409, 1406, 777, 775, 1403, 1411, + 1410, 778, 779, 1412, 1410, 777, 1414, 1413, + 1416, 1415, 777, 1417, 778, 1417, 779, 1417, + 783, 1418, 1419, 1420, 1421, 1422, 1423, 1424, + 1421, 785, 783, 1418, 1426, 1425, 786, 787, + 1427, 1425, 785, 1429, 1428, 1431, 1430, 785, + 1432, 786, 1432, 787, 1432, 1435, 1436, 1438, + 1439, 1440, 1434, 1441, 1442, 1443, 1444, 1445, + 1446, 1447, 1448, 1449, 1450, 1451, 1452, 1453, + 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1462, + 1463, 1464, 1465, 1466, 1467, 791, 791, 1433, + 1434, 1437, 1461, 1468, 1433, 1042, 791, 791, + 1470, 1471, 861, 842, 1472, 842, 1473, 1474, + 1475, 1476, 1477, 1478, 1479, 1480, 1481, 1482, + 1483, 916, 1484, 892, 1485, 1486, 1487, 1488, + 1489, 1490, 1491, 1492, 1493, 1494, 1495, 1496, + 791, 791, 791, 797, 937, 1469, 1042, 1497, + 791, 791, 791, 1042, 1497, 1042, 1042, 791, + 1497, 791, 1497, 791, 1497, 791, 1042, 1042, + 1042, 1042, 1042, 1497, 791, 1042, 1042, 1042, + 791, 1042, 791, 1497, 791, 1042, 1042, 1042, + 1042, 791, 1497, 1042, 791, 1042, 791, 1042, + 791, 1042, 1042, 791, 1042, 1497, 791, 1042, + 791, 1042, 791, 1042, 1497, 1042, 791, 1497, + 1042, 791, 1042, 791, 1497, 1042, 1042, 1042, + 1042, 1042, 1497, 791, 791, 1042, 791, 1042, + 1497, 1042, 791, 1497, 1042, 1042, 1497, 791, + 791, 1042, 791, 1042, 791, 1042, 1497, 1498, + 1499, 1500, 1501, 1502, 1503, 1504, 1505, 1506, + 1507, 1508, 1087, 1509, 1510, 1511, 1512, 1513, + 1514, 1515, 1516, 1517, 1518, 1519, 1520, 1519, + 1521, 1522, 1523, 1524, 1525, 1043, 1497, 1526, + 1527, 1528, 1529, 1530, 1531, 1532, 1533, 1534, + 1535, 1536, 1537, 1538, 1539, 1540, 1541, 1542, + 1543, 1544, 1097, 1545, 1546, 1547, 1064, 1548, + 1549, 1550, 1551, 1552, 1553, 1043, 1554, 1555, + 1556, 1557, 1558, 1559, 1560, 1561, 1046, 1562, + 1043, 1046, 1563, 1564, 1565, 1566, 1055, 1497, + 1567, 1568, 1569, 1570, 1075, 1571, 1572, 1055, + 1573, 1574, 1575, 1576, 1577, 1043, 1497, 1578, + 1537, 1579, 1580, 1581, 1055, 1582, 1583, 1046, + 1043, 1055, 797, 1497, 1547, 1043, 1046, 1055, + 797, 1055, 797, 1584, 1055, 1497, 797, 1046, + 1585, 1586, 1046, 1587, 1588, 1053, 1589, 1590, + 1591, 1592, 1593, 1543, 1594, 1595, 1596, 1597, + 1598, 1599, 1600, 1601, 1602, 1603, 1604, 1605, + 1562, 1606, 1046, 1055, 797, 1497, 1607, 1608, + 1055, 1043, 1497, 797, 1043, 1497, 1046, 1609, + 1103, 1610, 1611, 1612, 1613, 1614, 1615, 1616, + 1617, 1043, 1618, 1619, 1620, 1621, 1622, 1623, + 1043, 1055, 1497, 1625, 1626, 1627, 1628, 1629, + 1630, 1631, 1632, 1633, 1634, 1635, 1631, 1637, + 1638, 1639, 1640, 1624, 1636, 1624, 1497, 1624, + 1497, } -var _zcltok_trans_targs []int16 = []int16{ - 949, 1, 949, 949, 949, 3, 4, 958, - 949, 5, 959, 6, 7, 9, 10, 287, +var _hcltok_trans_targs []int16 = []int16{ + 1462, 1, 1462, 1462, 1462, 3, 4, 1470, + 1462, 5, 1471, 6, 7, 9, 10, 287, 13, 14, 15, 16, 17, 288, 289, 20, 290, 22, 23, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 329, 349, 354, - 128, 129, 130, 357, 152, 372, 376, 949, + 128, 129, 130, 357, 152, 372, 376, 1462, 11, 12, 18, 19, 21, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 65, 106, 121, 132, 155, 171, 284, 34, 35, @@ -2118,7 +3235,7 @@ var _zcltok_trans_targs []int16 = []int16{ 382, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 406, 407, - 408, 409, 411, 413, 415, 949, 963, 438, + 408, 409, 411, 413, 415, 1462, 1475, 438, 439, 440, 441, 418, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, @@ -2149,8 +3266,8 @@ var _zcltok_trans_targs []int16 = []int16{ 653, 654, 656, 657, 658, 659, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672, 674, 675, 676, 677, 678, - 679, 681, 683, 685, 687, 689, 690, 949, - 949, 691, 828, 829, 760, 830, 831, 832, + 679, 681, 683, 685, 687, 689, 690, 1462, + 1462, 691, 828, 829, 760, 830, 831, 832, 833, 834, 835, 789, 836, 725, 837, 838, 839, 840, 841, 842, 843, 844, 745, 845, 846, 847, 848, 849, 850, 851, 852, 853, @@ -2162,55 +3279,125 @@ var _zcltok_trans_targs []int16 = []int16{ 897, 899, 900, 901, 903, 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, 915, 916, 917, 918, 921, 923, 924, 926, 928, - 1001, 1002, 930, 931, 1001, 933, 1015, 1015, - 1015, 1016, 937, 938, 1017, 1018, 1022, 1022, - 1022, 1023, 944, 945, 1024, 1025, 950, 949, - 951, 952, 953, 949, 954, 955, 949, 956, - 957, 960, 961, 962, 949, 964, 949, 965, - 949, 966, 967, 968, 969, 970, 971, 972, - 973, 974, 975, 976, 977, 978, 979, 980, - 981, 982, 983, 984, 985, 986, 987, 988, - 989, 990, 991, 992, 993, 994, 995, 996, - 997, 998, 999, 1000, 949, 949, 949, 949, - 949, 949, 2, 949, 949, 8, 949, 949, - 949, 949, 949, 416, 417, 421, 422, 423, - 424, 425, 426, 427, 428, 429, 430, 431, - 432, 434, 436, 437, 469, 510, 525, 532, - 534, 536, 556, 559, 575, 688, 949, 949, - 949, 692, 693, 694, 695, 696, 697, 698, - 699, 700, 701, 702, 704, 705, 706, 707, - 708, 709, 710, 711, 712, 713, 714, 715, - 716, 717, 718, 719, 720, 721, 722, 723, - 724, 726, 727, 728, 729, 730, 731, 732, - 733, 734, 735, 736, 737, 738, 739, 740, - 742, 743, 744, 746, 747, 748, 749, 750, - 751, 752, 753, 754, 755, 756, 757, 758, - 759, 761, 762, 763, 764, 765, 766, 767, - 768, 769, 771, 772, 773, 774, 775, 776, - 777, 778, 779, 780, 781, 782, 783, 784, - 785, 786, 787, 788, 790, 791, 792, 793, - 794, 795, 796, 797, 798, 799, 800, 801, - 802, 803, 804, 805, 806, 807, 808, 809, - 810, 812, 813, 814, 815, 816, 817, 818, - 819, 820, 821, 822, 823, 824, 825, 826, - 827, 856, 881, 884, 885, 887, 894, 895, - 898, 902, 914, 919, 920, 922, 925, 927, - 1001, 1001, 1008, 1010, 1003, 1001, 1012, 1013, - 1014, 1001, 929, 932, 1004, 1005, 1006, 1007, - 1001, 1009, 1001, 1001, 1011, 1001, 1001, 1001, - 934, 935, 940, 941, 1015, 1019, 1020, 1021, - 1015, 936, 939, 1015, 1015, 1015, 1015, 1015, - 942, 947, 948, 1022, 1026, 1027, 1028, 1022, - 943, 946, 1022, 1022, 1022, 1022, 1022, + 1513, 1514, 930, 931, 1513, 933, 1527, 1527, + 1527, 1528, 937, 938, 1529, 1530, 1534, 1534, + 1534, 1535, 944, 945, 1536, 1537, 1541, 1542, + 1541, 971, 972, 973, 974, 951, 975, 976, + 977, 978, 979, 980, 981, 982, 983, 984, + 985, 986, 987, 988, 989, 990, 991, 992, + 993, 994, 995, 996, 997, 998, 999, 1000, + 1001, 1003, 1004, 1005, 1006, 1007, 1008, 1009, + 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, + 1018, 1019, 953, 1020, 1021, 1022, 1023, 1024, + 1025, 1026, 1027, 1028, 1029, 1030, 1031, 1032, + 1033, 1034, 1035, 1036, 1037, 952, 1038, 1039, + 1040, 1041, 1042, 1044, 1045, 1046, 1047, 1048, + 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, + 1057, 1059, 1060, 1061, 1062, 1063, 1064, 1068, + 1070, 1071, 1072, 1073, 968, 1074, 1075, 1076, + 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, + 1085, 1086, 1087, 1088, 1090, 1091, 1093, 1094, + 1095, 1096, 1097, 1098, 966, 1099, 1100, 1101, + 1102, 1103, 1104, 1105, 1106, 1107, 1109, 1141, + 1165, 1168, 1169, 1171, 1180, 1181, 1184, 1188, + 1206, 1066, 1213, 1215, 1217, 1219, 1110, 1111, + 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, + 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, + 1128, 1129, 1130, 1131, 1132, 1133, 1134, 1135, + 1136, 1137, 1138, 1139, 1140, 1142, 1143, 1144, + 1145, 1146, 1147, 1148, 1149, 1150, 1151, 1152, + 1153, 1154, 1155, 1156, 1157, 1158, 1159, 1160, + 1161, 1162, 1163, 1164, 1166, 1167, 1170, 1172, + 1173, 1174, 1175, 1176, 1177, 1178, 1179, 1182, + 1183, 1185, 1186, 1187, 1189, 1190, 1191, 1192, + 1193, 1194, 1195, 1196, 1197, 1198, 1199, 1200, + 1201, 1202, 1203, 1204, 1205, 1207, 1208, 1209, + 1210, 1211, 1212, 1214, 1216, 1218, 1220, 1222, + 1223, 1541, 1541, 1224, 1361, 1362, 1293, 1363, + 1364, 1365, 1366, 1367, 1368, 1322, 1369, 1258, + 1370, 1371, 1372, 1373, 1374, 1375, 1376, 1377, + 1278, 1378, 1379, 1380, 1381, 1382, 1383, 1384, + 1385, 1386, 1387, 1303, 1388, 1390, 1391, 1392, + 1393, 1394, 1395, 1396, 1397, 1398, 1399, 1236, + 1400, 1401, 1402, 1403, 1404, 1405, 1406, 1407, + 1408, 1274, 1409, 1410, 1411, 1412, 1413, 1344, + 1415, 1416, 1419, 1421, 1422, 1423, 1424, 1425, + 1426, 1429, 1430, 1432, 1433, 1434, 1436, 1437, + 1438, 1439, 1440, 1441, 1442, 1443, 1444, 1445, + 1446, 1448, 1449, 1450, 1451, 1454, 1456, 1457, + 1459, 1461, 1463, 1462, 1464, 1465, 1462, 1466, + 1467, 1462, 1468, 1469, 1472, 1473, 1474, 1462, + 1476, 1462, 1477, 1462, 1478, 1479, 1480, 1481, + 1482, 1483, 1484, 1485, 1486, 1487, 1488, 1489, + 1490, 1491, 1492, 1493, 1494, 1495, 1496, 1497, + 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1505, + 1506, 1507, 1508, 1509, 1510, 1511, 1512, 1462, + 1462, 1462, 1462, 1462, 2, 1462, 1462, 8, + 1462, 1462, 1462, 1462, 1462, 416, 417, 421, + 422, 423, 424, 425, 426, 427, 428, 429, + 430, 431, 432, 434, 436, 437, 469, 510, + 525, 532, 534, 536, 556, 559, 575, 688, + 1462, 1462, 1462, 692, 693, 694, 695, 696, + 697, 698, 699, 700, 701, 702, 704, 705, + 706, 707, 708, 709, 710, 711, 712, 713, + 714, 715, 716, 717, 718, 719, 720, 721, + 722, 723, 724, 726, 727, 728, 729, 730, + 731, 732, 733, 734, 735, 736, 737, 738, + 739, 740, 742, 743, 744, 746, 747, 748, + 749, 750, 751, 752, 753, 754, 755, 756, + 757, 758, 759, 761, 762, 763, 764, 765, + 766, 767, 768, 769, 771, 772, 773, 774, + 775, 776, 777, 778, 779, 780, 781, 782, + 783, 784, 785, 786, 787, 788, 790, 791, + 792, 793, 794, 795, 796, 797, 798, 799, + 800, 801, 802, 803, 804, 805, 806, 807, + 808, 809, 810, 812, 813, 814, 815, 816, + 817, 818, 819, 820, 821, 822, 823, 824, + 825, 826, 827, 856, 881, 884, 885, 887, + 894, 895, 898, 902, 914, 919, 920, 922, + 925, 927, 1513, 1513, 1520, 1522, 1515, 1513, + 1524, 1525, 1526, 1513, 929, 932, 1516, 1517, + 1518, 1519, 1513, 1521, 1513, 1513, 1523, 1513, + 1513, 1513, 934, 935, 940, 941, 1527, 1531, + 1532, 1533, 1527, 936, 939, 1527, 1527, 1527, + 1527, 1527, 942, 947, 948, 1534, 1538, 1539, + 1540, 1534, 943, 946, 1534, 1534, 1534, 1534, + 1534, 1541, 1543, 1544, 1545, 1546, 1547, 1548, + 1549, 1550, 1551, 1552, 1553, 1554, 1555, 1556, + 1557, 1558, 1559, 1560, 1561, 1562, 1563, 1564, + 1565, 1566, 1567, 1568, 1569, 1570, 1571, 1572, + 1573, 1574, 1575, 1576, 1577, 1541, 949, 950, + 954, 955, 956, 957, 958, 959, 960, 961, + 962, 963, 964, 965, 967, 969, 970, 1002, + 1043, 1058, 1065, 1067, 1069, 1089, 1092, 1108, + 1221, 1541, 1225, 1226, 1227, 1228, 1229, 1230, + 1231, 1232, 1233, 1234, 1235, 1237, 1238, 1239, + 1240, 1241, 1242, 1243, 1244, 1245, 1246, 1247, + 1248, 1249, 1250, 1251, 1252, 1253, 1254, 1255, + 1256, 1257, 1259, 1260, 1261, 1262, 1263, 1264, + 1265, 1266, 1267, 1268, 1269, 1270, 1271, 1272, + 1273, 1275, 1276, 1277, 1279, 1280, 1281, 1282, + 1283, 1284, 1285, 1286, 1287, 1288, 1289, 1290, + 1291, 1292, 1294, 1295, 1296, 1297, 1298, 1299, + 1300, 1301, 1302, 1304, 1305, 1306, 1307, 1308, + 1309, 1310, 1311, 1312, 1313, 1314, 1315, 1316, + 1317, 1318, 1319, 1320, 1321, 1323, 1324, 1325, + 1326, 1327, 1328, 1329, 1330, 1331, 1332, 1333, + 1334, 1335, 1336, 1337, 1338, 1339, 1340, 1341, + 1342, 1343, 1345, 1346, 1347, 1348, 1349, 1350, + 1351, 1352, 1353, 1354, 1355, 1356, 1357, 1358, + 1359, 1360, 1389, 1414, 1417, 1418, 1420, 1427, + 1428, 1431, 1435, 1447, 1452, 1453, 1455, 1458, + 1460, } -var _zcltok_trans_actions []byte = []byte{ - 131, 0, 71, 127, 87, 0, 0, 151, - 123, 0, 5, 0, 0, 0, 0, 0, +var _hcltok_trans_actions []byte = []byte{ + 143, 0, 85, 139, 101, 0, 0, 169, + 135, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 101, + 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2257,7 +3444,7 @@ var _zcltok_trans_actions []byte = []byte{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 125, 148, 0, + 0, 0, 0, 0, 0, 137, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2288,34 +3475,7 @@ var _zcltok_trans_actions []byte = []byte{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 129, - 105, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 27, 5, 0, 0, 29, 0, 49, 35, - 47, 136, 0, 0, 0, 0, 69, 55, - 67, 142, 0, 0, 0, 0, 0, 73, - 0, 0, 0, 99, 160, 0, 91, 5, - 154, 5, 0, 0, 93, 0, 95, 0, - 103, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 5, - 5, 5, 157, 157, 157, 157, 157, 157, - 5, 5, 157, 5, 117, 121, 107, 115, - 77, 83, 0, 113, 109, 0, 81, 75, - 89, 79, 111, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 85, 97, + 0, 0, 0, 0, 0, 0, 0, 141, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2328,22 +3488,119 @@ var _zcltok_trans_actions []byte = []byte{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 27, 5, 0, 0, 29, 0, 49, 35, + 47, 148, 0, 0, 0, 0, 69, 55, + 67, 154, 0, 0, 0, 0, 79, 160, + 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 13, 11, 0, 0, 5, 15, 0, 5, - 5, 21, 0, 0, 0, 5, 5, 5, - 23, 0, 17, 7, 0, 19, 9, 25, - 0, 0, 0, 0, 37, 0, 139, 139, - 43, 0, 0, 39, 31, 41, 33, 45, - 0, 0, 0, 57, 0, 145, 145, 63, - 0, 0, 59, 51, 61, 53, 65, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 81, 73, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 87, 0, 0, 113, 178, + 0, 105, 5, 172, 5, 0, 0, 107, + 0, 109, 0, 117, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 5, 5, 5, 175, 175, 175, + 175, 175, 175, 5, 5, 175, 5, 121, + 133, 129, 91, 97, 0, 127, 123, 0, + 95, 89, 103, 93, 125, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 131, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 13, 11, 0, 0, 5, 15, + 0, 5, 5, 21, 0, 0, 0, 5, + 5, 5, 23, 0, 17, 7, 0, 19, + 9, 25, 0, 0, 0, 0, 37, 0, + 151, 151, 43, 0, 0, 39, 31, 41, + 33, 45, 0, 0, 0, 57, 0, 157, + 157, 63, 0, 0, 59, 51, 61, 53, + 65, 71, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 5, 5, 5, 163, 163, 163, 163, 163, + 163, 5, 5, 163, 5, 75, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 77, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, } -var _zcltok_to_state_actions []byte = []byte{ +var _hcltok_to_state_actions []byte = []byte{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2462,7 +3719,71 @@ var _zcltok_to_state_actions []byte = []byte{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2470,12 +3791,17 @@ var _zcltok_to_state_actions []byte = []byte{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 133, - 0, 0, 0, 0, 0, 0, 133, 0, - 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 145, + 0, 0, 0, 0, 0, 0, 145, 0, + 0, 0, 0, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, } -var _zcltok_from_state_actions []byte = []byte{ +var _hcltok_from_state_actions []byte = []byte{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2594,7 +3920,71 @@ var _zcltok_from_state_actions []byte = []byte{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2604,10 +3994,15 @@ var _zcltok_from_state_actions []byte = []byte{ 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, - 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, } -var _zcltok_eof_trans []int16 = []int16{ +var _hcltok_eof_trans []int16 = []int16{ 0, 1, 4, 1, 1, 9, 9, 9, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, @@ -2726,29 +4121,99 @@ var _zcltok_eof_trans []int16 = []int16{ 672, 672, 672, 672, 672, 672, 672, 672, 672, 769, 769, 769, 769, 773, 773, 775, 777, 775, 775, 777, 0, 0, 783, 785, - 783, 783, 785, 0, 0, 0, 845, 846, - 847, 848, 846, 848, 848, 848, 852, 853, - 848, 848, 848, 859, 848, 848, 889, 889, - 889, 889, 889, 889, 889, 889, 889, 889, - 889, 889, 889, 889, 889, 889, 889, 889, - 889, 889, 889, 889, 889, 889, 889, 889, - 889, 889, 889, 889, 889, 889, 889, 889, - 889, 0, 1042, 1042, 1042, 1042, 1042, 1042, - 1049, 1051, 1049, 1054, 1056, 1056, 1056, 0, - 1065, 1068, 1070, 1072, 1072, 1072, 0, 1080, - 1083, 1085, 1087, 1087, 1087, + 783, 783, 785, 0, 0, 791, 791, 793, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 791, 791, 791, 791, 791, 791, 791, 791, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 0, 1192, + 1193, 1194, 1193, 1194, 1194, 1194, 1198, 1199, + 1194, 1194, 1194, 1205, 1194, 1194, 1235, 1235, + 1235, 1235, 1235, 1235, 1235, 1235, 1235, 1235, + 1235, 1235, 1235, 1235, 1235, 1235, 1235, 1235, + 1235, 1235, 1235, 1235, 1235, 1235, 1235, 1235, + 1235, 1235, 1235, 1235, 1235, 1235, 1235, 1235, + 1235, 0, 1388, 1388, 1388, 1388, 1388, 1388, + 1395, 1397, 1395, 1400, 1402, 1402, 1402, 0, + 1411, 1414, 1416, 1418, 1418, 1418, 0, 1426, + 1429, 1431, 1433, 1433, 1433, 0, 1470, 1498, + 1498, 1498, 1498, 1498, 1498, 1498, 1498, 1498, + 1498, 1498, 1498, 1498, 1498, 1498, 1498, 1498, + 1498, 1498, 1498, 1498, 1498, 1498, 1498, 1498, + 1498, 1498, 1498, 1498, 1498, 1498, 1498, 1498, + 1498, 1498, } -const zcltok_start int = 949 -const zcltok_first_final int = 949 -const zcltok_error int = 0 +const hcltok_start int = 1462 +const hcltok_first_final int = 1462 +const hcltok_error int = 0 -const zcltok_en_stringTemplate int = 1001 -const zcltok_en_heredocTemplate int = 1015 -const zcltok_en_bareTemplate int = 1022 -const zcltok_en_main int = 949 +const hcltok_en_stringTemplate int = 1513 +const hcltok_en_heredocTemplate int = 1527 +const hcltok_en_bareTemplate int = 1534 +const hcltok_en_identOnly int = 1541 +const hcltok_en_main int = 1462 -// line 15 "scan_tokens.rl" +// line 16 "scan_tokens.rl" func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []Token { f := &tokenAccum{ @@ -2757,7 +4222,7 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To Pos: start, } - // line 272 "scan_tokens.rl" + // line 276 "scan_tokens.rl" // Ragel state p := 0 // "Pointer" into data @@ -2772,9 +4237,11 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To var cs int // current state switch mode { case scanNormal: - cs = zcltok_en_main + cs = hcltok_en_main case scanTemplate: - cs = zcltok_en_bareTemplate + cs = hcltok_en_bareTemplate + case scanIdentOnly: + cs = hcltok_en_identOnly default: panic("invalid scanMode") } @@ -2783,7 +4250,7 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To var retBraces []int // stack of brace levels that cause us to use fret var heredocs []heredocInProgress // stack of heredocs we're currently processing - // line 305 "scan_tokens.rl" + // line 311 "scan_tokens.rl" // Make Go compiler happy _ = ts @@ -2803,7 +4270,7 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To f.emitToken(TokenType(b[0]), ts, te) } - // line 2816 "scan_tokens.go" + // line 4282 "scan_tokens.go" { top = 0 ts = 0 @@ -2811,7 +4278,7 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To act = 0 } - // line 2824 "scan_tokens.go" + // line 4290 "scan_tokens.go" { var _klen int var _trans int @@ -2825,25 +4292,25 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To goto _out } _resume: - _acts = int(_zcltok_from_state_actions[cs]) - _nacts = uint(_zcltok_actions[_acts]) + _acts = int(_hcltok_from_state_actions[cs]) + _nacts = uint(_hcltok_actions[_acts]) _acts++ for ; _nacts > 0; _nacts-- { _acts++ - switch _zcltok_actions[_acts-1] { + switch _hcltok_actions[_acts-1] { case 2: // line 1 "NONE" ts = p - // line 2848 "scan_tokens.go" + // line 4314 "scan_tokens.go" } } - _keys = int(_zcltok_key_offsets[cs]) - _trans = int(_zcltok_index_offsets[cs]) + _keys = int(_hcltok_key_offsets[cs]) + _trans = int(_hcltok_index_offsets[cs]) - _klen = int(_zcltok_single_lengths[cs]) + _klen = int(_hcltok_single_lengths[cs]) if _klen > 0 { _lower := int(_keys) var _mid int @@ -2855,9 +4322,9 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To _mid = _lower + ((_upper - _lower) >> 1) switch { - case data[p] < _zcltok_trans_keys[_mid]: + case data[p] < _hcltok_trans_keys[_mid]: _upper = _mid - 1 - case data[p] > _zcltok_trans_keys[_mid]: + case data[p] > _hcltok_trans_keys[_mid]: _lower = _mid + 1 default: _trans += int(_mid - int(_keys)) @@ -2868,7 +4335,7 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To _trans += _klen } - _klen = int(_zcltok_range_lengths[cs]) + _klen = int(_hcltok_range_lengths[cs]) if _klen > 0 { _lower := int(_keys) var _mid int @@ -2880,9 +4347,9 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To _mid = _lower + (((_upper - _lower) >> 1) & ^1) switch { - case data[p] < _zcltok_trans_keys[_mid]: + case data[p] < _hcltok_trans_keys[_mid]: _upper = _mid - 2 - case data[p] > _zcltok_trans_keys[_mid+1]: + case data[p] > _hcltok_trans_keys[_mid+1]: _lower = _mid + 2 default: _trans += int((_mid - int(_keys)) >> 1) @@ -2893,27 +4360,27 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To } _match: - _trans = int(_zcltok_indicies[_trans]) + _trans = int(_hcltok_indicies[_trans]) _eof_trans: - cs = int(_zcltok_trans_targs[_trans]) + cs = int(_hcltok_trans_targs[_trans]) - if _zcltok_trans_actions[_trans] == 0 { + if _hcltok_trans_actions[_trans] == 0 { goto _again } - _acts = int(_zcltok_trans_actions[_trans]) - _nacts = uint(_zcltok_actions[_acts]) + _acts = int(_hcltok_trans_actions[_trans]) + _nacts = uint(_hcltok_actions[_acts]) _acts++ for ; _nacts > 0; _nacts-- { _acts++ - switch _zcltok_actions[_acts-1] { + switch _hcltok_actions[_acts-1] { case 3: // line 1 "NONE" te = p + 1 case 4: - // line 138 "scan_tokens.rl" + // line 137 "scan_tokens.rl" te = p + 1 { @@ -2927,12 +4394,12 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To stack = append(stack, 0) stack[top] = cs top++ - cs = 949 + cs = 1462 goto _again } } case 5: - // line 148 "scan_tokens.rl" + // line 147 "scan_tokens.rl" te = p + 1 { @@ -2946,12 +4413,12 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To stack = append(stack, 0) stack[top] = cs top++ - cs = 949 + cs = 1462 goto _again } } case 6: - // line 80 "scan_tokens.rl" + // line 79 "scan_tokens.rl" te = p + 1 { @@ -2965,21 +4432,21 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To } case 7: - // line 222 "scan_tokens.rl" + // line 221 "scan_tokens.rl" te = p + 1 { token(TokenInvalid) } case 8: - // line 223 "scan_tokens.rl" + // line 222 "scan_tokens.rl" te = p + 1 { token(TokenBadUTF8) } case 9: - // line 138 "scan_tokens.rl" + // line 137 "scan_tokens.rl" te = p p-- @@ -2994,12 +4461,12 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To stack = append(stack, 0) stack[top] = cs top++ - cs = 949 + cs = 1462 goto _again } } case 10: - // line 148 "scan_tokens.rl" + // line 147 "scan_tokens.rl" te = p p-- @@ -3014,12 +4481,12 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To stack = append(stack, 0) stack[top] = cs top++ - cs = 949 + cs = 1462 goto _again } } case 11: - // line 221 "scan_tokens.rl" + // line 220 "scan_tokens.rl" te = p p-- @@ -3027,7 +4494,7 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To token(TokenQuotedLit) } case 12: - // line 222 "scan_tokens.rl" + // line 221 "scan_tokens.rl" te = p p-- @@ -3035,7 +4502,7 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To token(TokenInvalid) } case 13: - // line 223 "scan_tokens.rl" + // line 222 "scan_tokens.rl" te = p p-- @@ -3043,29 +4510,29 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To token(TokenBadUTF8) } case 14: - // line 221 "scan_tokens.rl" + // line 220 "scan_tokens.rl" p = (te) - 1 { token(TokenQuotedLit) } case 15: - // line 223 "scan_tokens.rl" + // line 222 "scan_tokens.rl" p = (te) - 1 { token(TokenBadUTF8) } case 16: - // line 126 "scan_tokens.rl" + // line 125 "scan_tokens.rl" act = 10 case 17: - // line 231 "scan_tokens.rl" + // line 230 "scan_tokens.rl" act = 11 case 18: - // line 138 "scan_tokens.rl" + // line 137 "scan_tokens.rl" te = p + 1 { @@ -3079,12 +4546,12 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To stack = append(stack, 0) stack[top] = cs top++ - cs = 949 + cs = 1462 goto _again } } case 19: - // line 148 "scan_tokens.rl" + // line 147 "scan_tokens.rl" te = p + 1 { @@ -3098,12 +4565,12 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To stack = append(stack, 0) stack[top] = cs top++ - cs = 949 + cs = 1462 goto _again } } case 20: - // line 107 "scan_tokens.rl" + // line 106 "scan_tokens.rl" te = p + 1 { @@ -3131,14 +4598,14 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To token(TokenStringLit) } case 21: - // line 231 "scan_tokens.rl" + // line 230 "scan_tokens.rl" te = p + 1 { token(TokenBadUTF8) } case 22: - // line 138 "scan_tokens.rl" + // line 137 "scan_tokens.rl" te = p p-- @@ -3153,12 +4620,12 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To stack = append(stack, 0) stack[top] = cs top++ - cs = 949 + cs = 1462 goto _again } } case 23: - // line 148 "scan_tokens.rl" + // line 147 "scan_tokens.rl" te = p p-- @@ -3173,12 +4640,12 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To stack = append(stack, 0) stack[top] = cs top++ - cs = 949 + cs = 1462 goto _again } } case 24: - // line 126 "scan_tokens.rl" + // line 125 "scan_tokens.rl" te = p p-- @@ -3190,7 +4657,7 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To token(TokenStringLit) } case 25: - // line 231 "scan_tokens.rl" + // line 230 "scan_tokens.rl" te = p p-- @@ -3198,7 +4665,7 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To token(TokenBadUTF8) } case 26: - // line 126 "scan_tokens.rl" + // line 125 "scan_tokens.rl" p = (te) - 1 { @@ -3235,15 +4702,15 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To } case 28: - // line 134 "scan_tokens.rl" + // line 133 "scan_tokens.rl" act = 14 case 29: - // line 238 "scan_tokens.rl" + // line 237 "scan_tokens.rl" act = 15 case 30: - // line 138 "scan_tokens.rl" + // line 137 "scan_tokens.rl" te = p + 1 { @@ -3257,12 +4724,12 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To stack = append(stack, 0) stack[top] = cs top++ - cs = 949 + cs = 1462 goto _again } } case 31: - // line 148 "scan_tokens.rl" + // line 147 "scan_tokens.rl" te = p + 1 { @@ -3276,26 +4743,26 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To stack = append(stack, 0) stack[top] = cs top++ - cs = 949 + cs = 1462 goto _again } } case 32: - // line 134 "scan_tokens.rl" + // line 133 "scan_tokens.rl" te = p + 1 { token(TokenStringLit) } case 33: - // line 238 "scan_tokens.rl" + // line 237 "scan_tokens.rl" te = p + 1 { token(TokenBadUTF8) } case 34: - // line 138 "scan_tokens.rl" + // line 137 "scan_tokens.rl" te = p p-- @@ -3310,12 +4777,12 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To stack = append(stack, 0) stack[top] = cs top++ - cs = 949 + cs = 1462 goto _again } } case 35: - // line 148 "scan_tokens.rl" + // line 147 "scan_tokens.rl" te = p p-- @@ -3330,12 +4797,12 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To stack = append(stack, 0) stack[top] = cs top++ - cs = 949 + cs = 1462 goto _again } } case 36: - // line 134 "scan_tokens.rl" + // line 133 "scan_tokens.rl" te = p p-- @@ -3343,7 +4810,7 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To token(TokenStringLit) } case 37: - // line 238 "scan_tokens.rl" + // line 237 "scan_tokens.rl" te = p p-- @@ -3351,7 +4818,7 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To token(TokenBadUTF8) } case 38: - // line 134 "scan_tokens.rl" + // line 133 "scan_tokens.rl" p = (te) - 1 { @@ -3380,112 +4847,180 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To } case 40: - // line 244 "scan_tokens.rl" + // line 241 "scan_tokens.rl" - act = 18 + act = 16 case 41: - // line 246 "scan_tokens.rl" + // line 242 "scan_tokens.rl" - act = 19 + act = 17 case 42: - // line 257 "scan_tokens.rl" + // line 242 "scan_tokens.rl" - act = 29 + te = p + 1 + { + token(TokenBadUTF8) + } case 43: - // line 268 "scan_tokens.rl" + // line 243 "scan_tokens.rl" - act = 36 + te = p + 1 + { + token(TokenInvalid) + } case 44: - // line 269 "scan_tokens.rl" + // line 241 "scan_tokens.rl" - act = 37 + te = p + p-- + { + token(TokenIdent) + } case 45: - // line 246 "scan_tokens.rl" + // line 242 "scan_tokens.rl" + + te = p + p-- + { + token(TokenBadUTF8) + } + case 46: + // line 241 "scan_tokens.rl" + + p = (te) - 1 + { + token(TokenIdent) + } + case 47: + // line 242 "scan_tokens.rl" + + p = (te) - 1 + { + token(TokenBadUTF8) + } + case 48: + // line 1 "NONE" + + switch act { + case 16: + { + p = (te) - 1 + token(TokenIdent) + } + case 17: + { + p = (te) - 1 + token(TokenBadUTF8) + } + } + + case 49: + // line 249 "scan_tokens.rl" + + act = 21 + case 50: + // line 251 "scan_tokens.rl" + + act = 22 + case 51: + // line 262 "scan_tokens.rl" + + act = 32 + case 52: + // line 272 "scan_tokens.rl" + + act = 38 + case 53: + // line 273 "scan_tokens.rl" + + act = 39 + case 54: + // line 251 "scan_tokens.rl" te = p + 1 { token(TokenComment) } - case 46: - // line 247 "scan_tokens.rl" + case 55: + // line 252 "scan_tokens.rl" te = p + 1 { token(TokenNewline) } - case 47: - // line 249 "scan_tokens.rl" + case 56: + // line 254 "scan_tokens.rl" te = p + 1 { token(TokenEqualOp) } - case 48: - // line 250 "scan_tokens.rl" + case 57: + // line 255 "scan_tokens.rl" te = p + 1 { token(TokenNotEqual) } - case 49: - // line 251 "scan_tokens.rl" + case 58: + // line 256 "scan_tokens.rl" te = p + 1 { token(TokenGreaterThanEq) } - case 50: - // line 252 "scan_tokens.rl" + case 59: + // line 257 "scan_tokens.rl" te = p + 1 { token(TokenLessThanEq) } - case 51: - // line 253 "scan_tokens.rl" + case 60: + // line 258 "scan_tokens.rl" te = p + 1 { token(TokenAnd) } - case 52: - // line 254 "scan_tokens.rl" + case 61: + // line 259 "scan_tokens.rl" te = p + 1 { token(TokenOr) } - case 53: - // line 255 "scan_tokens.rl" + case 62: + // line 260 "scan_tokens.rl" te = p + 1 { token(TokenEllipsis) } - case 54: - // line 256 "scan_tokens.rl" + case 63: + // line 261 "scan_tokens.rl" te = p + 1 { token(TokenFatArrow) } - case 55: - // line 257 "scan_tokens.rl" + case 64: + // line 262 "scan_tokens.rl" te = p + 1 { selfToken() } - case 56: - // line 158 "scan_tokens.rl" + case 65: + // line 157 "scan_tokens.rl" te = p + 1 { token(TokenOBrace) braces++ } - case 57: - // line 163 "scan_tokens.rl" + case 66: + // line 162 "scan_tokens.rl" te = p + 1 { @@ -3505,8 +5040,8 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To braces-- } } - case 58: - // line 175 "scan_tokens.rl" + case 67: + // line 174 "scan_tokens.rl" te = p + 1 { @@ -3535,8 +5070,8 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To braces-- } } - case 59: - // line 75 "scan_tokens.rl" + case 68: + // line 74 "scan_tokens.rl" te = p + 1 { @@ -3545,12 +5080,12 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To stack = append(stack, 0) stack[top] = cs top++ - cs = 1001 + cs = 1513 goto _again } } - case 60: - // line 85 "scan_tokens.rl" + case 69: + // line 84 "scan_tokens.rl" te = p + 1 { @@ -3576,156 +5111,148 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To stack = append(stack, 0) stack[top] = cs top++ - cs = 1015 + cs = 1527 goto _again } } - case 61: - // line 268 "scan_tokens.rl" + case 70: + // line 272 "scan_tokens.rl" te = p + 1 { token(TokenBadUTF8) } - case 62: - // line 269 "scan_tokens.rl" + case 71: + // line 273 "scan_tokens.rl" te = p + 1 { token(TokenInvalid) } - case 63: - // line 242 "scan_tokens.rl" + case 72: + // line 247 "scan_tokens.rl" te = p p-- - case 64: - // line 243 "scan_tokens.rl" + case 73: + // line 248 "scan_tokens.rl" te = p p-- { token(TokenNumberLit) } - case 65: - // line 244 "scan_tokens.rl" + case 74: + // line 249 "scan_tokens.rl" te = p p-- { token(TokenIdent) } - case 66: - // line 246 "scan_tokens.rl" + case 75: + // line 251 "scan_tokens.rl" te = p p-- { token(TokenComment) } - case 67: - // line 257 "scan_tokens.rl" + case 76: + // line 262 "scan_tokens.rl" te = p p-- { selfToken() } - case 68: - // line 267 "scan_tokens.rl" - - te = p - p-- - { - token(TokenTabs) - } - case 69: - // line 268 "scan_tokens.rl" + case 77: + // line 272 "scan_tokens.rl" te = p p-- { token(TokenBadUTF8) } - case 70: - // line 269 "scan_tokens.rl" + case 78: + // line 273 "scan_tokens.rl" te = p p-- { token(TokenInvalid) } - case 71: - // line 243 "scan_tokens.rl" + case 79: + // line 248 "scan_tokens.rl" p = (te) - 1 { token(TokenNumberLit) } - case 72: - // line 244 "scan_tokens.rl" + case 80: + // line 249 "scan_tokens.rl" p = (te) - 1 { token(TokenIdent) } - case 73: - // line 257 "scan_tokens.rl" + case 81: + // line 262 "scan_tokens.rl" p = (te) - 1 { selfToken() } - case 74: - // line 268 "scan_tokens.rl" + case 82: + // line 272 "scan_tokens.rl" p = (te) - 1 { token(TokenBadUTF8) } - case 75: + case 83: // line 1 "NONE" switch act { - case 18: + case 21: { p = (te) - 1 token(TokenIdent) } - case 19: + case 22: { p = (te) - 1 token(TokenComment) } - case 29: + case 32: { p = (te) - 1 selfToken() } - case 36: + case 38: { p = (te) - 1 token(TokenBadUTF8) } - case 37: + case 39: { p = (te) - 1 token(TokenInvalid) } } - // line 3592 "scan_tokens.go" + // line 5104 "scan_tokens.go" } } _again: - _acts = int(_zcltok_to_state_actions[cs]) - _nacts = uint(_zcltok_actions[_acts]) + _acts = int(_hcltok_to_state_actions[cs]) + _nacts = uint(_hcltok_actions[_acts]) _acts++ for ; _nacts > 0; _nacts-- { _acts++ - switch _zcltok_actions[_acts-1] { + switch _hcltok_actions[_acts-1] { case 0: // line 1 "NONE" @@ -3736,7 +5263,7 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To act = 0 - // line 3612 "scan_tokens.go" + // line 5124 "scan_tokens.go" } } @@ -3751,8 +5278,8 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To { } if p == eof { - if _zcltok_eof_trans[cs] > 0 { - _trans = int(_zcltok_eof_trans[cs] - 1) + if _hcltok_eof_trans[cs] > 0 { + _trans = int(_hcltok_eof_trans[cs] - 1) goto _eof_trans } } @@ -3762,12 +5289,12 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To } } - // line 328 "scan_tokens.rl" + // line 334 "scan_tokens.rl" // If we fall out here without being in a final state then we've // encountered something that the scanner can't match, which we'll // deal with as an invalid. - if cs < zcltok_first_final { + if cs < hcltok_first_final { f.emitToken(TokenInvalid, p, len(data)) } diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_tokens.rl b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_tokens.rl index 4a395c133..dc3f56b9b 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_tokens.rl +++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/scan_tokens.rl @@ -1,16 +1,17 @@ -package zclsyntax + +package hclsyntax import ( "bytes" - "github.com/zclconf/go-zcl/zcl" + "github.com/hashicorp/hcl2/hcl" ) // This file is generated from scan_tokens.rl. DO NOT EDIT. %%{ # (except you are actually in scan_tokens.rl here, so edit away!) - machine zcltok; + machine hcltok; write data; }%% @@ -62,15 +63,13 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To ("/*" any* "*/") ); - # Tabs are not valid, but we accept them in the scanner and mark them - # as tokens so that we can produce diagnostics advising the user to - # use spaces instead. - Tabs = 0x09+; - - # Note: zclwrite assumes that only ASCII spaces appear between tokens, + # Note: hclwrite assumes that only ASCII spaces appear between tokens, # and uses this assumption to recreate the spaces between tokens by - # looking at byte offset differences. - Spaces = ' '+; + # looking at byte offset differences. This means it will produce + # incorrect results in the presence of tabs, but that's acceptable + # because the canonical style (which hclwrite itself can impose + # automatically is to never use tabs). + Spaces = (' ' | 0x09)+; action beginStringTemplate { token(TokenOQuote); @@ -238,6 +237,12 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To BrokenUTF8 => { token(TokenBadUTF8); }; *|; + identOnly := |* + Ident => { token(TokenIdent) }; + BrokenUTF8 => { token(TokenBadUTF8) }; + AnyUTF8 => { token(TokenInvalid) }; + *|; + main := |* Spaces => {}; NumberLit => { token(TokenNumberLit) }; @@ -264,7 +269,6 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To BeginStringTmpl => beginStringTemplate; BeginHeredocTmpl => beginHeredocTemplate; - Tabs => { token(TokenTabs) }; BrokenUTF8 => { token(TokenBadUTF8) }; AnyUTF8 => { token(TokenInvalid) }; *|; @@ -284,9 +288,11 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To var cs int // current state switch mode { case scanNormal: - cs = zcltok_en_main + cs = hcltok_en_main case scanTemplate: - cs = zcltok_en_bareTemplate + cs = hcltok_en_bareTemplate + case scanIdentOnly: + cs = hcltok_en_identOnly default: panic("invalid scanMode") } @@ -330,7 +336,7 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To // If we fall out here without being in a final state then we've // encountered something that the scanner can't match, which we'll // deal with as an invalid. - if cs < zcltok_first_final { + if cs < hcltok_first_final { f.emitToken(TokenInvalid, p, len(data)) } diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/structure.go b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/structure.go index eb686d5de..d69f65b62 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/structure.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/structure.go @@ -25,7 +25,7 @@ func (b *Block) AsHCLBlock() *hcl.Block { } } -// Body is the implementation of hcl.Body for the zcl native syntax. +// Body is the implementation of hcl.Body for the HCL native syntax. type Body struct { Attributes Attributes Blocks Blocks diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go index 00d6d720a..53e847d13 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go @@ -110,6 +110,7 @@ type scanMode int const ( scanNormal scanMode = iota scanTemplate + scanIdentOnly ) type tokenAccum struct { diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token_type_string.go b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token_type_string.go index dfe69b21a..93de7ee9d 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token_type_string.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token_type_string.go @@ -2,7 +2,7 @@ package hclsyntax -import "fmt" +import "strconv" const _TokenType_name = "TokenNilTokenNewlineTokenBangTokenPercentTokenBitwiseAndTokenOParenTokenCParenTokenStarTokenPlusTokenCommaTokenMinusTokenDotTokenSlashTokenColonTokenSemicolonTokenLessThanTokenEqualTokenGreaterThanTokenQuestionTokenCommentTokenOHeredocTokenIdentTokenNumberLitTokenQuotedLitTokenStringLitTokenOBrackTokenCBrackTokenBitwiseXorTokenBacktickTokenCHeredocTokenOBraceTokenBitwiseOrTokenCBraceTokenBitwiseNotTokenOQuoteTokenCQuoteTokenTemplateControlTokenEllipsisTokenFatArrowTokenTemplateSeqEndTokenAndTokenOrTokenTemplateInterpTokenEqualOpTokenNotEqualTokenLessThanEqTokenGreaterThanEqTokenEOFTokenTabsTokenStarStarTokenInvalidTokenBadUTF8" @@ -65,5 +65,5 @@ func (i TokenType) String() string { if str, ok := _TokenType_map[i]; ok { return str } - return fmt.Sprintf("TokenType(%d)", i) + return "TokenType(" + strconv.FormatInt(int64(i), 10) + ")" } diff --git a/vendor/github.com/hashicorp/hcl2/hcl/json/navigation.go b/vendor/github.com/hashicorp/hcl2/hcl/json/navigation.go index d28e6037e..307eebb2c 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/json/navigation.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/json/navigation.go @@ -8,7 +8,7 @@ type navigation struct { root *objectVal } -// Implementation of zcled.ContextString +// Implementation of hcled.ContextString func (n navigation) ContextString(offset int) string { steps := navigationStepsRev(n.root, offset) if steps == nil { diff --git a/vendor/github.com/hashicorp/hcl2/hcl/json/parser.go b/vendor/github.com/hashicorp/hcl2/hcl/json/parser.go index 41d05fb4c..f36625e0f 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/json/parser.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/json/parser.go @@ -168,7 +168,7 @@ Token: } if colon.Type == tokenEquals { - // Possible confusion with native zcl syntax. + // Possible confusion with native HCL syntax. return nil, diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "Missing attribute value colon", diff --git a/vendor/github.com/hashicorp/hcl2/hcl/json/public.go b/vendor/github.com/hashicorp/hcl2/hcl/json/public.go index 8d4b05262..04ee9147c 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/json/public.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/json/public.go @@ -9,10 +9,10 @@ import ( ) // Parse attempts to parse the given buffer as JSON and, if successful, returns -// a hcl.File for the zcl configuration represented by it. +// a hcl.File for the HCL configuration represented by it. // // This is not a generic JSON parser. Instead, it deals only with the profile -// of JSON used to express zcl configuration. +// of JSON used to express HCL configuration. // // The returned file is valid only if the returned diagnostics returns false // from its HasErrors method. If HasErrors returns true, the file represents @@ -88,28 +88,3 @@ func ParseFile(filename string) (*hcl.File, hcl.Diagnostics) { return Parse(src, filename) } - -// ParseWithHIL is like Parse except the returned file will use the HIL -// template syntax for expressions in strings, rather than the native zcl -// template syntax. -// -// This is intended for providing backward compatibility for applications that -// used to use HCL/HIL and thus had a JSON-based format with HIL -// interpolations. -func ParseWithHIL(src []byte, filename string) (*hcl.File, hcl.Diagnostics) { - file, diags := Parse(src, filename) - if file != nil && file.Body != nil { - file.Body.(*body).useHIL = true - } - return file, diags -} - -// ParseFileWithHIL is like ParseWithHIL but it reads data from a file before -// parsing it. -func ParseFileWithHIL(filename string) (*hcl.File, hcl.Diagnostics) { - file, diags := ParseFile(filename) - if file != nil && file.Body != nil { - file.Body.(*body).useHIL = true - } - return file, diags -} diff --git a/vendor/github.com/hashicorp/hcl2/hcl/json/spec.md b/vendor/github.com/hashicorp/hcl2/hcl/json/spec.md index d6e8bf696..9b08a4f26 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/json/spec.md +++ b/vendor/github.com/hashicorp/hcl2/hcl/json/spec.md @@ -3,7 +3,7 @@ This is the specification for the JSON serialization for hcl. HCL is a system for defining configuration languages for applications. The HCL information model is designed to support multiple concrete syntaxes for configuration, -and this JSON-based format complements [the native syntax](../zclsyntax/spec.md) +and this JSON-based format complements [the native syntax](../hclsyntax/spec.md) by being easy to machine-generate, whereas the native syntax is oriented towards human authoring and maintenence. diff --git a/vendor/github.com/hashicorp/hcl2/hcl/json/structure.go b/vendor/github.com/hashicorp/hcl2/hcl/json/structure.go index d13607eed..fc9f580db 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/json/structure.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/json/structure.go @@ -3,8 +3,8 @@ package json import ( "fmt" - "github.com/hashicorp/hcl2/hcl/hclsyntax" "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hcl/hclsyntax" "github.com/zclconf/go-cty/cty" ) @@ -17,12 +17,6 @@ type body struct { // be treated as non-existing. This is used when Body.PartialContent is // called, to produce the "remaining content" Body. hiddenAttrs map[string]struct{} - - // If set, string values are turned into expressions using HIL's template - // language, rather than the native zcl language. This is intended to - // allow applications moving from HCL to zcl to continue to parse the - // JSON variant of their config that HCL handled previously. - useHIL bool } // expression is the implementation of "Expression" used for files processed @@ -133,7 +127,6 @@ func (b *body) PartialContent(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Bod unusedBody := &body{ obj: b.obj, hiddenAttrs: usedNames, - useHIL: b.useHIL, } return content, unusedBody, diags @@ -219,8 +212,7 @@ func (b *body) unpackBlock(v node, typeName string, typeRange *hcl.Range, labels Type: typeName, Labels: labels, Body: &body{ - obj: tv, - useHIL: b.useHIL, + obj: tv, }, DefRange: tv.OpenRange, @@ -245,8 +237,7 @@ func (b *body) unpackBlock(v node, typeName string, typeRange *hcl.Range, labels Type: typeName, Labels: labels, Body: &body{ - obj: ov, - useHIL: b.useHIL, + obj: ov, }, DefRange: tv.OpenRange, @@ -269,7 +260,7 @@ func (e *expression) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) { switch v := e.src.(type) { case *stringVal: if ctx != nil { - // Parse string contents as a zcl native language expression. + // Parse string contents as a HCL native language expression. // We only do this if we have a context, so passing a nil context // is how the caller specifies that interpolations are not allowed // and that the string should just be returned verbatim. @@ -279,7 +270,7 @@ func (e *expression) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) { v.SrcRange.Filename, // This won't produce _exactly_ the right result, since - // the zclsyntax parser can't "see" any escapes we removed + // the hclsyntax parser can't "see" any escapes we removed // while parsing JSON, but it's better than nothing. hcl.Pos{ Line: v.SrcRange.Start.Line, @@ -297,8 +288,6 @@ func (e *expression) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) { return val, diags } - // FIXME: Once the native zcl template language parser is implemented, - // parse string values as templates and evaluate them. return cty.StringVal(v.Value), nil case *numberVal: return cty.NumberVal(v.Value), nil @@ -330,8 +319,26 @@ func (e *expression) Variables() []hcl.Traversal { switch v := e.src.(type) { case *stringVal: - // FIXME: Once the native zcl template language parser is implemented, - // parse with that and look for variables in there too, + templateSrc := v.Value + expr, diags := hclsyntax.ParseTemplate( + []byte(templateSrc), + v.SrcRange.Filename, + + // This won't produce _exactly_ the right result, since + // the hclsyntax parser can't "see" any escapes we removed + // while parsing JSON, but it's better than nothing. + hcl.Pos{ + Line: v.SrcRange.Start.Line, + + // skip over the opening quote mark + Byte: v.SrcRange.Start.Byte + 1, + Column: v.SrcRange.Start.Column + 1, + }, + ) + if diags.HasErrors() { + return vars + } + return expr.Variables() case *arrayVal: for _, jsonVal := range v.Values { @@ -353,3 +360,34 @@ func (e *expression) Range() hcl.Range { func (e *expression) StartRange() hcl.Range { return e.src.StartRange() } + +// Implementation for hcl.AbsTraversalForExpr. +func (e *expression) AsTraversal() hcl.Traversal { + // In JSON-based syntax a traversal is given as a string containing + // traversal syntax as defined by hclsyntax.ParseTraversalAbs. + + switch v := e.src.(type) { + case *stringVal: + traversal, diags := hclsyntax.ParseTraversalAbs([]byte(v.Value), v.SrcRange.Filename, v.SrcRange.Start) + if diags.HasErrors() { + return nil + } + return traversal + default: + return nil + } +} + +// Implementation for hcl.ExprList. +func (e *expression) ExprList() []hcl.Expression { + switch v := e.src.(type) { + case *arrayVal: + ret := make([]hcl.Expression, len(v.Values)) + for i, node := range v.Values { + ret[i] = &expression{src: node} + } + return ret + default: + return nil + } +} diff --git a/vendor/github.com/hashicorp/hcl2/hcl/ops.go b/vendor/github.com/hashicorp/hcl2/hcl/ops.go index 80312b010..f4e30b09c 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/ops.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/ops.go @@ -8,7 +8,7 @@ import ( ) // Index is a helper function that performs the same operation as the index -// operator in the zcl expression language. That is, the result is the +// operator in the HCL expression language. That is, the result is the // same as it would be for collection[key] in a configuration expression. // // This is exported so that applications can perform indexing in a manner diff --git a/vendor/github.com/hashicorp/hcl2/hcl/pos.go b/vendor/github.com/hashicorp/hcl2/hcl/pos.go index 3ccdfacb8..1a4b329dc 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/pos.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/pos.go @@ -60,6 +60,40 @@ func RangeBetween(start, end Range) Range { } } +// RangeOver returns a new range that covers both of the given ranges and +// possibly additional content between them if the two ranges do not overlap. +// +// If either range is empty then it is ignored. The result is empty if both +// given ranges are empty. +// +// The result is meaningless if the two ranges to not belong to the same +// source file. +func RangeOver(a, b Range) Range { + if a.Empty() { + return b + } + if b.Empty() { + return a + } + + var start, end Pos + if a.Start.Byte < b.Start.Byte { + start = a.Start + } else { + start = b.Start + } + if a.End.Byte > b.End.Byte { + end = a.End + } else { + end = b.End + } + return Range{ + Filename: a.Filename, + Start: start, + End: end, + } +} + // ContainsOffset returns true if and only if the given byte offset is within // the receiving Range. func (r Range) ContainsOffset(offset int) bool { @@ -94,3 +128,135 @@ func (r Range) String() string { ) } } + +func (r Range) Empty() bool { + return r.Start.Byte == r.End.Byte +} + +// CanSliceBytes returns true if SliceBytes could return an accurate +// sub-slice of the given slice. +// +// This effectively tests whether the start and end offsets of the range +// are within the bounds of the slice, and thus whether SliceBytes can be +// trusted to produce an accurate start and end position within that slice. +func (r Range) CanSliceBytes(b []byte) bool { + switch { + case r.Start.Byte < 0 || r.Start.Byte > len(b): + return false + case r.End.Byte < 0 || r.End.Byte > len(b): + return false + case r.End.Byte < r.Start.Byte: + return false + default: + return true + } +} + +// SliceBytes returns a sub-slice of the given slice that is covered by the +// receiving range, assuming that the given slice is the source code of the +// file indicated by r.Filename. +// +// If the receiver refers to any byte offsets that are outside of the slice +// then the result is constrained to the overlapping portion only, to avoid +// a panic. Use CanSliceBytes to determine if the result is guaranteed to +// be an accurate span of the requested range. +func (r Range) SliceBytes(b []byte) []byte { + start := r.Start.Byte + end := r.End.Byte + if start < 0 { + start = 0 + } else if start > len(b) { + start = len(b) + } + if end < 0 { + end = 0 + } else if end > len(b) { + end = len(b) + } + if end < start { + end = start + } + return b[start:end] +} + +// Overlaps returns true if the receiver and the other given range share any +// characters in common. +func (r Range) Overlaps(other Range) bool { + switch { + case r.Filename != other.Filename: + // If the ranges are in different files then they can't possibly overlap + return false + case r.Empty() || other.Empty(): + // Empty ranges can never overlap + return false + case r.ContainsOffset(other.Start.Byte) || r.ContainsOffset(other.End.Byte): + return true + case other.ContainsOffset(r.Start.Byte) || other.ContainsOffset(r.End.Byte): + return true + default: + return false + } +} + +// Overlap finds a range that is either identical to or a sub-range of both +// the receiver and the other given range. It returns an empty range +// within the receiver if there is no overlap between the two ranges. +// +// A non-empty result is either identical to or a subset of the receiver. +func (r Range) Overlap(other Range) Range { + if !r.Overlaps(other) { + // Start == End indicates an empty range + return Range{ + Filename: r.Filename, + Start: r.Start, + End: r.Start, + } + } + + var start, end Pos + if r.Start.Byte > other.Start.Byte { + start = r.Start + } else { + start = other.Start + } + if r.End.Byte < other.End.Byte { + end = r.End + } else { + end = other.End + } + + return Range{ + Filename: r.Filename, + Start: start, + End: end, + } +} + +// PartitionAround finds the portion of the given range that overlaps with +// the reciever and returns three ranges: the portion of the reciever that +// precedes the overlap, the overlap itself, and then the portion of the +// reciever that comes after the overlap. +// +// If the two ranges do not overlap then all three returned ranges are empty. +// +// If the given range aligns with or extends beyond either extent of the +// reciever then the corresponding outer range will be empty. +func (r Range) PartitionAround(other Range) (before, overlap, after Range) { + overlap = r.Overlap(other) + if overlap.Empty() { + return overlap, overlap, overlap + } + + before = Range{ + Filename: r.Filename, + Start: r.Start, + End: overlap.Start, + } + after = Range{ + Filename: r.Filename, + Start: overlap.End, + End: r.End, + } + + return before, overlap, after +} diff --git a/vendor/github.com/hashicorp/hcl2/hcl/pos_scanner.go b/vendor/github.com/hashicorp/hcl2/hcl/pos_scanner.go new file mode 100644 index 000000000..7c8f2dfa5 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl2/hcl/pos_scanner.go @@ -0,0 +1,148 @@ +package hcl + +import ( + "bufio" + "bytes" + + "github.com/apparentlymart/go-textseg/textseg" +) + +// RangeScanner is a helper that will scan over a buffer using a bufio.SplitFunc +// and visit a source range for each token matched. +// +// For example, this can be used with bufio.ScanLines to find the source range +// for each line in the file, skipping over the actual newline characters, which +// may be useful when printing source code snippets as part of diagnostic +// messages. +// +// The line and column information in the returned ranges is produced by +// counting newline characters and grapheme clusters respectively, which +// mimics the behavior we expect from a parser when producing ranges. +type RangeScanner struct { + filename string + b []byte + cb bufio.SplitFunc + + pos Pos // position of next byte to process in b + cur Range // latest range + tok []byte // slice of b that is covered by cur + err error // error from last scan, if any +} + +// Create a new RangeScanner for the given buffer, producing ranges for the +// given filename. +// +// Since ranges have grapheme-cluster granularity rather than byte granularity, +// the scanner will produce incorrect results if the given SplitFunc creates +// tokens between grapheme cluster boundaries. In particular, it is incorrect +// to use RangeScanner with bufio.ScanRunes because it will produce tokens +// around individual UTF-8 sequences, which will split any multi-sequence +// grapheme clusters. +func NewRangeScanner(b []byte, filename string, cb bufio.SplitFunc) *RangeScanner { + return &RangeScanner{ + filename: filename, + b: b, + cb: cb, + pos: Pos{ + Byte: 0, + Line: 1, + Column: 1, + }, + } +} + +func (sc *RangeScanner) Scan() bool { + if sc.pos.Byte >= len(sc.b) || sc.err != nil { + // All done + return false + } + + // Since we're operating on an in-memory buffer, we always pass the whole + // remainder of the buffer to our SplitFunc and set isEOF to let it know + // that it has the whole thing. + advance, token, err := sc.cb(sc.b[sc.pos.Byte:], true) + + // Since we are setting isEOF to true this should never happen, but + // if it does we will just abort and assume the SplitFunc is misbehaving. + if advance == 0 && token == nil && err == nil { + return false + } + + if err != nil { + sc.err = err + sc.cur = Range{ + Filename: sc.filename, + Start: sc.pos, + End: sc.pos, + } + sc.tok = nil + return false + } + + sc.tok = token + start := sc.pos + end := sc.pos + new := sc.pos + + // adv is similar to token but it also includes any subsequent characters + // we're being asked to skip over by the SplitFunc. + // adv is a slice covering any additional bytes we are skipping over, based + // on what the SplitFunc told us to do with advance. + adv := sc.b[sc.pos.Byte : sc.pos.Byte+advance] + + // We now need to scan over our token to count the grapheme clusters + // so we can correctly advance Column, and count the newlines so we + // can correctly advance Line. + advR := bytes.NewReader(adv) + gsc := bufio.NewScanner(advR) + advanced := 0 + gsc.Split(textseg.ScanGraphemeClusters) + for gsc.Scan() { + gr := gsc.Bytes() + new.Byte += len(gr) + new.Column++ + + // We rely here on the fact that \r\n is considered a grapheme cluster + // and so we don't need to worry about miscounting additional lines + // on files with Windows-style line endings. + if len(gr) != 0 && (gr[0] == '\r' || gr[0] == '\n') { + new.Column = 1 + new.Line++ + } + + if advanced < len(token) { + // If we've not yet found the end of our token then we'll + // also push our "end" marker along. + // (if advance > len(token) then we'll stop moving "end" early + // so that the caller only sees the range covered by token.) + end = new + } + advanced += len(gr) + } + + sc.cur = Range{ + Filename: sc.filename, + Start: start, + End: end, + } + sc.pos = new + return true +} + +// Range returns a range that covers the latest token obtained after a call +// to Scan returns true. +func (sc *RangeScanner) Range() Range { + return sc.cur +} + +// Bytes returns the slice of the input buffer that is covered by the range +// that would be returned by Range. +func (sc *RangeScanner) Bytes() []byte { + return sc.tok +} + +// Err can be called after Scan returns false to determine if the latest read +// resulted in an error, and obtain that error if so. +func (sc *RangeScanner) Err() error { + return sc.err +} diff --git a/vendor/github.com/hashicorp/hcl2/hcl/spec.md b/vendor/github.com/hashicorp/hcl2/hcl/spec.md index db4e9ef97..e50bd73a2 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/spec.md +++ b/vendor/github.com/hashicorp/hcl2/hcl/spec.md @@ -7,7 +7,7 @@ concrete syntaxes for configuration, each with a mapping to the model defined in this specification. The two primary syntaxes intended for use in conjunction with this model are -[the HCL native syntax](./zclsyntax/spec.md) and [the JSON syntax](./json/spec.md). +[the HCL native syntax](./hclsyntax/spec.md) and [the JSON syntax](./json/spec.md). In principle other syntaxes are possible as long as either their language model is sufficiently rich to express the concepts described in this specification or the language targets a well-defined subset of the specification. @@ -159,7 +159,7 @@ a computation in terms of literal values, variables, and functions. Each syntax defines its own representation of expressions. For syntaxes based in languages that do not have any non-literal expression syntax, it is recommended to embed the template language from -[the native syntax](./zclsyntax/spec.md) e.g. as a post-processing step on +[the native syntax](./hclsyntax/spec.md) e.g. as a post-processing step on string literals. ### Expression Evaluation diff --git a/vendor/github.com/hashicorp/hcl2/hcl/traversal.go b/vendor/github.com/hashicorp/hcl2/hcl/traversal.go index 867ed425f..24f4c91b7 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/traversal.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/traversal.go @@ -156,6 +156,17 @@ func (t Traversal) RootName() string { return t[0].(TraverseRoot).Name } +// SourceRange returns the source range for the traversal. +func (t Traversal) SourceRange() Range { + if len(t) == 0 { + // Nothing useful to return here, but we'll return something + // that's correctly-typed at least. + return Range{} + } + + return RangeBetween(t[0].SourceRange(), t[len(t)-1].SourceRange()) +} + // TraversalSplit represents a pair of traversals, the first of which is // an absolute traversal and the second of which is relative to the first. // @@ -206,6 +217,7 @@ func (t TraversalSplit) RootName() string { // A Traverser is a step within a Traversal. type Traverser interface { TraversalStep(cty.Value) (cty.Value, Diagnostics) + SourceRange() Range isTraverserSigil() isTraverser } @@ -231,6 +243,10 @@ func (tn TraverseRoot) TraversalStep(cty.Value) (cty.Value, Diagnostics) { panic("Cannot traverse an absolute traversal") } +func (tn TraverseRoot) SourceRange() Range { + return tn.SrcRange +} + // TraverseAttr looks up an attribute in its initial value. type TraverseAttr struct { isTraverser @@ -301,6 +317,10 @@ func (tn TraverseAttr) TraversalStep(val cty.Value) (cty.Value, Diagnostics) { } } +func (tn TraverseAttr) SourceRange() Range { + return tn.SrcRange +} + // TraverseIndex applies the index operation to its initial value. type TraverseIndex struct { isTraverser @@ -312,6 +332,10 @@ func (tn TraverseIndex) TraversalStep(val cty.Value) (cty.Value, Diagnostics) { return Index(val, tn.Key, &tn.SrcRange) } +func (tn TraverseIndex) SourceRange() Range { + return tn.SrcRange +} + // TraverseSplat applies the splat operation to its initial value. type TraverseSplat struct { isTraverser @@ -322,3 +346,7 @@ type TraverseSplat struct { func (tn TraverseSplat) TraversalStep(val cty.Value) (cty.Value, Diagnostics) { panic("TraverseSplat not yet implemented") } + +func (tn TraverseSplat) SourceRange() Range { + return tn.SrcRange +} diff --git a/vendor/github.com/hashicorp/hcl2/hcl/traversal_for_expr.go b/vendor/github.com/hashicorp/hcl2/hcl/traversal_for_expr.go new file mode 100644 index 000000000..5f529467b --- /dev/null +++ b/vendor/github.com/hashicorp/hcl2/hcl/traversal_for_expr.go @@ -0,0 +1,121 @@ +package hcl + +// AbsTraversalForExpr attempts to interpret the given expression as +// an absolute traversal, or returns error diagnostic(s) if that is +// not possible for the given expression. +// +// A particular Expression implementation can support this function by +// offering a method called AsTraversal that takes no arguments and +// returns either a valid absolute traversal or nil to indicate that +// no traversal is possible. Alternatively, an implementation can support +// UnwrapExpression to delegate handling of this function to a wrapped +// Expression object. +// +// In most cases the calling application is interested in the value +// that results from an expression, but in rarer cases the application +// needs to see the the name of the variable and subsequent +// attributes/indexes itself, for example to allow users to give references +// to the variables themselves rather than to their values. An implementer +// of this function should at least support attribute and index steps. +func AbsTraversalForExpr(expr Expression) (Traversal, Diagnostics) { + type asTraversal interface { + AsTraversal() Traversal + } + + physExpr := UnwrapExpressionUntil(expr, func(expr Expression) bool { + _, supported := expr.(asTraversal) + return supported + }) + + if asT, supported := physExpr.(asTraversal); supported { + if traversal := asT.AsTraversal(); traversal != nil { + return traversal, nil + } + } + return nil, Diagnostics{ + &Diagnostic{ + Severity: DiagError, + Summary: "Invalid expression", + Detail: "A static variable reference is required.", + Subject: expr.Range().Ptr(), + }, + } +} + +// RelTraversalForExpr is similar to AbsTraversalForExpr but it returns +// a relative traversal instead. Due to the nature of HCL expressions, the +// first element of the returned traversal is always a TraverseAttr, and +// then it will be followed by zero or more other expressions. +// +// Any expression accepted by AbsTraversalForExpr is also accepted by +// RelTraversalForExpr. +func RelTraversalForExpr(expr Expression) (Traversal, Diagnostics) { + traversal, diags := AbsTraversalForExpr(expr) + if len(traversal) > 0 { + root := traversal[0].(TraverseRoot) + traversal[0] = TraverseAttr{ + Name: root.Name, + SrcRange: root.SrcRange, + } + } + return traversal, diags +} + +// ExprAsKeyword attempts to interpret the given expression as a static keyword, +// returning the keyword string if possible, and the empty string if not. +// +// A static keyword, for the sake of this function, is a single identifier. +// For example, the following attribute has an expression that would produce +// the keyword "foo": +// +// example = foo +// +// This function is a variant of AbsTraversalForExpr, which uses the same +// interface on the given expression. This helper constrains the result +// further by requiring only a single root identifier. +// +// This function is intended to be used with the following idiom, to recognize +// situations where one of a fixed set of keywords is required and arbitrary +// expressions are not allowed: +// +// switch hcl.ExprAsKeyword(expr) { +// case "allow": +// // (take suitable action for keyword "allow") +// case "deny": +// // (take suitable action for keyword "deny") +// default: +// diags = append(diags, &hcl.Diagnostic{ +// // ... "invalid keyword" diagnostic message ... +// }) +// } +// +// The above approach will generate the same message for both the use of an +// unrecognized keyword and for not using a keyword at all, which is usually +// reasonable if the message specifies that the given value must be a keyword +// from that fixed list. +// +// Note that in the native syntax the keywords "true", "false", and "null" are +// recognized as literal values during parsing and so these reserved words +// cannot not be accepted as keywords by this function. +// +// Since interpreting an expression as a keyword bypasses usual expression +// evaluation, it should be used sparingly for situations where e.g. one of +// a fixed set of keywords is used in a structural way in a special attribute +// to affect the further processing of a block. +func ExprAsKeyword(expr Expression) string { + type asTraversal interface { + AsTraversal() Traversal + } + + physExpr := UnwrapExpressionUntil(expr, func(expr Expression) bool { + _, supported := expr.(asTraversal) + return supported + }) + + if asT, supported := physExpr.(asTraversal); supported { + if traversal := asT.AsTraversal(); len(traversal) == 1 { + return traversal.RootName() + } + } + return "" +} diff --git a/vendor/github.com/hashicorp/hcl2/hcldec/public.go b/vendor/github.com/hashicorp/hcl2/hcldec/public.go index 3e58f7b3c..5d1f10a3d 100644 --- a/vendor/github.com/hashicorp/hcl2/hcldec/public.go +++ b/vendor/github.com/hashicorp/hcl2/hcldec/public.go @@ -51,3 +51,28 @@ func ImpliedType(spec Spec) cty.Type { func SourceRange(body hcl.Body, spec Spec) hcl.Range { return sourceRange(body, nil, spec) } + +// ChildBlockTypes returns a map of all of the child block types declared +// by the given spec, with block type names as keys and the associated +// nested body specs as values. +func ChildBlockTypes(spec Spec) map[string]Spec { + ret := map[string]Spec{} + + // visitSameBodyChildren walks through the spec structure, calling + // the given callback for each descendent spec encountered. We are + // interested in the specs that reference attributes and blocks. + var visit visitFunc + visit = func(s Spec) { + if bs, ok := s.(blockSpec); ok { + for _, blockS := range bs.blockHeaderSchemata() { + ret[blockS.Type] = bs.nestedSpec() + } + } + + s.visitSameBodyChildren(visit) + } + + visit(spec) + + return ret +} diff --git a/vendor/github.com/hashicorp/hcl2/hcldec/spec.go b/vendor/github.com/hashicorp/hcl2/hcldec/spec.go index f0e6842be..25cafcd97 100644 --- a/vendor/github.com/hashicorp/hcl2/hcldec/spec.go +++ b/vendor/github.com/hashicorp/hcl2/hcldec/spec.go @@ -7,6 +7,7 @@ import ( "github.com/hashicorp/hcl2/hcl" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/convert" + "github.com/zclconf/go-cty/cty/function" ) // A Spec is a description of how to decode a hcl.Body to a cty.Value. @@ -52,6 +53,7 @@ type attrSpec interface { // blockSpec is implemented by specs that require blocks from the body. type blockSpec interface { blockHeaderSchemata() []hcl.BlockHeaderSchema + nestedSpec() Spec } // specNeedingVariables is implemented by specs that can use variables @@ -298,6 +300,11 @@ func (s *BlockSpec) blockHeaderSchemata() []hcl.BlockHeaderSchema { } } +// blockSpec implementation +func (s *BlockSpec) nestedSpec() Spec { + return s.Nested +} + // specNeedingVariables implementation func (s *BlockSpec) variablesNeeded(content *hcl.BodyContent) []hcl.Traversal { var childBlock *hcl.Block @@ -409,6 +416,11 @@ func (s *BlockListSpec) blockHeaderSchemata() []hcl.BlockHeaderSchema { } } +// blockSpec implementation +func (s *BlockListSpec) nestedSpec() Spec { + return s.Nested +} + // specNeedingVariables implementation func (s *BlockListSpec) variablesNeeded(content *hcl.BodyContent) []hcl.Traversal { var ret []hcl.Traversal @@ -519,6 +531,11 @@ func (s *BlockSetSpec) blockHeaderSchemata() []hcl.BlockHeaderSchema { } } +// blockSpec implementation +func (s *BlockSetSpec) nestedSpec() Spec { + return s.Nested +} + // specNeedingVariables implementation func (s *BlockSetSpec) variablesNeeded(content *hcl.BodyContent) []hcl.Traversal { var ret []hcl.Traversal @@ -631,6 +648,11 @@ func (s *BlockMapSpec) blockHeaderSchemata() []hcl.BlockHeaderSchema { } } +// blockSpec implementation +func (s *BlockMapSpec) nestedSpec() Spec { + return s.Nested +} + // specNeedingVariables implementation func (s *BlockMapSpec) variablesNeeded(content *hcl.BodyContent) []hcl.Traversal { var ret []hcl.Traversal @@ -857,3 +879,120 @@ func (s *DefaultSpec) sourceRange(content *hcl.BodyContent, blockLabels []blockL // reasonable source range to return anyway. return s.Primary.sourceRange(content, blockLabels) } + +// TransformExprSpec is a spec that wraps another and then evaluates a given +// hcl.Expression on the result. +// +// The implied type of this spec is determined by evaluating the expression +// with an unknown value of the nested spec's implied type, which may cause +// the result to be imprecise. This spec should not be used in situations where +// precise result type information is needed. +type TransformExprSpec struct { + Wrapped Spec + Expr hcl.Expression + TransformCtx *hcl.EvalContext + VarName string +} + +func (s *TransformExprSpec) visitSameBodyChildren(cb visitFunc) { + cb(s.Wrapped) +} + +func (s *TransformExprSpec) decode(content *hcl.BodyContent, blockLabels []blockLabel, ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) { + wrappedVal, diags := s.Wrapped.decode(content, blockLabels, ctx) + if diags.HasErrors() { + // We won't try to run our function in this case, because it'll probably + // generate confusing additional errors that will distract from the + // root cause. + return cty.UnknownVal(s.impliedType()), diags + } + + chiCtx := s.TransformCtx.NewChild() + chiCtx.Variables = map[string]cty.Value{ + s.VarName: wrappedVal, + } + resultVal, resultDiags := s.Expr.Value(chiCtx) + diags = append(diags, resultDiags...) + return resultVal, diags +} + +func (s *TransformExprSpec) impliedType() cty.Type { + wrappedTy := s.Wrapped.impliedType() + chiCtx := s.TransformCtx.NewChild() + chiCtx.Variables = map[string]cty.Value{ + s.VarName: cty.UnknownVal(wrappedTy), + } + resultVal, _ := s.Expr.Value(chiCtx) + return resultVal.Type() +} + +func (s *TransformExprSpec) sourceRange(content *hcl.BodyContent, blockLabels []blockLabel) hcl.Range { + // We'll just pass through our wrapped range here, even though that's + // not super-accurate, because there's nothing better to return. + return s.Wrapped.sourceRange(content, blockLabels) +} + +// TransformFuncSpec is a spec that wraps another and then evaluates a given +// cty function with the result. The given function must expect exactly one +// argument, where the result of the wrapped spec will be passed. +// +// The implied type of this spec is determined by type-checking the function +// with an unknown value of the nested spec's implied type, which may cause +// the result to be imprecise. This spec should not be used in situations where +// precise result type information is needed. +// +// If the given function produces an error when run, this spec will produce +// a non-user-actionable diagnostic message. It's the caller's responsibility +// to ensure that the given function cannot fail for any non-error result +// of the wrapped spec. +type TransformFuncSpec struct { + Wrapped Spec + Func function.Function +} + +func (s *TransformFuncSpec) visitSameBodyChildren(cb visitFunc) { + cb(s.Wrapped) +} + +func (s *TransformFuncSpec) decode(content *hcl.BodyContent, blockLabels []blockLabel, ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) { + wrappedVal, diags := s.Wrapped.decode(content, blockLabels, ctx) + if diags.HasErrors() { + // We won't try to run our function in this case, because it'll probably + // generate confusing additional errors that will distract from the + // root cause. + return cty.UnknownVal(s.impliedType()), diags + } + + resultVal, err := s.Func.Call([]cty.Value{wrappedVal}) + if err != nil { + // This is not a good example of a diagnostic because it is reporting + // a programming error in the calling application, rather than something + // an end-user could act on. + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Transform function failed", + Detail: fmt.Sprintf("Decoder transform returned an error: %s", err), + Subject: s.sourceRange(content, blockLabels).Ptr(), + }) + return cty.UnknownVal(s.impliedType()), diags + } + + return resultVal, diags +} + +func (s *TransformFuncSpec) impliedType() cty.Type { + wrappedTy := s.Wrapped.impliedType() + resultTy, err := s.Func.ReturnType([]cty.Type{wrappedTy}) + if err != nil { + // Should never happen with a correctly-configured spec + return cty.DynamicPseudoType + } + + return resultTy +} + +func (s *TransformFuncSpec) sourceRange(content *hcl.BodyContent, blockLabels []blockLabel) hcl.Range { + // We'll just pass through our wrapped range here, even though that's + // not super-accurate, because there's nothing better to return. + return s.Wrapped.sourceRange(content, blockLabels) +} diff --git a/vendor/github.com/hashicorp/hcl2/hcltest/mock.go b/vendor/github.com/hashicorp/hcl2/hcltest/mock.go index a886bed8e..510fc2497 100644 --- a/vendor/github.com/hashicorp/hcl2/hcltest/mock.go +++ b/vendor/github.com/hashicorp/hcl2/hcltest/mock.go @@ -3,13 +3,15 @@ package hcltest import ( "fmt" + "github.com/hashicorp/hcl2/hcl/hclsyntax" + "github.com/hashicorp/hcl2/hcl" "github.com/zclconf/go-cty/cty" ) // MockBody returns a hcl.Body implementation that works in terms of a // caller-constructed hcl.BodyContent, thus avoiding the need to parse -// a "real" zcl config file to use as input to a test. +// a "real" HCL config file to use as input to a test. func MockBody(content *hcl.BodyContent) hcl.Body { return mockBody{content} } @@ -149,6 +151,21 @@ func (e mockExprLiteral) StartRange() hcl.Range { return e.Range() } +// Implementation for hcl.ExprList +func (e mockExprLiteral) ExprList() []hcl.Expression { + v := e.V + ty := v.Type() + if v.IsKnown() && !v.IsNull() && (ty.IsListType() || ty.IsTupleType()) { + ret := make([]hcl.Expression, 0, v.LengthInt()) + for it := v.ElementIterator(); it.Next(); { + _, v := it.Element() + ret = append(ret, MockExprLiteral(v)) + } + return ret + } + return nil +} + // MockExprVariable returns a hcl.Expression that evaluates to the value of // the variable with the given name. func MockExprVariable(name string) hcl.Expression { @@ -197,6 +214,111 @@ func (e mockExprVariable) StartRange() hcl.Range { return e.Range() } +// Implementation for hcl.AbsTraversalForExpr and hcl.RelTraversalForExpr. +func (e mockExprVariable) AsTraversal() hcl.Traversal { + return hcl.Traversal{ + hcl.TraverseRoot{ + Name: string(e), + SrcRange: e.Range(), + }, + } +} + +// MockExprTraversal returns a hcl.Expression that evaluates the given +// absolute traversal. +func MockExprTraversal(traversal hcl.Traversal) hcl.Expression { + return mockExprTraversal{ + Traversal: traversal, + } +} + +// MockExprTraversalSrc is like MockExprTraversal except it takes a +// traversal string as defined by the native syntax and parses it first. +// +// This method is primarily for testing with hard-coded traversal strings, so +// it will panic if the given string is not syntactically correct. +func MockExprTraversalSrc(src string) hcl.Expression { + traversal, diags := hclsyntax.ParseTraversalAbs([]byte(src), "MockExprTraversal", hcl.Pos{}) + if diags.HasErrors() { + panic("invalid traversal string") + } + return MockExprTraversal(traversal) +} + +type mockExprTraversal struct { + Traversal hcl.Traversal +} + +func (e mockExprTraversal) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) { + return e.Traversal.TraverseAbs(ctx) +} + +func (e mockExprTraversal) Variables() []hcl.Traversal { + return []hcl.Traversal{e.Traversal} +} + +func (e mockExprTraversal) Range() hcl.Range { + return e.Traversal.SourceRange() +} + +func (e mockExprTraversal) StartRange() hcl.Range { + return e.Range() +} + +// Implementation for hcl.AbsTraversalForExpr and hcl.RelTraversalForExpr. +func (e mockExprTraversal) AsTraversal() hcl.Traversal { + return e.Traversal +} + +func MockExprList(exprs []hcl.Expression) hcl.Expression { + return mockExprList{ + Exprs: exprs, + } +} + +type mockExprList struct { + Exprs []hcl.Expression +} + +func (e mockExprList) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) { + if len(e.Exprs) == 0 { + return cty.ListValEmpty(cty.DynamicPseudoType), nil + } + vals := make([]cty.Value, 0, len(e.Exprs)) + var diags hcl.Diagnostics + + for _, expr := range e.Exprs { + val, valDiags := expr.Value(ctx) + diags = append(diags, valDiags...) + vals = append(vals, val) + } + + return cty.ListVal(vals), diags +} + +func (e mockExprList) Variables() []hcl.Traversal { + var traversals []hcl.Traversal + for _, expr := range e.Exprs { + traversals = append(traversals, expr.Variables()...) + } + return traversals +} + +func (e mockExprList) Range() hcl.Range { + return hcl.Range{ + Filename: "MockExprList", + } +} + +func (e mockExprList) StartRange() hcl.Range { + return e.Range() +} + +// Implementation for hcl.ExprList +func (e mockExprList) ExprList() []hcl.Expression { + return e.Exprs +} + // MockAttrs constructs and returns a hcl.Attributes map with attributes // derived from the given expression map. // diff --git a/vendor/github.com/hashicorp/hcl2/hclwrite/ast.go b/vendor/github.com/hashicorp/hcl2/hclwrite/ast.go index 35c746bd6..19d00703c 100644 --- a/vendor/github.com/hashicorp/hcl2/hclwrite/ast.go +++ b/vendor/github.com/hashicorp/hcl2/hclwrite/ast.go @@ -82,7 +82,7 @@ func (n *Body) AppendUnstructuredTokens(seq *TokenSeq) { // given name, or returns nil if there is currently no matching attribute. // // A valid AST has only one definition of each attribute, but that constraint -// is not enforced in the zclwrite AST, so a tree that has been mutated by +// is not enforced in the hclwrite AST, so a tree that has been mutated by // other calls may contain additional matching attributes that cannot be seen // by this method. func (n *Body) FindAttribute(name string) *Attribute { diff --git a/vendor/github.com/hashicorp/hcl2/hclwrite/format.go b/vendor/github.com/hashicorp/hcl2/hclwrite/format.go index 1ccc78f5f..64f161dab 100644 --- a/vendor/github.com/hashicorp/hcl2/hclwrite/format.go +++ b/vendor/github.com/hashicorp/hcl2/hclwrite/format.go @@ -49,7 +49,7 @@ func formatIndent(lines []formatLine) { // We'll start our indent stack at a reasonable capacity to minimize the // chance of us needing to grow it; 10 here means 10 levels of indent, - // which should be more than enough for reasonable zcl uses. + // which should be more than enough for reasonable HCL uses. indents := make([]int, 0, 10) for i := range lines { diff --git a/vendor/github.com/hashicorp/hcl2/hclwrite/parser.go b/vendor/github.com/hashicorp/hcl2/hclwrite/parser.go index 72802d311..4a4851adc 100644 --- a/vendor/github.com/hashicorp/hcl2/hclwrite/parser.go +++ b/vendor/github.com/hashicorp/hcl2/hclwrite/parser.go @@ -3,18 +3,18 @@ package hclwrite import ( "sort" - "github.com/hashicorp/hcl2/hcl/hclsyntax" "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hcl/hclsyntax" ) // Our "parser" here is actually not doing any parsing of its own. Instead, -// it leans on the native parser in zclsyntax, and then uses the source ranges +// it leans on the native parser in hclsyntax, and then uses the source ranges // from the AST to partition the raw token sequence to match the raw tokens // up to AST nodes. // // This strategy feels somewhat counter-intuitive, since most of the work the // parser does is thrown away here, but this strategy is chosen because the -// normal parsing work done by zclsyntax is considered to be the "main case", +// normal parsing work done by hclsyntax is considered to be the "main case", // while modifying and re-printing source is more of an edge case, used only // in ancillary tools, and so it's good to keep all the main parsing logic // with the main case but keep all of the extra complexity of token wrangling @@ -30,7 +30,7 @@ func parse(src []byte, filename string, start hcl.Pos) (*File, hcl.Diagnostics) return nil, diags } - // To do our work here, we use the "native" tokens (those from zclsyntax) + // To do our work here, we use the "native" tokens (those from hclsyntax) // to match against source ranges in the AST, but ultimately produce // slices from our sequence of "writer" tokens, which contain only // *relative* position information that is more appropriate for @@ -333,7 +333,7 @@ func parseExpression(nativeExpr hclsyntax.Expression, from inputTokens) *Express } } -// writerTokens takes a sequence of tokens as produced by the main zclsyntax +// writerTokens takes a sequence of tokens as produced by the main hclsyntax // package and transforms it into an equivalent sequence of tokens using // this package's own token model. // @@ -389,7 +389,7 @@ func writerTokens(nativeTokens hclsyntax.Tokens) Tokens { // true then it will make a best effort that may produce strange results at // the boundaries. // -// Native zclsyntax tokens are used here, because they contain the necessary +// Native hclsyntax tokens are used here, because they contain the necessary // absolute position information. However, since writerTokens produces a // correlatable sequence of writer tokens, the resulting indices can be // used also to index into its result, allowing the partitioning of writer @@ -488,7 +488,7 @@ func partitionLineEndTokens(toks hclsyntax.Tokens) (afterComment, afterNewline i return len(toks), len(toks) } -// lexConfig uses the zclsyntax scanner to get a token stream and then +// lexConfig uses the hclsyntax scanner to get a token stream and then // rewrites it into this package's token model. // // Any errors produced during scanning are ignored, so the results of this diff --git a/vendor/github.com/hashicorp/hcl2/hclwrite/public.go b/vendor/github.com/hashicorp/hcl2/hclwrite/public.go index ee2d92044..d9b0dd5aa 100644 --- a/vendor/github.com/hashicorp/hcl2/hclwrite/public.go +++ b/vendor/github.com/hashicorp/hcl2/hclwrite/public.go @@ -6,7 +6,7 @@ import ( "github.com/hashicorp/hcl2/hcl" ) -// ParseConfig interprets the given source bytes into a *zclwrite.File. The +// ParseConfig interprets the given source bytes into a *hclwrite.File. The // resulting AST can be used to perform surgical edits on the source code // before turning it back into bytes again. func ParseConfig(src []byte, filename string, start hcl.Pos) (*File, hcl.Diagnostics) { diff --git a/vendor/github.com/hashicorp/hcl2/hclwrite/tokens.go b/vendor/github.com/hashicorp/hcl2/hclwrite/tokens.go index 18689ea47..8fb87f252 100644 --- a/vendor/github.com/hashicorp/hcl2/hclwrite/tokens.go +++ b/vendor/github.com/hashicorp/hcl2/hclwrite/tokens.go @@ -9,7 +9,7 @@ import ( ) // TokenGen is an abstract type that can append tokens to a list. It is the -// low-level foundation underlying the zclwrite AST; the AST provides a +// low-level foundation underlying the hclwrite AST; the AST provides a // convenient abstraction over raw token sequences to facilitate common tasks, // but it's also possible to directly manipulate the tree of token generators // to make changes that the AST API doesn't directly allow. @@ -22,7 +22,7 @@ type TokenGen interface { type TokenCallback func(*Token) // Token is a single sequence of bytes annotated with a type. It is similar -// in purpose to zclsyntax.Token, but discards the source position information +// in purpose to hclsyntax.Token, but discards the source position information // since that is not useful in code generation. type Token struct { Type hclsyntax.TokenType diff --git a/vendor/vendor.json b/vendor/vendor.json index 0a9fdf263..3465886dd 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -1697,56 +1697,56 @@ { "checksumSHA1": "6kxMiZSmgazD/CZgmnEeEMJSAOM=", "path": "github.com/hashicorp/hcl2/gohcl", - "revision": "44bad6dbf5490f5da17ec991e664df3d017b706f", - "revisionTime": "2017-10-03T23:27:34Z" + "revision": "5ca9713bf06addcefc0a4e16f779e43a2c88570c", + "revisionTime": "2018-02-05T02:55:09Z" }, { - "checksumSHA1": "TsNlThzf92FMwcnM4Fc0mArHroU=", + "checksumSHA1": "l2zkxDVi2EUwFdvsVcIfyuOr4zo=", "path": "github.com/hashicorp/hcl2/hcl", - "revision": "44bad6dbf5490f5da17ec991e664df3d017b706f", - "revisionTime": "2017-10-03T23:27:34Z" + "revision": "5ca9713bf06addcefc0a4e16f779e43a2c88570c", + "revisionTime": "2018-02-05T02:55:09Z" }, { - "checksumSHA1": "+Dv8V2cfl7Vy6rUklhXj5Cli8aU=", + "checksumSHA1": "iLOUzHOej23ORpmbXAndg5Ft5H0=", "path": "github.com/hashicorp/hcl2/hcl/hclsyntax", - "revision": "44bad6dbf5490f5da17ec991e664df3d017b706f", - "revisionTime": "2017-10-03T23:27:34Z" + "revision": "5ca9713bf06addcefc0a4e16f779e43a2c88570c", + "revisionTime": "2018-02-05T02:55:09Z" }, { - "checksumSHA1": "GAArMzjaoFNPa7HFnhjZmaeBZII=", + "checksumSHA1": "O8jJfHiwuQFmAo0ivcBhni4pWyg=", "path": "github.com/hashicorp/hcl2/hcl/json", - "revision": "44bad6dbf5490f5da17ec991e664df3d017b706f", - "revisionTime": "2017-10-03T23:27:34Z" + "revision": "5ca9713bf06addcefc0a4e16f779e43a2c88570c", + "revisionTime": "2018-02-05T02:55:09Z" }, { - "checksumSHA1": "u6YPoPz3GflgHb1dN1YN8nCWAXY=", + "checksumSHA1": "672O/GQ9z+OFsG3eHLKq1yg3ZGM=", "path": "github.com/hashicorp/hcl2/hcldec", - "revision": "44bad6dbf5490f5da17ec991e664df3d017b706f", - "revisionTime": "2017-10-03T23:27:34Z" + "revision": "5ca9713bf06addcefc0a4e16f779e43a2c88570c", + "revisionTime": "2018-02-05T02:55:09Z" }, { "checksumSHA1": "sySYF9Ew71VS/LfrG+s/0jK+1VQ=", "path": "github.com/hashicorp/hcl2/hcled", - "revision": "44bad6dbf5490f5da17ec991e664df3d017b706f", - "revisionTime": "2017-10-03T23:27:34Z" + "revision": "5ca9713bf06addcefc0a4e16f779e43a2c88570c", + "revisionTime": "2018-02-05T02:55:09Z" }, { "checksumSHA1": "IzmftuG99BqNhbFGhxZaGwtiMtM=", "path": "github.com/hashicorp/hcl2/hclparse", - "revision": "44bad6dbf5490f5da17ec991e664df3d017b706f", - "revisionTime": "2017-10-03T23:27:34Z" + "revision": "5ca9713bf06addcefc0a4e16f779e43a2c88570c", + "revisionTime": "2018-02-05T02:55:09Z" }, { - "checksumSHA1": "4supppf3CMdAEUEDrXP8niXvAR0=", + "checksumSHA1": "v5qx2XghQ+EtvFLa4a0Efjiwt9I=", "path": "github.com/hashicorp/hcl2/hcltest", - "revision": "44bad6dbf5490f5da17ec991e664df3d017b706f", - "revisionTime": "2017-10-03T23:27:34Z" + "revision": "5ca9713bf06addcefc0a4e16f779e43a2c88570c", + "revisionTime": "2018-02-05T02:55:09Z" }, { - "checksumSHA1": "p+dun/Fx4beswXTtoEjVnwDJE+Y=", + "checksumSHA1": "9UCSLRG+TEAsNKOZJUaJj/7d6r8=", "path": "github.com/hashicorp/hcl2/hclwrite", - "revision": "44bad6dbf5490f5da17ec991e664df3d017b706f", - "revisionTime": "2017-10-03T23:27:34Z" + "revision": "5ca9713bf06addcefc0a4e16f779e43a2c88570c", + "revisionTime": "2018-02-05T02:55:09Z" }, { "checksumSHA1": "M09yxoBoCEtG7EcHR8aEWLzMMJc=", From e15ec486bf4701d04dd6eb6ed6cedf9861375b1c Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 2 Feb 2018 17:22:25 -0800 Subject: [PATCH 07/25] configs: Parser.LoadConfigFile This is a first pass of decoding of the main Terraform configuration file format. It hasn't yet been tested with any real-world configurations, so it will need to be revised further as we test it more thoroughly. --- configs/backend.go | 10 + configs/depends_on.go | 37 ++++ configs/module.go | 2 +- configs/module_call.go | 80 +++++++- configs/named_values.go | 181 ++++++++++++++++++ configs/parser_config.go | 235 +++++++++++++++++++++++ configs/provider.go | 67 ++++++- configs/provisioner.go | 185 ++++++++++++++++++ configs/resource.go | 341 ++++++++++++++++++++++++++++++---- configs/util.go | 18 ++ configs/version_constraint.go | 48 ++++- 11 files changed, 1164 insertions(+), 40 deletions(-) create mode 100644 configs/depends_on.go create mode 100644 configs/parser_config.go create mode 100644 configs/provisioner.go create mode 100644 configs/util.go diff --git a/configs/backend.go b/configs/backend.go index 43e872bed..20dc97657 100644 --- a/configs/backend.go +++ b/configs/backend.go @@ -10,5 +10,15 @@ type Backend struct { Type string Config hcl.Body + TypeRange hcl.Range DeclRange hcl.Range } + +func decodeBackendBlock(block *hcl.Block) (*Backend, hcl.Diagnostics) { + return &Backend{ + Type: block.Labels[0], + TypeRange: block.LabelRanges[0], + Config: block.Body, + DeclRange: block.DefRange, + }, nil +} diff --git a/configs/depends_on.go b/configs/depends_on.go new file mode 100644 index 000000000..f20c58167 --- /dev/null +++ b/configs/depends_on.go @@ -0,0 +1,37 @@ +package configs + +import ( + "fmt" + + "github.com/hashicorp/hcl2/hcl" +) + +func decodeDependsOn(attr *hcl.Attribute) ([]hcl.Traversal, hcl.Diagnostics) { + var ret []hcl.Traversal + exprs, diags := hcl.ExprList(attr.Expr) + + for _, expr := range exprs { + // A dependency reference was given as a string literal in the legacy + // configuration language and there are lots of examples out there + // showing that usage, so we'll sniff for that situation here and + // produce a specialized error message for it to help users find + // the new correct form. + if exprIsNativeQuotedString(expr) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid explicit dependency reference", + Detail: fmt.Sprintf("%s elements must not be given in quotes.", attr.Name), + Subject: attr.Expr.Range().Ptr(), + }) + continue + } + + traversal, travDiags := hcl.AbsTraversalForExpr(expr) + diags = append(diags, travDiags...) + if len(traversal) != 0 { + ret = append(ret, traversal) + } + } + + return ret, diags +} diff --git a/configs/module.go b/configs/module.go index 5b9f51041..0a4d5b916 100644 --- a/configs/module.go +++ b/configs/module.go @@ -49,7 +49,7 @@ func NewModule(primaryFiles, overrideFiles []*File) (*Module, hcl.Diagnostics) { // analysis of individual elements, but must be built into a Module to detect // duplicate declarations. type File struct { - CoreVersionConstraints []*VersionConstraint + CoreVersionConstraints []VersionConstraint Backends []*Backend ProviderConfigs []*Provider diff --git a/configs/module_call.go b/configs/module_call.go index ca3205cc4..befccb28a 100644 --- a/configs/module_call.go +++ b/configs/module_call.go @@ -1,18 +1,94 @@ package configs import ( + "github.com/hashicorp/hcl2/gohcl" "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hcl/hclsyntax" ) // ModuleCall represents a "module" block in a module or file. type ModuleCall struct { - Source string - SourceRange hcl.Range + Name string + + SourceAddr string + SourceAddrRange hcl.Range + + Config hcl.Body Version VersionConstraint Count hcl.Expression ForEach hcl.Expression + DependsOn []hcl.Traversal + DeclRange hcl.Range } + +func decodeModuleBlock(block *hcl.Block) (*ModuleCall, hcl.Diagnostics) { + mc := &ModuleCall{ + Name: block.Labels[0], + DeclRange: block.DefRange, + } + + content, remain, diags := block.Body.PartialContent(moduleBlockSchema) + mc.Config = remain + + if !hclsyntax.ValidIdentifier(mc.Name) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid module instance name", + Detail: badIdentifierDetail, + Subject: &block.LabelRanges[0], + }) + } + + if attr, exists := content.Attributes["source"]; exists { + valDiags := gohcl.DecodeExpression(attr.Expr, nil, &mc.SourceAddr) + diags = append(diags, valDiags...) + mc.SourceAddrRange = attr.Expr.Range() + } + + if attr, exists := content.Attributes["version"]; exists { + var versionDiags hcl.Diagnostics + mc.Version, versionDiags = decodeVersionConstraint(attr) + diags = append(diags, versionDiags...) + } + + if attr, exists := content.Attributes["count"]; exists { + mc.Count = attr.Expr + } + + if attr, exists := content.Attributes["for_each"]; exists { + mc.ForEach = attr.Expr + } + + if attr, exists := content.Attributes["depends_on"]; exists { + deps, depsDiags := decodeDependsOn(attr) + diags = append(diags, depsDiags...) + mc.DependsOn = append(mc.DependsOn, deps...) + } + + return mc, diags +} + +var moduleBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "source", + Required: true, + }, + { + Name: "version", + }, + { + Name: "count", + }, + { + Name: "for_each", + }, + { + Name: "depends_on", + }, + }, +} diff --git a/configs/named_values.go b/configs/named_values.go index c2d2c8b83..052b04017 100644 --- a/configs/named_values.go +++ b/configs/named_values.go @@ -1,10 +1,17 @@ package configs import ( + "fmt" + + "github.com/hashicorp/hcl2/gohcl" "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hcl/hclsyntax" "github.com/zclconf/go-cty/cty" ) +// A consistent detail message for all "not a valid identifier" diagnostics. +const badIdentifierDetail = "A name must start with a letter and may contain only letters, digits, underscores, and dashes." + // Variable represents a "variable" block in a module or file. type Variable struct { Name string @@ -15,6 +22,82 @@ type Variable struct { DeclRange hcl.Range } +func decodeVariableBlock(block *hcl.Block) (*Variable, hcl.Diagnostics) { + v := &Variable{ + Name: block.Labels[0], + DeclRange: block.DefRange, + } + + content, diags := block.Body.Content(variableBlockSchema) + + if !hclsyntax.ValidIdentifier(v.Name) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid variable name", + Detail: badIdentifierDetail, + Subject: &block.LabelRanges[0], + }) + } + + // Don't allow declaration of variables that would conflict with the + // reserved attribute and block type names in a "module" block, since + // these won't be usable for child modules. + for _, attr := range moduleBlockSchema.Attributes { + if attr.Name == v.Name { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid variable name", + Detail: fmt.Sprintf("The variable name %q is reserved due to its special meaning inside module blocks.", attr.Name), + Subject: &block.LabelRanges[0], + }) + } + } + + if attr, exists := content.Attributes["description"]; exists { + valDiags := gohcl.DecodeExpression(attr.Expr, nil, &v.Description) + diags = append(diags, valDiags...) + } + + if attr, exists := content.Attributes["default"]; exists { + val, valDiags := attr.Expr.Value(nil) + diags = append(diags, valDiags...) + v.Default = val + } + + if attr, exists := content.Attributes["type"]; exists { + switch hcl.ExprAsKeyword(attr.Expr) { + case "string": + v.TypeHint = TypeHintString + case "list": + v.TypeHint = TypeHintList + case "map": + v.TypeHint = TypeHintMap + default: + // In our legacy configuration format these keywords would've been + // provided as quoted strings, so we'll generate a special error + // message for that to help those who find outdated examples and + // would otherwise be confused. + if exprIsNativeQuotedString(attr.Expr) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid variable type hint", + Detail: "The type hint keyword must not be given in quotes.", + Subject: attr.Expr.Range().Ptr(), + }) + } else { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid variable type hint", + Detail: "The type argument requires one of the following keywords: string, list, or map.", + Subject: attr.Expr.Range().Ptr(), + }) + } + } + } + + return v, diags +} + // Output represents an "output" block in a module or file. type Output struct { Name string @@ -26,6 +109,46 @@ type Output struct { DeclRange hcl.Range } +func decodeOutputBlock(block *hcl.Block) (*Output, hcl.Diagnostics) { + o := &Output{ + Name: block.Labels[0], + DeclRange: block.DefRange, + } + + content, diags := block.Body.Content(outputBlockSchema) + + if !hclsyntax.ValidIdentifier(o.Name) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid output name", + Detail: badIdentifierDetail, + Subject: &block.LabelRanges[0], + }) + } + + if attr, exists := content.Attributes["description"]; exists { + valDiags := gohcl.DecodeExpression(attr.Expr, nil, &o.Description) + diags = append(diags, valDiags...) + } + + if attr, exists := content.Attributes["value"]; exists { + o.Expr = attr.Expr + } + + if attr, exists := content.Attributes["sensitive"]; exists { + valDiags := gohcl.DecodeExpression(attr.Expr, nil, &o.Sensitive) + diags = append(diags, valDiags...) + } + + if attr, exists := content.Attributes["depends_on"]; exists { + deps, depsDiags := decodeDependsOn(attr) + diags = append(diags, depsDiags...) + o.DependsOn = append(o.DependsOn, deps...) + } + + return o, diags +} + // Local represents a single entry from a "locals" block in a module or file. // The "locals" block itself is not represented, because it serves only to // provide context for us to interpret its contents. @@ -35,3 +158,61 @@ type Local struct { DeclRange hcl.Range } + +func decodeLocalsBlock(block *hcl.Block) ([]*Local, hcl.Diagnostics) { + attrs, diags := block.Body.JustAttributes() + if len(attrs) == 0 { + return nil, diags + } + + locals := make([]*Local, 0, len(attrs)) + for name, attr := range attrs { + if !hclsyntax.ValidIdentifier(name) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid local value name", + Detail: badIdentifierDetail, + Subject: &attr.NameRange, + }) + } + + locals = append(locals, &Local{ + Name: name, + Expr: attr.Expr, + DeclRange: attr.Range, + }) + } + return locals, diags +} + +var variableBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "description", + }, + { + Name: "default", + }, + { + Name: "type", + }, + }, +} + +var outputBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "description", + }, + { + Name: "value", + Required: true, + }, + { + Name: "depends_on", + }, + { + Name: "sensitive", + }, + }, +} diff --git a/configs/parser_config.go b/configs/parser_config.go new file mode 100644 index 000000000..16eb77c34 --- /dev/null +++ b/configs/parser_config.go @@ -0,0 +1,235 @@ +package configs + +import ( + "github.com/hashicorp/hcl2/hcl" +) + +// LoadConfigFile reads the file at the given path and parses it as a config +// file. +// +// If the file cannot be read -- for example, if it does not exist -- then +// a nil *File will be returned along with error diagnostics. Callers may wish +// to disregard the returned diagnostics in this case and instead generate +// their own error message(s) with additional context. +// +// If the returned diagnostics has errors when a non-nil map is returned +// then the map may be incomplete but should be valid enough for careful +// static analysis. +// +// This method wraps LoadHCLFile, and so it inherits the syntax selection +// behaviors documented for that method. +func (p *Parser) LoadConfigFile(path string) (*File, hcl.Diagnostics) { + body, diags := p.LoadHCLFile(path) + if body == nil { + return nil, diags + } + + file := &File{} + + var reqDiags hcl.Diagnostics + file.CoreVersionConstraints, reqDiags = sniffCoreVersionRequirements(body) + diags = append(diags, reqDiags...) + + content, contentDiags := body.Content(configFileSchema) + diags = append(diags, contentDiags...) + + for _, block := range content.Blocks { + switch block.Type { + + case "terraform": + content, contentDiags := block.Body.Content(terraformBlockSchema) + diags = append(diags, contentDiags...) + + // We ignore the "terraform_version" attribute here because + // sniffCoreVersionRequirements already dealt with that above. + + for _, innerBlock := range content.Blocks { + switch innerBlock.Type { + + case "backend": + backendCfg, cfgDiags := decodeBackendBlock(innerBlock) + diags = append(diags, cfgDiags...) + if backendCfg != nil { + file.Backends = append(file.Backends, backendCfg) + } + + case "required_providers": + reqs, reqsDiags := decodeRequiredProvidersBlock(innerBlock) + diags = append(diags, reqsDiags...) + file.ProviderRequirements = append(file.ProviderRequirements, reqs...) + + default: + // Should never happen because the above cases should be exhaustive + // for all block type names in our schema. + continue + + } + } + + case "provider": + cfg, cfgDiags := decodeProviderBlock(block) + diags = append(diags, cfgDiags...) + if cfg != nil { + file.ProviderConfigs = append(file.ProviderConfigs, cfg) + } + + case "variable": + cfg, cfgDiags := decodeVariableBlock(block) + diags = append(diags, cfgDiags...) + if cfg != nil { + file.Variables = append(file.Variables, cfg) + } + + case "locals": + defs, defsDiags := decodeLocalsBlock(block) + diags = append(diags, defsDiags...) + file.Locals = append(file.Locals, defs...) + + case "output": + cfg, cfgDiags := decodeOutputBlock(block) + diags = append(diags, cfgDiags...) + if cfg != nil { + file.Outputs = append(file.Outputs, cfg) + } + + case "module": + cfg, cfgDiags := decodeModuleBlock(block) + diags = append(diags, cfgDiags...) + if cfg != nil { + file.ModuleCalls = append(file.ModuleCalls, cfg) + } + + case "resource": + cfg, cfgDiags := decodeResourceBlock(block) + diags = append(diags, cfgDiags...) + if cfg != nil { + file.ManagedResources = append(file.ManagedResources, cfg) + } + + case "data": + cfg, cfgDiags := decodeDataBlock(block) + diags = append(diags, cfgDiags...) + if cfg != nil { + file.DataResources = append(file.DataResources, cfg) + } + + default: + // Should never happen because the above cases should be exhaustive + // for all block type names in our schema. + continue + + } + } + + return file, diags +} + +// sniffCoreVersionRequirements does minimal parsing of the given body for +// "terraform" blocks with "required_version" attributes, returning the +// requirements found. +// +// This is intended to maximize the chance that we'll be able to read the +// requirements (syntax errors notwithstanding) even if the config file contains +// constructs that might've been added in future Terraform versions +// +// This is a "best effort" sort of method which will return constraints it is +// able to find, but may return no constraints at all if the given body is +// so invalid that it cannot be decoded at all. +func sniffCoreVersionRequirements(body hcl.Body) ([]VersionConstraint, hcl.Diagnostics) { + rootContent, _, diags := body.PartialContent(configFileVersionSniffRootSchema) + + var constraints []VersionConstraint + + for _, block := range rootContent.Blocks { + content, _, blockDiags := block.Body.PartialContent(configFileVersionSniffBlockSchema) + diags = append(diags, blockDiags...) + + attr, exists := content.Attributes["required_version"] + if !exists { + continue + } + + constraint, constraintDiags := decodeVersionConstraint(attr) + diags = append(diags, constraintDiags...) + if !constraintDiags.HasErrors() { + constraints = append(constraints, constraint) + } + } + + return constraints, diags +} + +// configFileSchema is the schema for the top-level of a config file. We use +// the low-level HCL API for this level so we can easily deal with each +// block type separately with its own decoding logic. +var configFileSchema = &hcl.BodySchema{ + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "terraform", + }, + { + Type: "provider", + LabelNames: []string{"name"}, + }, + { + Type: "variable", + LabelNames: []string{"name"}, + }, + { + Type: "locals", + }, + { + Type: "output", + LabelNames: []string{"name"}, + }, + { + Type: "module", + LabelNames: []string{"name"}, + }, + { + Type: "resource", + LabelNames: []string{"type", "name"}, + }, + { + Type: "data", + LabelNames: []string{"type", "name"}, + }, + }, +} + +// terraformBlockSchema is the schema for a top-level "terraform" block in +// a configuration file. +var terraformBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "required_version", + }, + }, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "backend", + LabelNames: []string{"type"}, + }, + { + Type: "required_providers", + }, + }, +} + +// configFileVersionSniffRootSchema is a schema for sniffCoreVersionRequirements +var configFileVersionSniffRootSchema = &hcl.BodySchema{ + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "terraform", + }, + }, +} + +// configFileVersionSniffBlockSchema is a schema for sniffCoreVersionRequirements +var configFileVersionSniffBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "required_version", + }, + }, +} diff --git a/configs/provider.go b/configs/provider.go index 8145a87dc..d55874ad3 100644 --- a/configs/provider.go +++ b/configs/provider.go @@ -1,7 +1,11 @@ package configs import ( + "fmt" + + "github.com/hashicorp/hcl2/gohcl" "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hcl/hclsyntax" ) // Provider represents a "provider" block in a module or file. A provider @@ -9,8 +13,9 @@ import ( // configurations for each actual provider. type Provider struct { Name string + NameRange hcl.Range Alias string - AliasRange hcl.Range + AliasRange *hcl.Range // nil if no alias set Version VersionConstraint @@ -19,6 +24,39 @@ type Provider struct { DeclRange hcl.Range } +func decodeProviderBlock(block *hcl.Block) (*Provider, hcl.Diagnostics) { + content, config, diags := block.Body.PartialContent(providerBlockSchema) + + provider := &Provider{ + Name: block.Labels[0], + NameRange: block.LabelRanges[0], + Config: config, + DeclRange: block.DefRange, + } + + if attr, exists := content.Attributes["alias"]; exists { + valDiags := gohcl.DecodeExpression(attr.Expr, nil, &provider.Alias) + diags = append(diags, valDiags...) + provider.AliasRange = attr.Expr.Range().Ptr() + + if !hclsyntax.ValidIdentifier(provider.Alias) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid provider configuration alias", + Detail: fmt.Sprintf("An alias must be a valid name. %s", badIdentifierDetail), + }) + } + } + + if attr, exists := content.Attributes["version"]; exists { + var versionDiags hcl.Diagnostics + provider.Version, versionDiags = decodeVersionConstraint(attr) + diags = append(diags, versionDiags...) + } + + return provider, diags +} + // ProviderRequirement represents a declaration of a dependency on a particular // provider version without actually configuring that provider. This is used in // child modules that expect a provider to be passed in from their parent. @@ -26,3 +64,30 @@ type ProviderRequirement struct { Name string Requirement VersionConstraint } + +func decodeRequiredProvidersBlock(block *hcl.Block) ([]*ProviderRequirement, hcl.Diagnostics) { + attrs, diags := block.Body.JustAttributes() + var reqs []*ProviderRequirement + for name, attr := range attrs { + req, reqDiags := decodeVersionConstraint(attr) + diags = append(diags, reqDiags...) + if !diags.HasErrors() { + reqs = append(reqs, &ProviderRequirement{ + Name: name, + Requirement: req, + }) + } + } + return reqs, diags +} + +var providerBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "alias", + }, + { + Name: "version", + }, + }, +} diff --git a/configs/provisioner.go b/configs/provisioner.go new file mode 100644 index 000000000..03c14e16c --- /dev/null +++ b/configs/provisioner.go @@ -0,0 +1,185 @@ +package configs + +import ( + "fmt" + + "github.com/hashicorp/hcl2/gohcl" + "github.com/hashicorp/hcl2/hcl" +) + +// Provisioner represents a "provisioner" block when used within a +// "resource" block in a module or file. +type Provisioner struct { + Type string + Config hcl.Body + Connection *Connection + When ProvisionerWhen + OnFailure ProvisionerOnFailure + + DeclRange hcl.Range + TypeRange hcl.Range +} + +func decodeProvisionerBlock(block *hcl.Block) (*Provisioner, hcl.Diagnostics) { + pv := &Provisioner{ + Type: block.Labels[0], + TypeRange: block.LabelRanges[0], + DeclRange: block.DefRange, + When: ProvisionerWhenCreate, + OnFailure: ProvisionerOnFailureFail, + } + + content, config, diags := block.Body.PartialContent(provisionerBlockSchema) + pv.Config = config + + if attr, exists := content.Attributes["when"]; exists { + switch hcl.ExprAsKeyword(attr.Expr) { + case "create": + pv.When = ProvisionerWhenCreate + case "destroy": + pv.When = ProvisionerWhenDestroy + default: + if exprIsNativeQuotedString(attr.Expr) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid \"when\" keyword", + Detail: "The \"when\" argument keyword must not be given in quotes.", + Subject: attr.Expr.Range().Ptr(), + }) + } else { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid \"when\" keyword", + Detail: "The \"when\" argument requires one of the following keywords: create or destroy.", + Subject: attr.Expr.Range().Ptr(), + }) + } + } + } + + if attr, exists := content.Attributes["on_failure"]; exists { + switch hcl.ExprAsKeyword(attr.Expr) { + case "continue": + pv.OnFailure = ProvisionerOnFailureContinue + case "fail": + pv.OnFailure = ProvisionerOnFailureFail + default: + if exprIsNativeQuotedString(attr.Expr) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid \"on_failure\" keyword", + Detail: "The \"on_failure\" argument keyword must not be given in quotes.", + Subject: attr.Expr.Range().Ptr(), + }) + } else { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid \"on_failure\" keyword", + Detail: "The \"on_failure\" argument requires one of the following keywords: continue or fail.", + Subject: attr.Expr.Range().Ptr(), + }) + } + } + } + + var seenConnection *hcl.Block + for _, block := range content.Blocks { + switch block.Type { + + case "connection": + if seenConnection != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate connection block", + Detail: fmt.Sprintf("This provisioner already has a connection block at %s.", seenConnection.DefRange), + Subject: &block.DefRange, + }) + continue + } + seenConnection = block + + conn, connDiags := decodeConnectionBlock(block) + diags = append(diags, connDiags...) + pv.Connection = conn + + default: + // Should never happen because there are no other block types + // declared in our schema. + } + } + + return pv, diags +} + +// Connection represents a "connection" block when used within either a +// "resource" or "provisioner" block in a module or file. +type Connection struct { + Type string + Config hcl.Body + + DeclRange hcl.Range + TypeRange *hcl.Range // nil if type is not set +} + +func decodeConnectionBlock(block *hcl.Block) (*Connection, hcl.Diagnostics) { + content, config, diags := block.Body.PartialContent(&hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "type", + }, + }, + }) + + conn := &Connection{ + Type: "ssh", + Config: config, + DeclRange: block.DefRange, + } + + if attr, exists := content.Attributes["type"]; exists { + valDiags := gohcl.DecodeExpression(attr.Expr, nil, &conn.Type) + diags = append(diags, valDiags...) + conn.TypeRange = attr.Expr.Range().Ptr() + } + + return conn, diags +} + +// ProvisionerWhen is an enum for valid values for when to run provisioners. +type ProvisionerWhen int + +//go:generate stringer -type ProvisionerWhen + +const ( + ProvisionerWhenInvalid ProvisionerWhen = iota + ProvisionerWhenCreate + ProvisionerWhenDestroy +) + +// ProvisionerOnFailure is an enum for valid values for on_failure options +// for provisioners. +type ProvisionerOnFailure int + +//go:generate stringer -type ProvisionerOnFailure + +const ( + ProvisionerOnFailureInvalid ProvisionerOnFailure = iota + ProvisionerOnFailureContinue + ProvisionerOnFailureFail +) + +var provisionerBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "when", + }, + { + Name: "on_failure", + }, + }, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "connection", + }, + }, +} diff --git a/configs/resource.go b/configs/resource.go index 459ce0d3d..027d04b2b 100644 --- a/configs/resource.go +++ b/configs/resource.go @@ -1,7 +1,11 @@ package configs import ( + "fmt" + + "github.com/hashicorp/hcl2/gohcl" "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hcl/hclsyntax" ) // ManagedResource represents a "resource" block in a module or file. @@ -12,7 +16,7 @@ type ManagedResource struct { Count hcl.Expression ForEach hcl.Expression - ProviderConfigAddr hcl.Traversal + ProviderConfigRef *ProviderConfigRef DependsOn []hcl.Traversal @@ -24,6 +28,130 @@ type ManagedResource struct { IgnoreChanges []hcl.Traversal DeclRange hcl.Range + TypeRange hcl.Range +} + +func decodeResourceBlock(block *hcl.Block) (*ManagedResource, hcl.Diagnostics) { + r := &ManagedResource{ + Type: block.Labels[0], + Name: block.Labels[1], + DeclRange: block.DefRange, + TypeRange: block.LabelRanges[0], + } + + content, remain, diags := block.Body.PartialContent(resourceBlockSchema) + r.Config = remain + + if !hclsyntax.ValidIdentifier(r.Type) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid resource type name", + Detail: badIdentifierDetail, + Subject: &block.LabelRanges[0], + }) + } + if !hclsyntax.ValidIdentifier(r.Name) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid resource name", + Detail: badIdentifierDetail, + Subject: &block.LabelRanges[0], + }) + } + + if attr, exists := content.Attributes["count"]; exists { + r.Count = attr.Expr + } + + if attr, exists := content.Attributes["for_each"]; exists { + r.Count = attr.Expr + } + + if attr, exists := content.Attributes["provider"]; exists { + var providerDiags hcl.Diagnostics + r.ProviderConfigRef, providerDiags = decodeProviderConfigRef(attr) + diags = append(diags, providerDiags...) + } + + if attr, exists := content.Attributes["depends_on"]; exists { + deps, depsDiags := decodeDependsOn(attr) + diags = append(diags, depsDiags...) + r.DependsOn = append(r.DependsOn, deps...) + } + + var seenLifecycle *hcl.Block + var seenConnection *hcl.Block + for _, block := range content.Blocks { + switch block.Type { + case "lifecycle": + if seenLifecycle != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate lifecycle block", + Detail: fmt.Sprintf("This resource already has a lifecycle block at %s.", seenLifecycle.DefRange), + Subject: &block.DefRange, + }) + continue + } + seenLifecycle = block + + lcContent, lcDiags := block.Body.Content(resourceLifecycleBlockSchema) + diags = append(diags, lcDiags...) + + if attr, exists := lcContent.Attributes["create_before_destroy"]; exists { + valDiags := gohcl.DecodeExpression(attr.Expr, nil, &r.CreateBeforeDestroy) + diags = append(diags, valDiags...) + } + + if attr, exists := lcContent.Attributes["prevent_destroy"]; exists { + valDiags := gohcl.DecodeExpression(attr.Expr, nil, &r.PreventDestroy) + diags = append(diags, valDiags...) + } + + if attr, exists := lcContent.Attributes["ignore_changes"]; exists { + exprs, listDiags := hcl.ExprList(attr.Expr) + diags = append(diags, listDiags...) + + for _, expr := range exprs { + traversal, travDiags := hcl.RelTraversalForExpr(expr) + diags = append(diags, travDiags...) + if len(traversal) != 0 { + r.IgnoreChanges = append(r.IgnoreChanges, traversal) + } + } + } + + case "connection": + if seenConnection != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate connection block", + Detail: fmt.Sprintf("This resource already has a connection block at %s.", seenConnection.DefRange), + Subject: &block.DefRange, + }) + continue + } + seenConnection = block + + conn, connDiags := decodeConnectionBlock(block) + diags = append(diags, connDiags...) + r.Connection = conn + + case "provisioner": + pv, pvDiags := decodeProvisionerBlock(block) + diags = append(diags, pvDiags...) + if pv != nil { + r.Provisioners = append(r.Provisioners, pv) + } + + default: + // Should never happen, because the above cases should always be + // exhaustive for all the types specified in our schema. + continue + } + } + + return r, diags } // DataResource represents a "data" block in a module or file. @@ -34,53 +162,196 @@ type DataResource struct { Count hcl.Expression ForEach hcl.Expression - ProviderConfigAddr hcl.Traversal + ProviderConfigRef *ProviderConfigRef DependsOn []hcl.Traversal DeclRange hcl.Range + TypeRange hcl.Range } -// Provisioner represents a "provisioner" block when used within a -// "resource" block in a module or file. -type Provisioner struct { - Type string - Config hcl.Body - Connection *Connection - When ProvisionerWhen - OnFailure ProvisionerOnFailure +func decodeDataBlock(block *hcl.Block) (*DataResource, hcl.Diagnostics) { + r := &DataResource{ + Type: block.Labels[0], + Name: block.Labels[1], + DeclRange: block.DefRange, + TypeRange: block.LabelRanges[0], + } - DeclRange hcl.Range + content, remain, diags := block.Body.PartialContent(dataBlockSchema) + r.Config = remain + + if !hclsyntax.ValidIdentifier(r.Type) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid data source name", + Detail: badIdentifierDetail, + Subject: &block.LabelRanges[0], + }) + } + if !hclsyntax.ValidIdentifier(r.Name) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid data resource name", + Detail: badIdentifierDetail, + Subject: &block.LabelRanges[0], + }) + } + + if attr, exists := content.Attributes["count"]; exists { + r.Count = attr.Expr + } + + if attr, exists := content.Attributes["for_each"]; exists { + r.Count = attr.Expr + } + + if attr, exists := content.Attributes["provider"]; exists { + var providerDiags hcl.Diagnostics + r.ProviderConfigRef, providerDiags = decodeProviderConfigRef(attr) + diags = append(diags, providerDiags...) + } + + if attr, exists := content.Attributes["depends_on"]; exists { + deps, depsDiags := decodeDependsOn(attr) + diags = append(diags, depsDiags...) + r.DependsOn = append(r.DependsOn, deps...) + } + + for _, block := range content.Blocks { + // Our schema only allows for "lifecycle" blocks, so we can assume + // that this is all we will see here. We don't have any lifecycle + // attributes for data resources currently, so we'll just produce + // an error. + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unsupported lifecycle block", + Detail: "Data resources do not have lifecycle settings, so a lifecycle block is not allowed.", + Subject: &block.DefRange, + }) + break + } + + return r, diags } -// Connection represents a "connection" block when used within either a -// "resource" or "provisioner" block in a module or file. -type Connection struct { - Type string - Config hcl.Body - - DeclRange hcl.Range +type ProviderConfigRef struct { + Name string + NameRange hcl.Range + Alias string + AliasRange *hcl.Range // nil if alias not set } -// ProvisionerWhen is an enum for valid values for when to run provisioners. -type ProvisionerWhen int +func decodeProviderConfigRef(attr *hcl.Attribute) (*ProviderConfigRef, hcl.Diagnostics) { + var diags hcl.Diagnostics + traversal, travDiags := hcl.AbsTraversalForExpr(attr.Expr) -//go:generate stringer -type ProvisionerWhen + // AbsTraversalForExpr produces only generic errors, so we'll discard + // the errors given and produce our own with extra context. If we didn't + // get any errors then we might still have warnings, though. + if !travDiags.HasErrors() { + diags = append(diags, travDiags...) + } -const ( - ProvisionerWhenInvalid ProvisionerWhen = iota - ProvisionerWhenCreate - ProvisionerWhenDestroy -) + if len(traversal) < 1 && len(traversal) > 2 { + // A provider reference was given as a string literal in the legacy + // configuration language and there are lots of examples out there + // showing that usage, so we'll sniff for that situation here and + // produce a specialized error message for it to help users find + // the new correct form. + if exprIsNativeQuotedString(attr.Expr) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid provider configuration reference", + Detail: "A provider configuration reference must not be given in quotes.", + Subject: attr.Expr.Range().Ptr(), + }) + return nil, diags + } -// ProvisionerOnFailure is an enum for valid values for on_failure options -// for provisioners. -type ProvisionerOnFailure int + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid provider configuration reference", + Detail: fmt.Sprintf("The %s argument requires a provider type name, optionally followed by a period and then a configuration alias.", attr.Name), + Subject: attr.Expr.Range().Ptr(), + }) + return nil, diags + } -//go:generate stringer -type ProvisionerOnFailure + ret := &ProviderConfigRef{ + Name: traversal.RootName(), + NameRange: traversal[0].SourceRange(), + } -const ( - ProvisionerOnFailureInvalid ProvisionerOnFailure = iota - ProvisionerOnFailureContinue - ProvisionerOnFailureFail -) + if len(traversal) > 1 { + aliasStep, ok := traversal[1].(hcl.TraverseAttr) + if !ok { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid provider configuration reference", + Detail: "Provider name must either stand alone or be followed by a period and then a configuration alias.", + Subject: traversal[1].SourceRange().Ptr(), + }) + return ret, diags + } + + ret.Alias = aliasStep.Name + ret.AliasRange = aliasStep.SourceRange().Ptr() + } + + return ret, diags +} + +var commonResourceAttributes = []hcl.AttributeSchema{ + { + Name: "count", + }, + { + Name: "for_each", + }, + { + Name: "provider", + }, + { + Name: "depends_on", + }, +} + +var resourceBlockSchema = &hcl.BodySchema{ + Attributes: commonResourceAttributes, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "lifecycle", + }, + { + Type: "connection", + }, + { + Type: "provisioner", + LabelNames: []string{"type"}, + }, + }, +} + +var dataBlockSchema = &hcl.BodySchema{ + Attributes: commonResourceAttributes, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "lifecycle", + }, + }, +} + +var resourceLifecycleBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "create_before_destroy", + }, + { + Name: "prevent_destroy", + }, + { + Name: "ignore_changes", + }, + }, +} diff --git a/configs/util.go b/configs/util.go new file mode 100644 index 000000000..9594cbeee --- /dev/null +++ b/configs/util.go @@ -0,0 +1,18 @@ +package configs + +import ( + "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hcl/hclsyntax" +) + +// exprIsNativeQuotedString determines whether the given expression looks like +// it's a quoted string in the HCL native syntax. +// +// This should be used sparingly only for situations where our legacy HCL +// decoding would've expected a keyword or reference in quotes but our new +// decoding expects the keyword or reference to be provided directly as +// an identifier-based expression. +func exprIsNativeQuotedString(expr hcl.Expression) bool { + _, ok := expr.(*hclsyntax.TemplateExpr) + return ok +} diff --git a/configs/version_constraint.go b/configs/version_constraint.go index 6b2882cee..00dc739a7 100644 --- a/configs/version_constraint.go +++ b/configs/version_constraint.go @@ -1,8 +1,12 @@ package configs import ( + "fmt" + version "github.com/hashicorp/go-version" "github.com/hashicorp/hcl2/hcl" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" ) // VersionConstraint represents a version constraint on some resource @@ -10,6 +14,48 @@ import ( // a source range so that a helpful diagnostic can be printed in the event // that a particular constraint does not match. type VersionConstraint struct { - Required []version.Constraints + Required version.Constraints DeclRange hcl.Range } + +func decodeVersionConstraint(attr *hcl.Attribute) (VersionConstraint, hcl.Diagnostics) { + ret := VersionConstraint{ + DeclRange: attr.Range, + } + + val, diags := attr.Expr.Value(nil) + var err error + val, err = convert.Convert(val, cty.String) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid version constraint", + Detail: fmt.Sprintf("A string value is required for %s.", attr.Name), + Subject: attr.Expr.Range().Ptr(), + }) + return ret, diags + } + + if val.IsNull() { + // A null version constraint is strange, but we'll just treat it + // like an empty constraint set. + return ret, diags + } + + constraintStr := val.AsString() + constraints, err := version.NewConstraint(constraintStr) + if err != nil { + // NewConstraint doesn't return user-friendly errors, so we'll just + // ignore the provided error and produce our own generic one. + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid version constraint", + Detail: "This string does not use correct version constraint syntax.", // Not very actionable :( + Subject: attr.Expr.Range().Ptr(), + }) + return ret, diags + } + + ret.Required = constraints + return ret, diags +} From e524d1eb95c22e5b37dade909935f550db8b2190 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 2 Feb 2018 17:24:28 -0800 Subject: [PATCH 08/25] configs: Simple test of loading valid configuration files This test is intended to be an easy-to-maintain catalog of good examples that we can use to catch certain parsing or decoding regressions easily. It's not a fully-comprehensive test since it doesn't check the result of decoding, instead just accepting any decode that completes without errors. However, an easy-to-maintain test like this is a good complement to some more specialized tests since we can easily collect good examples over time and just add them in here. --- configs/parser_config_test.go | 43 +++++++++++++++++++ configs/test-fixtures/valid-files/backend.tf | 10 +++++ .../test-fixtures/valid-files/data-sources.tf | 15 +++++++ configs/test-fixtures/valid-files/empty.tf | 0 .../test-fixtures/valid-files/empty.tf.json | 1 + configs/test-fixtures/valid-files/locals.tf | 16 +++++++ .../test-fixtures/valid-files/locals.tf.json | 10 +++++ .../test-fixtures/valid-files/module-calls.tf | 26 +++++++++++ configs/test-fixtures/valid-files/outputs.tf | 25 +++++++++++ .../valid-files/provider-configs.tf | 15 +++++++ .../valid-files/required-providers.tf | 7 +++ .../valid-files/required-version.tf | 4 ++ .../test-fixtures/valid-files/resources.tf | 42 ++++++++++++++++++ .../test-fixtures/valid-files/variables.tf | 24 +++++++++++ .../valid-files/variables.tf.json | 21 +++++++++ 15 files changed, 259 insertions(+) create mode 100644 configs/parser_config_test.go create mode 100644 configs/test-fixtures/valid-files/backend.tf create mode 100644 configs/test-fixtures/valid-files/data-sources.tf create mode 100644 configs/test-fixtures/valid-files/empty.tf create mode 100644 configs/test-fixtures/valid-files/empty.tf.json create mode 100644 configs/test-fixtures/valid-files/locals.tf create mode 100644 configs/test-fixtures/valid-files/locals.tf.json create mode 100644 configs/test-fixtures/valid-files/module-calls.tf create mode 100644 configs/test-fixtures/valid-files/outputs.tf create mode 100644 configs/test-fixtures/valid-files/provider-configs.tf create mode 100644 configs/test-fixtures/valid-files/required-providers.tf create mode 100644 configs/test-fixtures/valid-files/required-version.tf create mode 100644 configs/test-fixtures/valid-files/resources.tf create mode 100644 configs/test-fixtures/valid-files/variables.tf create mode 100644 configs/test-fixtures/valid-files/variables.tf.json diff --git a/configs/parser_config_test.go b/configs/parser_config_test.go new file mode 100644 index 000000000..affc94626 --- /dev/null +++ b/configs/parser_config_test.go @@ -0,0 +1,43 @@ +package configs + +import ( + "io/ioutil" + "path/filepath" + "testing" +) + +// TestParseLoadConfigFileSuccess is a simple test that just verifies that +// a number of test configuration files (in test-fixtures/valid-files) can +// be parsed without raising any diagnostics. +// +// This test does not verify that reading these files produces the correct +// file element contents. More detailed assertions may be made on some subset +// of these configuration files in other tests. +func TestParserLoadConfigFileSuccess(t *testing.T) { + files, err := ioutil.ReadDir("test-fixtures/valid-files") + if err != nil { + t.Fatal(err) + } + + for _, info := range files { + name := info.Name() + t.Run(name, func(t *testing.T) { + src, err := ioutil.ReadFile(filepath.Join("test-fixtures/valid-files", name)) + if err != nil { + t.Fatal(err) + } + + parser := testParser(map[string]string{ + name: string(src), + }) + + _, diags := parser.LoadConfigFile(name) + if len(diags) != 0 { + t.Errorf("unexpected diagnostics") + for _, diag := range diags { + t.Logf("- %s", diag) + } + } + }) + } +} diff --git a/configs/test-fixtures/valid-files/backend.tf b/configs/test-fixtures/valid-files/backend.tf new file mode 100644 index 000000000..bd8e0f669 --- /dev/null +++ b/configs/test-fixtures/valid-files/backend.tf @@ -0,0 +1,10 @@ + +terraform { + backend "example" { + foo = "bar" + + baz { + bar = "foo" + } + } +} diff --git a/configs/test-fixtures/valid-files/data-sources.tf b/configs/test-fixtures/valid-files/data-sources.tf new file mode 100644 index 000000000..f14dffdac --- /dev/null +++ b/configs/test-fixtures/valid-files/data-sources.tf @@ -0,0 +1,15 @@ +data "http" "example1" { +} + +data "http" "example2" { + url = "http://example.com/" + + request_headers = { + "Accept" = "application/json" + } + + count = 5 + depends_on = [ + data.http.example1, + ] +} diff --git a/configs/test-fixtures/valid-files/empty.tf b/configs/test-fixtures/valid-files/empty.tf new file mode 100644 index 000000000..e69de29bb diff --git a/configs/test-fixtures/valid-files/empty.tf.json b/configs/test-fixtures/valid-files/empty.tf.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/configs/test-fixtures/valid-files/empty.tf.json @@ -0,0 +1 @@ +{} diff --git a/configs/test-fixtures/valid-files/locals.tf b/configs/test-fixtures/valid-files/locals.tf new file mode 100644 index 000000000..f2ee4a7c4 --- /dev/null +++ b/configs/test-fixtures/valid-files/locals.tf @@ -0,0 +1,16 @@ + +locals { + # This block intentionally left blank +} + +locals { + foo = "foo" + bar = true +} + +locals { + baz = "oink" + dunno = "🤷" + rowing = "🚣‍♀️" + π = 3.14159265359 +} diff --git a/configs/test-fixtures/valid-files/locals.tf.json b/configs/test-fixtures/valid-files/locals.tf.json new file mode 100644 index 000000000..525f69d6f --- /dev/null +++ b/configs/test-fixtures/valid-files/locals.tf.json @@ -0,0 +1,10 @@ +{ + "locals": { + "foo": "foo", + "bar": true, + "baz": "oink", + "dunno": "🤷", + "rowing": "🚣‍♀️", + "π": 3.14159265359 + } +} diff --git a/configs/test-fixtures/valid-files/module-calls.tf b/configs/test-fixtures/valid-files/module-calls.tf new file mode 100644 index 000000000..abd423586 --- /dev/null +++ b/configs/test-fixtures/valid-files/module-calls.tf @@ -0,0 +1,26 @@ + +module "foo" { + source = "./foo" + # this block intentionally left (almost) blank +} + +module "bar" { + source = "hashicorp/bar/aws" + + boom = "🎆" + yes = true +} + +module "baz" { + source = "git::https://example.com/" + + a = 1 + + count = 12 + for_each = ["a", "b", "c"] + + depends_on = [ + module.bar, + ] +} + diff --git a/configs/test-fixtures/valid-files/outputs.tf b/configs/test-fixtures/valid-files/outputs.tf new file mode 100644 index 000000000..7a8066686 --- /dev/null +++ b/configs/test-fixtures/valid-files/outputs.tf @@ -0,0 +1,25 @@ + +output "foo" { + value = "hello" +} + +output "bar" { + value = local.bar +} + +output "baz" { + value = "ssshhhhhhh" + sensitive = true +} + +output "cheeze_pizza" { + description = "Nothing special" + value = "🍕" +} + +output "π" { + value = 3.14159265359 + depends_on = [ + pizza.cheese, + ] +} diff --git a/configs/test-fixtures/valid-files/provider-configs.tf b/configs/test-fixtures/valid-files/provider-configs.tf new file mode 100644 index 000000000..a07f08be8 --- /dev/null +++ b/configs/test-fixtures/valid-files/provider-configs.tf @@ -0,0 +1,15 @@ + +provider "foo" { +} + +provider "bar" { + version = ">= 1.0.2" + + other = 12 +} + +provider "bar" { + other = 13 + + alias = "bar" +} diff --git a/configs/test-fixtures/valid-files/required-providers.tf b/configs/test-fixtures/valid-files/required-providers.tf new file mode 100644 index 000000000..271df4a57 --- /dev/null +++ b/configs/test-fixtures/valid-files/required-providers.tf @@ -0,0 +1,7 @@ + +terraform { + required_providers { + aws = "~> 1.0.0" + consul = "~> 1.2.0" + } +} diff --git a/configs/test-fixtures/valid-files/required-version.tf b/configs/test-fixtures/valid-files/required-version.tf new file mode 100644 index 000000000..77c9a3570 --- /dev/null +++ b/configs/test-fixtures/valid-files/required-version.tf @@ -0,0 +1,4 @@ + +terraform { + required_version = "~> 0.12.0" +} diff --git a/configs/test-fixtures/valid-files/resources.tf b/configs/test-fixtures/valid-files/resources.tf new file mode 100644 index 000000000..53fb74533 --- /dev/null +++ b/configs/test-fixtures/valid-files/resources.tf @@ -0,0 +1,42 @@ +resource "aws_security_group" "firewall" { + lifecycle { + create_before_destroy = true + prevent_destroy = true + ignore_changes = [ + description, + ] + } + + connection { + host = "127.0.0.1" + } + + provisioner "local-exec" { + command = "echo hello" + + connection { + host = "10.1.2.1" + } + } + + provisioner "local-exec" { + command = "echo hello" + } +} + +resource "aws_instance" "web" { + ami = "ami-1234" + security_groups = [ + "foo", + "bar", + ] + + network_interface { + device_index = 0 + description = "Main network interface" + } + + depends_on = [ + aws_security_group.firewall, + ] +} diff --git a/configs/test-fixtures/valid-files/variables.tf b/configs/test-fixtures/valid-files/variables.tf new file mode 100644 index 000000000..185d9de55 --- /dev/null +++ b/configs/test-fixtures/valid-files/variables.tf @@ -0,0 +1,24 @@ + +variable "foo" { +} + +variable "bar" { + default = "hello" +} + +variable "baz" { + type = list +} + +variable "bar-baz" { + default = [] + type = list +} + +variable "cheeze_pizza" { + description = "Nothing special" +} + +variable "π" { + default = 3.14159265359 +} diff --git a/configs/test-fixtures/valid-files/variables.tf.json b/configs/test-fixtures/valid-files/variables.tf.json new file mode 100644 index 000000000..3c97d6178 --- /dev/null +++ b/configs/test-fixtures/valid-files/variables.tf.json @@ -0,0 +1,21 @@ +{ + "variable": { + "foo": {}, + "bar": { + "default": "hello" + }, + "baz": { + "type": "list" + }, + "bar-baz": { + "default": [], + "type": "list" + }, + "cheese_pizza": { + "description": "Nothing special" + }, + "π": { + "default": 3.14159265359 + } + } +} From 9be399d49c2521ccb3b7dafb8d5eab227b6910fa Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 2 Feb 2018 17:40:28 -0800 Subject: [PATCH 09/25] configs: Another simple test for _invalid_ config files Much like TestParserLoadConfigFileSuccess, this is intended to be an easy-to-maintain collection of bad examples to test different permutations of our error handling. As with TestParserLoadConfigFileSuccess, we should also have more specific tests alongside this that check that the error outcome is what was expected, since this test just accepts any error and may thus not be testing what we think it is. --- configs/parser_config_test.go | 101 ++++++++++++++++++ .../invalid-files/data-resource-lifecycle.tf | 5 + .../resource-lifecycle-badbool.tf | 5 + .../invalid-files/unexpected-attr.tf | 1 + .../invalid-files/unexpected-block.tf | 2 + .../invalid-files/variable-type-quoted.tf | 3 + .../invalid-files/variable-type-unknown.tf | 3 + .../invalid-files/zerolen.tf.json | 0 8 files changed, 120 insertions(+) create mode 100644 configs/test-fixtures/invalid-files/data-resource-lifecycle.tf create mode 100644 configs/test-fixtures/invalid-files/resource-lifecycle-badbool.tf create mode 100644 configs/test-fixtures/invalid-files/unexpected-attr.tf create mode 100644 configs/test-fixtures/invalid-files/unexpected-block.tf create mode 100644 configs/test-fixtures/invalid-files/variable-type-quoted.tf create mode 100644 configs/test-fixtures/invalid-files/variable-type-unknown.tf create mode 100644 configs/test-fixtures/invalid-files/zerolen.tf.json diff --git a/configs/parser_config_test.go b/configs/parser_config_test.go index affc94626..2f118933c 100644 --- a/configs/parser_config_test.go +++ b/configs/parser_config_test.go @@ -4,6 +4,8 @@ import ( "io/ioutil" "path/filepath" "testing" + + "github.com/hashicorp/hcl2/hcl" ) // TestParseLoadConfigFileSuccess is a simple test that just verifies that @@ -41,3 +43,102 @@ func TestParserLoadConfigFileSuccess(t *testing.T) { }) } } + +// TestParseLoadConfigFileFailure is a simple test that just verifies that +// a number of test configuration files (in test-fixtures/invalid-files) +// produce errors as expected. +// +// This test does not verify specific error messages, so more detailed +// assertions should be made on some subset of these configuration files in +// other tests. +func TestParserLoadConfigFileFailure(t *testing.T) { + files, err := ioutil.ReadDir("test-fixtures/invalid-files") + if err != nil { + t.Fatal(err) + } + + for _, info := range files { + name := info.Name() + t.Run(name, func(t *testing.T) { + src, err := ioutil.ReadFile(filepath.Join("test-fixtures/invalid-files", name)) + if err != nil { + t.Fatal(err) + } + + parser := testParser(map[string]string{ + name: string(src), + }) + + _, diags := parser.LoadConfigFile(name) + if !diags.HasErrors() { + t.Errorf("LoadConfigFile succeeded; want errors") + } + for _, diag := range diags { + t.Logf("- %s", diag) + } + }) + } +} + +// This test uses a subset of the same fixture files as +// TestParserLoadConfigFileFailure, but additionally verifies that each +// file produces the expected diagnostic summary. +func TestParserLoadConfigFileFailureMessages(t *testing.T) { + tests := []struct { + Filename string + WantError string + }{ + { + "data-resource-lifecycle.tf", + "Unsupported lifecycle block", + }, + { + "variable-type-unknown.tf", + "Invalid variable type hint", + }, + { + "variable-type-quoted.tf", + "Invalid variable type hint", + }, + { + "unexpected-attr.tf", + "Unsupported attribute", + }, + { + "unexpected-block.tf", + "Unsupported block type", + }, + { + "resource-lifecycle-badbool.tf", + "Unsuitable value type", + }, + } + + for _, test := range tests { + t.Run(test.Filename, func(t *testing.T) { + src, err := ioutil.ReadFile(filepath.Join("test-fixtures/invalid-files", test.Filename)) + if err != nil { + t.Fatal(err) + } + + parser := testParser(map[string]string{ + test.Filename: string(src), + }) + + _, diags := parser.LoadConfigFile(test.Filename) + if len(diags) != 1 { + t.Errorf("Wrong number of diagnostics %d; want 1", len(diags)) + for _, diag := range diags { + t.Logf("- %s", diag) + } + return + } + if diags[0].Severity != hcl.DiagError { + t.Errorf("Wrong diagnostic severity %s; want %s", diags[0].Severity, hcl.DiagError) + } + if diags[0].Summary != test.WantError { + t.Errorf("Wrong diagnostic summary\ngot: %s\nwant: %s", diags[0].Summary, test.WantError) + } + }) + } +} diff --git a/configs/test-fixtures/invalid-files/data-resource-lifecycle.tf b/configs/test-fixtures/invalid-files/data-resource-lifecycle.tf new file mode 100644 index 000000000..b0c34d463 --- /dev/null +++ b/configs/test-fixtures/invalid-files/data-resource-lifecycle.tf @@ -0,0 +1,5 @@ +data "example" "example" { + lifecycle { + # This block intentionally left blank + } +} diff --git a/configs/test-fixtures/invalid-files/resource-lifecycle-badbool.tf b/configs/test-fixtures/invalid-files/resource-lifecycle-badbool.tf new file mode 100644 index 000000000..fdddb6944 --- /dev/null +++ b/configs/test-fixtures/invalid-files/resource-lifecycle-badbool.tf @@ -0,0 +1,5 @@ +resource "example" "example" { + lifecycle { + create_before_destroy = "ABSOLUTELY NOT" + } +} diff --git a/configs/test-fixtures/invalid-files/unexpected-attr.tf b/configs/test-fixtures/invalid-files/unexpected-attr.tf new file mode 100644 index 000000000..5abc475eb --- /dev/null +++ b/configs/test-fixtures/invalid-files/unexpected-attr.tf @@ -0,0 +1 @@ +foo = "bar" diff --git a/configs/test-fixtures/invalid-files/unexpected-block.tf b/configs/test-fixtures/invalid-files/unexpected-block.tf new file mode 100644 index 000000000..491173c38 --- /dev/null +++ b/configs/test-fixtures/invalid-files/unexpected-block.tf @@ -0,0 +1,2 @@ +varyable "whoops" { +} diff --git a/configs/test-fixtures/invalid-files/variable-type-quoted.tf b/configs/test-fixtures/invalid-files/variable-type-quoted.tf new file mode 100644 index 000000000..15db803f2 --- /dev/null +++ b/configs/test-fixtures/invalid-files/variable-type-quoted.tf @@ -0,0 +1,3 @@ +variable "bad_type" { + type = "string" +} diff --git a/configs/test-fixtures/invalid-files/variable-type-unknown.tf b/configs/test-fixtures/invalid-files/variable-type-unknown.tf new file mode 100644 index 000000000..bcbb88d90 --- /dev/null +++ b/configs/test-fixtures/invalid-files/variable-type-unknown.tf @@ -0,0 +1,3 @@ +variable "bad_type" { + type = notatype +} diff --git a/configs/test-fixtures/invalid-files/zerolen.tf.json b/configs/test-fixtures/invalid-files/zerolen.tf.json new file mode 100644 index 000000000..e69de29bb From 4e5efa498a8648220e160c9c4559f0a2723188a4 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 6 Feb 2018 16:28:40 -0800 Subject: [PATCH 10/25] configs: Parser.LoadConfigDir This method wraps LoadConfigFile to load all of the .tf and .tf.json files in a given directory and then bundle them together into a Module object. This function also deals with the distinction between primary and override files, first appending together the primary files in lexicographic order by filename, and then merging in override files in the same order. The merging behavior is not fully implemented as of this commit, and so will be expanded in future commits. --- configs/module.go | 338 +++++++++++++++++- configs/module_merge.go | 212 +++++++++++ configs/module_merge_body.go | 47 +++ configs/parser_config_dir.go | 125 +++++++ configs/parser_config_dir_test.go | 136 +++++++ configs/provider.go | 7 + configs/resource.go | 8 + .../invalid-files/json-as-native-syntax.tf | 3 + .../native-syntax-as-json.tf.json | 2 + .../override-nonexist-variable/override.tf | 3 + .../test-fixtures/valid-modules/empty/README | 2 + .../override-variable/a_override.tf | 9 + .../override-variable/b_override.tf | 9 + .../override-variable/primary.tf | 11 + 14 files changed, 898 insertions(+), 14 deletions(-) create mode 100644 configs/module_merge.go create mode 100644 configs/module_merge_body.go create mode 100644 configs/parser_config_dir.go create mode 100644 configs/parser_config_dir_test.go create mode 100644 configs/test-fixtures/invalid-files/json-as-native-syntax.tf create mode 100644 configs/test-fixtures/invalid-files/native-syntax-as-json.tf.json create mode 100644 configs/test-fixtures/invalid-modules/override-nonexist-variable/override.tf create mode 100644 configs/test-fixtures/valid-modules/empty/README create mode 100644 configs/test-fixtures/valid-modules/override-variable/a_override.tf create mode 100644 configs/test-fixtures/valid-modules/override-variable/b_override.tf create mode 100644 configs/test-fixtures/valid-modules/override-variable/primary.tf diff --git a/configs/module.go b/configs/module.go index 0a4d5b916..dec21d045 100644 --- a/configs/module.go +++ b/configs/module.go @@ -1,6 +1,8 @@ package configs import ( + "fmt" + "github.com/hashicorp/hcl2/hcl" ) @@ -23,20 +25,6 @@ type Module struct { DataResources map[string]*DataResource } -// NewModule takes a list of primary files and a list of override files and -// produces a *Module by combining the files together. -// -// If there are any conflicting declarations in the given files -- for example, -// if the same variable name is defined twice -- then the resulting module -// will be incomplete and error diagnostics will be returned. Careful static -// analysis of the returned Module is still possible in this case, but the -// module will probably not be semantically valid. -func NewModule(primaryFiles, overrideFiles []*File) (*Module, hcl.Diagnostics) { - // TODO: process each file in turn, combining and merging as necessary - // to produce a single flat *Module. - panic("NewModule not yet implemented") -} - // File describes the contents of a single configuration file. // // Individual files are not usually used alone, but rather combined together @@ -64,3 +52,325 @@ type File struct { ManagedResources []*ManagedResource DataResources []*DataResource } + +// NewModule takes a list of primary files and a list of override files and +// produces a *Module by combining the files together. +// +// If there are any conflicting declarations in the given files -- for example, +// if the same variable name is defined twice -- then the resulting module +// will be incomplete and error diagnostics will be returned. Careful static +// analysis of the returned Module is still possible in this case, but the +// module will probably not be semantically valid. +func NewModule(primaryFiles, overrideFiles []*File) (*Module, hcl.Diagnostics) { + var diags hcl.Diagnostics + mod := &Module{ + ProviderConfigs: map[string]*Provider{}, + ProviderRequirements: map[string][]VersionConstraint{}, + Variables: map[string]*Variable{}, + Locals: map[string]*Local{}, + Outputs: map[string]*Output{}, + ModuleCalls: map[string]*ModuleCall{}, + ManagedResources: map[string]*ManagedResource{}, + DataResources: map[string]*DataResource{}, + } + + for _, file := range primaryFiles { + fileDiags := mod.appendFile(file) + diags = append(diags, fileDiags...) + } + + for _, file := range overrideFiles { + fileDiags := mod.mergeFile(file) + diags = append(diags, fileDiags...) + } + + return mod, diags +} + +func (m *Module) appendFile(file *File) hcl.Diagnostics { + var diags hcl.Diagnostics + + for _, constraint := range file.CoreVersionConstraints { + // If there are any conflicting requirements then we'll catch them + // when we actually check these constraints. + m.CoreVersionConstraints = append(m.CoreVersionConstraints, constraint) + } + + for _, b := range file.Backends { + if m.Backend != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate backend configuration", + Detail: fmt.Sprintf("A module may have only one backend configuration. The backend was previously configured at %s.", m.Backend.DeclRange), + Subject: &b.DeclRange, + }) + continue + } + m.Backend = b + } + + for _, pc := range file.ProviderConfigs { + key := pc.moduleUniqueKey() + if existing, exists := m.ProviderConfigs[key]; exists { + if existing.Alias == "" { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate provider configuration", + Detail: fmt.Sprintf("A default (non-aliased) provider configuration for %q was already given at %s. If multiple configurations are required, set the \"alias\" argument for alternative configurations.", existing.Name, existing.DeclRange), + Subject: &pc.DeclRange, + }) + } else { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate provider configuration", + Detail: fmt.Sprintf("A provider configuration for %q with alias %q was already given at %s. Each configuration for the same provider must have a distinct alias.", existing.Name, existing.Alias, existing.DeclRange), + Subject: &pc.DeclRange, + }) + } + continue + } + m.ProviderConfigs[key] = pc + } + + for _, reqd := range file.ProviderRequirements { + m.ProviderRequirements[reqd.Name] = append(m.ProviderRequirements[reqd.Name], reqd.Requirement) + } + + for _, v := range file.Variables { + if existing, exists := m.Variables[v.Name]; exists { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate variable declaration", + Detail: fmt.Sprintf("A variable named %q was already declared at %s. Variable names must be unique within a module.", existing.Name, existing.DeclRange), + Subject: &v.DeclRange, + }) + } + m.Variables[v.Name] = v + } + + for _, l := range file.Locals { + if existing, exists := m.Locals[l.Name]; exists { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate local value definition", + Detail: fmt.Sprintf("A local value named %q was already defined at %s. Local value names must be unique within a module.", existing.Name, existing.DeclRange), + Subject: &l.DeclRange, + }) + } + m.Locals[l.Name] = l + } + + for _, o := range file.Outputs { + if existing, exists := m.Outputs[o.Name]; exists { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate output definition", + Detail: fmt.Sprintf("An output named %q was already defined at %s. Output names must be unique within a module.", existing.Name, existing.DeclRange), + Subject: &o.DeclRange, + }) + } + m.Outputs[o.Name] = o + } + + for _, mc := range file.ModuleCalls { + if existing, exists := m.ModuleCalls[mc.Name]; exists { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate module call", + Detail: fmt.Sprintf("An module call named %q was already defined at %s. Module calls must have unique names within a module.", existing.Name, existing.DeclRange), + Subject: &mc.DeclRange, + }) + } + m.ModuleCalls[mc.Name] = mc + } + + for _, r := range file.ManagedResources { + key := r.moduleUniqueKey() + if existing, exists := m.ManagedResources[key]; exists { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Duplicate resource %q configuration", existing.Type), + Detail: fmt.Sprintf("A %s resource named %q was already declared at %s. Resource names must be unique per type in each module.", existing.Type, existing.Name, existing.DeclRange), + Subject: &r.DeclRange, + }) + continue + } + m.ManagedResources[key] = r + } + + for _, r := range file.DataResources { + key := r.moduleUniqueKey() + if existing, exists := m.DataResources[key]; exists { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Duplicate data %q configuration", existing.Type), + Detail: fmt.Sprintf("A %s data resource named %q was already declared at %s. Resource names must be unique per type in each module.", existing.Type, existing.Name, existing.DeclRange), + Subject: &r.DeclRange, + }) + continue + } + m.DataResources[key] = r + } + + return diags +} + +func (m *Module) mergeFile(file *File) hcl.Diagnostics { + var diags hcl.Diagnostics + + if len(file.CoreVersionConstraints) != 0 { + // This is a bit of a strange case for overriding since we normally + // would union together across multiple files anyway, but we'll + // allow it and have each override file clobber any existing list. + m.CoreVersionConstraints = nil + for _, constraint := range file.CoreVersionConstraints { + m.CoreVersionConstraints = append(m.CoreVersionConstraints, constraint) + } + } + + if len(file.Backends) != 0 { + switch len(file.Backends) { + case 1: + m.Backend = file.Backends[0] + default: + // An override file with multiple backends is still invalid, even + // though it can override backends from _other_ files. + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate backend configuration", + Detail: fmt.Sprintf("Each override file may have only one backend configuration. A backend was previously configured at %s.", file.Backends[0].DeclRange), + Subject: &file.Backends[1].DeclRange, + }) + } + } + + for _, pc := range file.ProviderConfigs { + key := pc.moduleUniqueKey() + existing, exists := m.ProviderConfigs[key] + if pc.Alias == "" { + // We allow overriding a non-existing _default_ provider configuration + // because the user model is that an absent provider configuration + // implies an empty provider configuration, which is what the user + // is therefore overriding here. + if exists { + mergeDiags := existing.merge(pc) + diags = append(diags, mergeDiags...) + } else { + m.ProviderConfigs[key] = pc + } + } else { + // For aliased providers, there must be a base configuration to + // override. This allows us to detect and report alias typos + // that might otherwise cause the override to not apply. + if !exists { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing base provider configuration for override", + Detail: fmt.Sprintf("There is no %s provider configuration with the alias %q. An override file can only override an aliased provider configuration that was already defined in a primary configuration file.", pc.Name, pc.Alias), + Subject: &pc.DeclRange, + }) + continue + } + mergeDiags := existing.merge(pc) + diags = append(diags, mergeDiags...) + } + } + + if len(file.ProviderRequirements) != 0 { + mergeProviderVersionConstraints(m.ProviderRequirements, file.ProviderRequirements) + } + + for _, v := range file.Variables { + existing, exists := m.Variables[v.Name] + if !exists { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing base variable declaration to override", + Detail: fmt.Sprintf("There is no variable named %q. An override file can only override a variable that was already declared in a primary configuration file.", v.Name), + Subject: &v.DeclRange, + }) + continue + } + mergeDiags := existing.merge(v) + diags = append(diags, mergeDiags...) + } + + for _, l := range file.Locals { + existing, exists := m.Locals[l.Name] + if !exists { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing base local value definition to override", + Detail: fmt.Sprintf("There is no local value named %q. An override file can only override a local value that was already defined in a primary configuration file.", l.Name), + Subject: &l.DeclRange, + }) + continue + } + mergeDiags := existing.merge(l) + diags = append(diags, mergeDiags...) + } + + for _, o := range file.Outputs { + existing, exists := m.Outputs[o.Name] + if !exists { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing base output definition to override", + Detail: fmt.Sprintf("There is no output named %q. An override file can only override an output that was already defined in a primary configuration file.", o.Name), + Subject: &o.DeclRange, + }) + continue + } + mergeDiags := existing.merge(o) + diags = append(diags, mergeDiags...) + } + + for _, mc := range file.ModuleCalls { + existing, exists := m.ModuleCalls[mc.Name] + if !exists { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing module call to override", + Detail: fmt.Sprintf("There is no module call named %q. An override file can only override a module call that was defined in a primary configuration file.", mc.Name), + Subject: &mc.DeclRange, + }) + continue + } + mergeDiags := existing.merge(mc) + diags = append(diags, mergeDiags...) + } + + for _, r := range file.ManagedResources { + key := r.moduleUniqueKey() + existing, exists := m.ManagedResources[key] + if !exists { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing resource to override", + Detail: fmt.Sprintf("There is no %s resource named %q. An override file can only override a resource block defined in a primary configuration file.", r.Type, r.Name), + Subject: &r.DeclRange, + }) + continue + } + mergeDiags := existing.merge(r) + diags = append(diags, mergeDiags...) + } + + for _, r := range file.DataResources { + key := r.moduleUniqueKey() + existing, exists := m.DataResources[key] + if !exists { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing data resource to override", + Detail: fmt.Sprintf("There is no %s data resource named %q. An override file can only override a data block defined in a primary configuration file.", r.Type, r.Name), + Subject: &r.DeclRange, + }) + continue + } + mergeDiags := existing.merge(r) + diags = append(diags, mergeDiags...) + } + + return diags +} diff --git a/configs/module_merge.go b/configs/module_merge.go new file mode 100644 index 000000000..d6adc5442 --- /dev/null +++ b/configs/module_merge.go @@ -0,0 +1,212 @@ +package configs + +import ( + "github.com/hashicorp/hcl2/hcl" + "github.com/zclconf/go-cty/cty" +) + +// The methods in this file are used by Module.mergeFile to apply overrides +// to our different configuration elements. These methods all follow the +// pattern of mutating the receiver to incorporate settings from the parameter, +// returning error diagnostics if any aspect of the parameter cannot be merged +// into the receiver for some reason. +// +// User expectation is that anything _explicitly_ set in the given object +// should take precedence over the corresponding settings in the receiver, +// but that anything omitted in the given object should be left unchanged. +// In some cases it may be reasonable to do a "deep merge" of certain nested +// features, if it is possible to unambiguously correlate the nested elements +// and their behaviors are orthogonal to each other. + +func (p *Provider) merge(op *Provider) hcl.Diagnostics { + var diags hcl.Diagnostics + + if op.Version.Required != nil { + p.Version = op.Version + } + + p.Config = mergeBodies(p.Config, op.Config) + + return diags +} + +func mergeProviderVersionConstraints(recv map[string][]VersionConstraint, ovrd []*ProviderRequirement) { + // Any provider name that's mentioned in the override gets nilled out in + // our map so that we'll rebuild it below. Any provider not mentioned is + // left unchanged. + for _, reqd := range ovrd { + delete(recv, reqd.Name) + } + for _, reqd := range ovrd { + recv[reqd.Name] = append(recv[reqd.Name], reqd.Requirement) + } +} + +func (v *Variable) merge(ov *Variable) hcl.Diagnostics { + var diags hcl.Diagnostics + + if ov.Description != "" { + v.Description = ov.Description + } + if ov.Default != cty.NilVal { + v.Default = ov.Default + } + if ov.TypeHint != TypeHintNone { + v.TypeHint = ov.TypeHint + } + + return diags +} + +func (l *Local) merge(ol *Local) hcl.Diagnostics { + var diags hcl.Diagnostics + + // Since a local is just a single expression in configuration, the + // override definition entirely replaces the base definition, including + // the source range so that we'll send the user to the right place if + // there is an error. + l.Expr = ol.Expr + l.DeclRange = ol.DeclRange + + return diags +} + +func (o *Output) merge(oo *Output) hcl.Diagnostics { + var diags hcl.Diagnostics + + if oo.Description != "" { + o.Description = oo.Description + } + if oo.Expr != nil { + o.Expr = oo.Expr + } + if oo.Sensitive { + // Since this is just a bool, we can't distinguish false from unset + // and so the override can only make the output _more_ sensitive. + o.Sensitive = oo.Sensitive + } + + // We don't allow depends_on to be overridden because that is likely to + // cause confusing misbehavior. + if len(oo.DependsOn) != 0 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unsupported override", + Detail: "The depends_on argument may not be overridden.", + Subject: oo.DependsOn[0].SourceRange().Ptr(), // the first item is the closest range we have + }) + } + + return diags +} + +func (mc *ModuleCall) merge(omc *ModuleCall) hcl.Diagnostics { + var diags hcl.Diagnostics + + if omc.SourceAddr != "" { + mc.SourceAddr = omc.SourceAddr + mc.SourceAddrRange = omc.SourceAddrRange + } + + if omc.Count != nil { + mc.Count = omc.Count + } + + if omc.ForEach != nil { + mc.ForEach = omc.ForEach + } + + if len(omc.Version.Required) != 0 { + mc.Version = omc.Version + } + + mc.Config = mergeBodies(mc.Config, omc.Config) + + // We don't allow depends_on to be overridden because that is likely to + // cause confusing misbehavior. + if len(mc.DependsOn) != 0 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unsupported override", + Detail: "The depends_on argument may not be overridden.", + Subject: mc.DependsOn[0].SourceRange().Ptr(), // the first item is the closest range we have + }) + } + + return diags +} + +func (r *ManagedResource) merge(or *ManagedResource) hcl.Diagnostics { + var diags hcl.Diagnostics + + if or.Connection != nil { + r.Connection = or.Connection + } + if or.Count != nil { + r.Count = or.Count + } + if or.CreateBeforeDestroy { + // We can't distinguish false from unset here + r.CreateBeforeDestroy = or.CreateBeforeDestroy + } + if or.ForEach != nil { + r.ForEach = or.ForEach + } + if len(or.IgnoreChanges) != 0 { + r.IgnoreChanges = or.IgnoreChanges + } + if or.PreventDestroy { + // We can't distinguish false from unset here + r.PreventDestroy = or.PreventDestroy + } + if or.ProviderConfigRef != nil { + r.ProviderConfigRef = or.ProviderConfigRef + } + if len(or.Provisioners) != 0 { + r.Provisioners = or.Provisioners + } + + r.Config = mergeBodies(r.Config, or.Config) + + // We don't allow depends_on to be overridden because that is likely to + // cause confusing misbehavior. + if len(r.DependsOn) != 0 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unsupported override", + Detail: "The depends_on argument may not be overridden.", + Subject: r.DependsOn[0].SourceRange().Ptr(), // the first item is the closest range we have + }) + } + + return diags +} + +func (r *DataResource) merge(or *DataResource) hcl.Diagnostics { + var diags hcl.Diagnostics + + if or.Count != nil { + r.Count = or.Count + } + if or.ForEach != nil { + r.ForEach = or.ForEach + } + if or.ProviderConfigRef != nil { + r.ProviderConfigRef = or.ProviderConfigRef + } + + r.Config = mergeBodies(r.Config, or.Config) + + // We don't allow depends_on to be overridden because that is likely to + // cause confusing misbehavior. + if len(r.DependsOn) != 0 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unsupported override", + Detail: "The depends_on argument may not be overridden.", + Subject: r.DependsOn[0].SourceRange().Ptr(), // the first item is the closest range we have + }) + } + + return diags +} diff --git a/configs/module_merge_body.go b/configs/module_merge_body.go new file mode 100644 index 000000000..166d20430 --- /dev/null +++ b/configs/module_merge_body.go @@ -0,0 +1,47 @@ +package configs + +import ( + "github.com/hashicorp/hcl2/hcl" +) + +func mergeBodies(base, override hcl.Body) hcl.Body { + return mergeBody{ + Base: base, + Override: override, + } +} + +// mergeBody is a hcl.Body implementation that wraps a pair of other bodies +// and allows attributes and blocks within the override to take precedence +// over those defined in the base body. +// +// This is used to deal with dynamically-processed bodies in Module.mergeFile. +// It uses a shallow-only merging strategy where direct attributes defined +// in Override will override attributes of the same name in Base, while any +// blocks defined in Override will hide all blocks of the same type in Base. +// +// This cannot possibly "do the right thing" in all cases, because we don't +// have enough information about user intent. However, this behavior is intended +// to be reasonable for simple overriding use-cases. +type mergeBody struct { + Base hcl.Body + Override hcl.Body +} + +var _ hcl.Body = mergeBody{} + +func (b mergeBody) Content(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Diagnostics) { + panic("mergeBody.Content not yet implemented") +} + +func (b mergeBody) PartialContent(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Body, hcl.Diagnostics) { + panic("mergeBody.Content not yet implemented") +} + +func (b mergeBody) JustAttributes() (hcl.Attributes, hcl.Diagnostics) { + panic("mergeBody.JustAttributes not yet implemented") +} + +func (b mergeBody) MissingItemRange() hcl.Range { + return b.Base.MissingItemRange() +} diff --git a/configs/parser_config_dir.go b/configs/parser_config_dir.go new file mode 100644 index 000000000..57c72b604 --- /dev/null +++ b/configs/parser_config_dir.go @@ -0,0 +1,125 @@ +package configs + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/hashicorp/hcl2/hcl" +) + +// LoadConfigDir reads the .tf and .tf.json files in the given directory +// as config files (using LoadConfigFile) and then combines these files into +// a single Module. +// +// If this method returns nil, that indicates that the given directory does not +// exist at all or could not be opened for some reason. Callers may wish to +// detect this case and ignore the returned diagnostics so that they can +// produce a more context-aware error message in that case. +// +// If this method returns a non-nil module while error diagnostics are returned +// then the module may be incomplete but can be used carefully for static +// analysis. +// +// This file does not consider a directory with no files to be an error, and +// will simply return an empty module in that case. Callers should first call +// Parser.IsConfigDir if they wish to recognize that situation. +// +// .tf files are parsed using the HCL native syntax while .tf.json files are +// parsed using the HCL JSON syntax. +func (p *Parser) LoadConfigDir(path string) (*Module, hcl.Diagnostics) { + primaryPaths, overridePaths, diags := p.dirFiles(path) + if diags.HasErrors() { + return nil, diags + } + + primary, fDiags := p.loadFiles(primaryPaths) + diags = append(diags, fDiags...) + override, fDiags := p.loadFiles(overridePaths) + diags = append(diags, fDiags...) + + mod, modDiags := NewModule(primary, override) + diags = append(diags, modDiags...) + + return mod, diags +} + +// IsConfigDir determines whether the given path refers to a directory that +// exists and contains at least one Terraform config file (with a .tf or +// .tf.json extension.) +func (p *Parser) IsConfigDir(path string) bool { + primaryPaths, overridePaths, _ := p.dirFiles(path) + return (len(primaryPaths) + len(overridePaths)) > 0 +} + +func (p *Parser) loadFiles(paths []string) ([]*File, hcl.Diagnostics) { + var files []*File + var diags hcl.Diagnostics + + for _, path := range paths { + f, fDiags := p.LoadConfigFile(path) + diags = append(diags, fDiags...) + if f != nil { + files = append(files, f) + } + } + + return files, diags +} + +func (p *Parser) dirFiles(dir string) (primary, override []string, diags hcl.Diagnostics) { + infos, err := p.fs.ReadDir(dir) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to read module directory", + Detail: fmt.Sprintf("Module directory %s does not exist or cannot be read.", dir), + }) + return + } + + for _, info := range infos { + if info.IsDir() { + // We only care about files + continue + } + + name := info.Name() + ext := fileExt(name) + if ext == "" || IsIgnoredFile(name) { + continue + } + + baseName := name[:len(name)-len(ext)] // strip extension + isOverride := baseName == "override" || strings.HasSuffix(baseName, "_override") + + fullPath := filepath.Join(dir, name) + if isOverride { + override = append(override, fullPath) + } else { + primary = append(primary, fullPath) + } + } + + return +} + +// fileExt returns the Terraform configuration extension of the given +// path, or a blank string if it is not a recognized extension. +func fileExt(path string) string { + if strings.HasSuffix(path, ".tf") { + return ".tf" + } else if strings.HasSuffix(path, ".tf.json") { + return ".tf.json" + } else { + return "" + } +} + +// IsIgnoredFile returns true if the given filename (which must not have a +// directory path ahead of it) should be ignored as e.g. an editor swap file. +func IsIgnoredFile(name string) bool { + return strings.HasPrefix(name, ".") || // Unix-like hidden files + strings.HasSuffix(name, "~") || // vim + strings.HasPrefix(name, "#") && strings.HasSuffix(name, "#") // emacs +} diff --git a/configs/parser_config_dir_test.go b/configs/parser_config_dir_test.go new file mode 100644 index 000000000..703fc122d --- /dev/null +++ b/configs/parser_config_dir_test.go @@ -0,0 +1,136 @@ +package configs + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "testing" +) + +// TestParseLoadConfigDirSuccess is a simple test that just verifies that +// a number of test configuration directories (in test-fixtures/valid-modules) +// can be parsed without raising any diagnostics. +// +// It also re-tests the individual files in test-fixtures/valid-files as if +// they were single-file modules, to ensure that they can be bundled into +// modules correctly. +// +// This test does not verify that reading these modules produces the correct +// module element contents. More detailed assertions may be made on some subset +// of these configuration files in other tests. +func TestParserLoadConfigDirSuccess(t *testing.T) { + dirs, err := ioutil.ReadDir("test-fixtures/valid-modules") + if err != nil { + t.Fatal(err) + } + + for _, info := range dirs { + name := info.Name() + t.Run(name, func(t *testing.T) { + parser := NewParser(nil) + path := filepath.Join("test-fixtures/valid-modules", name) + + _, diags := parser.LoadConfigDir(path) + if len(diags) != 0 { + t.Errorf("unexpected diagnostics") + for _, diag := range diags { + t.Logf("- %s", diag) + } + } + }) + } + + // The individual files in test-fixtures/valid-files should also work + // when loaded as modules. + files, err := ioutil.ReadDir("test-fixtures/valid-files") + if err != nil { + t.Fatal(err) + } + + for _, info := range files { + name := info.Name() + t.Run(fmt.Sprintf("%s as module", name), func(t *testing.T) { + src, err := ioutil.ReadFile(filepath.Join("test-fixtures/valid-files", name)) + if err != nil { + t.Fatal(err) + } + + parser := testParser(map[string]string{ + "mod/" + name: string(src), + }) + + _, diags := parser.LoadConfigDir("mod") + if len(diags) != 0 { + t.Errorf("unexpected diagnostics") + for _, diag := range diags { + t.Logf("- %s", diag) + } + } + }) + } + +} + +// TestParseLoadConfigDirFailure is a simple test that just verifies that +// a number of test configuration directories (in test-fixtures/invalid-modules) +// produce diagnostics when parsed. +// +// It also re-tests the individual files in test-fixtures/invalid-files as if +// they were single-file modules, to ensure that their errors are still +// detected when loading as part of a module. +// +// This test does not verify that reading these modules produces any +// diagnostics in particular. More detailed assertions may be made on some subset +// of these configuration files in other tests. +func TestParserLoadConfigDirFailure(t *testing.T) { + dirs, err := ioutil.ReadDir("test-fixtures/invalid-modules") + if err != nil { + t.Fatal(err) + } + + for _, info := range dirs { + name := info.Name() + t.Run(name, func(t *testing.T) { + parser := NewParser(nil) + path := filepath.Join("test-fixtures/invalid-modules", name) + + _, diags := parser.LoadConfigDir(path) + if !diags.HasErrors() { + t.Errorf("no errors; want at least one") + for _, diag := range diags { + t.Logf("- %s", diag) + } + } + }) + } + + // The individual files in test-fixtures/valid-files should also work + // when loaded as modules. + files, err := ioutil.ReadDir("test-fixtures/invalid-files") + if err != nil { + t.Fatal(err) + } + + for _, info := range files { + name := info.Name() + t.Run(fmt.Sprintf("%s as module", name), func(t *testing.T) { + src, err := ioutil.ReadFile(filepath.Join("test-fixtures/invalid-files", name)) + if err != nil { + t.Fatal(err) + } + + parser := testParser(map[string]string{ + "mod/" + name: string(src), + }) + + _, diags := parser.LoadConfigDir("mod") + if !diags.HasErrors() { + t.Errorf("no errors; want at least one") + for _, diag := range diags { + t.Logf("- %s", diag) + } + } + }) + } + +} diff --git a/configs/provider.go b/configs/provider.go index d55874ad3..927490963 100644 --- a/configs/provider.go +++ b/configs/provider.go @@ -57,6 +57,13 @@ func decodeProviderBlock(block *hcl.Block) (*Provider, hcl.Diagnostics) { return provider, diags } +func (p *Provider) moduleUniqueKey() string { + if p.Alias != "" { + return fmt.Sprintf("%s.%s", p.Name, p.Alias) + } + return p.Name +} + // ProviderRequirement represents a declaration of a dependency on a particular // provider version without actually configuring that provider. This is used in // child modules that expect a provider to be passed in from their parent. diff --git a/configs/resource.go b/configs/resource.go index 027d04b2b..b66d12e38 100644 --- a/configs/resource.go +++ b/configs/resource.go @@ -31,6 +31,10 @@ type ManagedResource struct { TypeRange hcl.Range } +func (r *ManagedResource) moduleUniqueKey() string { + return fmt.Sprintf("%s.%s", r.Name, r.Type) +} + func decodeResourceBlock(block *hcl.Block) (*ManagedResource, hcl.Diagnostics) { r := &ManagedResource{ Type: block.Labels[0], @@ -170,6 +174,10 @@ type DataResource struct { TypeRange hcl.Range } +func (r *DataResource) moduleUniqueKey() string { + return fmt.Sprintf("data.%s.%s", r.Name, r.Type) +} + func decodeDataBlock(block *hcl.Block) (*DataResource, hcl.Diagnostics) { r := &DataResource{ Type: block.Labels[0], diff --git a/configs/test-fixtures/invalid-files/json-as-native-syntax.tf b/configs/test-fixtures/invalid-files/json-as-native-syntax.tf new file mode 100644 index 000000000..2e2809093 --- /dev/null +++ b/configs/test-fixtures/invalid-files/json-as-native-syntax.tf @@ -0,0 +1,3 @@ +{ + "terraform": {} +} diff --git a/configs/test-fixtures/invalid-files/native-syntax-as-json.tf.json b/configs/test-fixtures/invalid-files/native-syntax-as-json.tf.json new file mode 100644 index 000000000..ca88e62b4 --- /dev/null +++ b/configs/test-fixtures/invalid-files/native-syntax-as-json.tf.json @@ -0,0 +1,2 @@ +terraform { +} diff --git a/configs/test-fixtures/invalid-modules/override-nonexist-variable/override.tf b/configs/test-fixtures/invalid-modules/override-nonexist-variable/override.tf new file mode 100644 index 000000000..720db27b8 --- /dev/null +++ b/configs/test-fixtures/invalid-modules/override-nonexist-variable/override.tf @@ -0,0 +1,3 @@ +variable "foo" { + description = "overridden" +} diff --git a/configs/test-fixtures/valid-modules/empty/README b/configs/test-fixtures/valid-modules/empty/README new file mode 100644 index 000000000..6d937077a --- /dev/null +++ b/configs/test-fixtures/valid-modules/empty/README @@ -0,0 +1,2 @@ +This directory is intentionally empty, to test what happens when we load +a module that contains no configuration files. diff --git a/configs/test-fixtures/valid-modules/override-variable/a_override.tf b/configs/test-fixtures/valid-modules/override-variable/a_override.tf new file mode 100644 index 000000000..6ec4d1ef3 --- /dev/null +++ b/configs/test-fixtures/valid-modules/override-variable/a_override.tf @@ -0,0 +1,9 @@ +variable "fully_overridden" { + default = "a_override" + description = "a_override description" + type = string +} + +variable "partially_overridden" { + default = "a_override partial" +} diff --git a/configs/test-fixtures/valid-modules/override-variable/b_override.tf b/configs/test-fixtures/valid-modules/override-variable/b_override.tf new file mode 100644 index 000000000..21dbe82e9 --- /dev/null +++ b/configs/test-fixtures/valid-modules/override-variable/b_override.tf @@ -0,0 +1,9 @@ +variable "fully_overridden" { + default = "b_override" + description = "b_override description" + type = string +} + +variable "partially_overridden" { + default = "b_override partial" +} diff --git a/configs/test-fixtures/valid-modules/override-variable/primary.tf b/configs/test-fixtures/valid-modules/override-variable/primary.tf new file mode 100644 index 000000000..981b86b8e --- /dev/null +++ b/configs/test-fixtures/valid-modules/override-variable/primary.tf @@ -0,0 +1,11 @@ +variable "fully_overridden" { + default = "base" + description = "base description" + type = string +} + +variable "partially_overridden" { + default = "base" + description = "base description" + type = string +} From 7c8efe103e9bd013c5d093f351e62e3b34799345 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 6 Feb 2018 18:05:14 -0800 Subject: [PATCH 11/25] configs: allow overrides files to omit args that primary files can't Some of the fields in our config structs are either mandatory in primary files or there is a default value that we apply if absent. Unfortunately override files impose the additional constraint that we be allowed to omit required fields (which have presumably already been set in the primary files) and that we are able to distinguish between a default value and omitting a value entirely. Since most of our fields were already acceptable for override files, here we just add some new fields to deal with the few cases where special handling is required and a helper function to disable the "Required" flag on attributes in a given schema. --- configs/module_call.go | 11 +++- configs/module_merge.go | 19 +++--- configs/module_merge_test.go | 61 ++++++++++++++++++ configs/named_values.go | 17 ++++- configs/parser_config.go | 16 ++++- configs/parser_config_dir.go | 14 ++-- configs/parser_test.go | 64 +++++++++++++++++++ configs/resource.go | 5 ++ .../override-module/a_override.tf | 5 ++ .../override-module/b_override.tf | 5 ++ .../valid-modules/override-module/primary.tf | 7 ++ .../override-output-sensitive/override.tf | 3 + .../override-output-sensitive/primary.tf | 3 + configs/util.go | 27 ++++++++ 14 files changed, 238 insertions(+), 19 deletions(-) create mode 100644 configs/module_merge_test.go create mode 100644 configs/test-fixtures/valid-modules/override-module/a_override.tf create mode 100644 configs/test-fixtures/valid-modules/override-module/b_override.tf create mode 100644 configs/test-fixtures/valid-modules/override-module/primary.tf create mode 100644 configs/test-fixtures/valid-modules/override-output-sensitive/override.tf create mode 100644 configs/test-fixtures/valid-modules/override-output-sensitive/primary.tf diff --git a/configs/module_call.go b/configs/module_call.go index befccb28a..86f840216 100644 --- a/configs/module_call.go +++ b/configs/module_call.go @@ -12,6 +12,7 @@ type ModuleCall struct { SourceAddr string SourceAddrRange hcl.Range + SourceSet bool Config hcl.Body @@ -25,13 +26,18 @@ type ModuleCall struct { DeclRange hcl.Range } -func decodeModuleBlock(block *hcl.Block) (*ModuleCall, hcl.Diagnostics) { +func decodeModuleBlock(block *hcl.Block, override bool) (*ModuleCall, hcl.Diagnostics) { mc := &ModuleCall{ Name: block.Labels[0], DeclRange: block.DefRange, } - content, remain, diags := block.Body.PartialContent(moduleBlockSchema) + schema := moduleBlockSchema + if override { + schema = schemaForOverrides(schema) + } + + content, remain, diags := block.Body.PartialContent(schema) mc.Config = remain if !hclsyntax.ValidIdentifier(mc.Name) { @@ -47,6 +53,7 @@ func decodeModuleBlock(block *hcl.Block) (*ModuleCall, hcl.Diagnostics) { valDiags := gohcl.DecodeExpression(attr.Expr, nil, &mc.SourceAddr) diags = append(diags, valDiags...) mc.SourceAddrRange = attr.Expr.Range() + mc.SourceSet = true } if attr, exists := content.Attributes["version"]; exists { diff --git a/configs/module_merge.go b/configs/module_merge.go index d6adc5442..2e5ed3249 100644 --- a/configs/module_merge.go +++ b/configs/module_merge.go @@ -45,8 +45,9 @@ func mergeProviderVersionConstraints(recv map[string][]VersionConstraint, ovrd [ func (v *Variable) merge(ov *Variable) hcl.Diagnostics { var diags hcl.Diagnostics - if ov.Description != "" { + if ov.DescriptionSet { v.Description = ov.Description + v.DescriptionSet = ov.DescriptionSet } if ov.Default != cty.NilVal { v.Default = ov.Default @@ -80,10 +81,9 @@ func (o *Output) merge(oo *Output) hcl.Diagnostics { if oo.Expr != nil { o.Expr = oo.Expr } - if oo.Sensitive { - // Since this is just a bool, we can't distinguish false from unset - // and so the override can only make the output _more_ sensitive. + if oo.SensitiveSet { o.Sensitive = oo.Sensitive + o.SensitiveSet = oo.SensitiveSet } // We don't allow depends_on to be overridden because that is likely to @@ -103,9 +103,10 @@ func (o *Output) merge(oo *Output) hcl.Diagnostics { func (mc *ModuleCall) merge(omc *ModuleCall) hcl.Diagnostics { var diags hcl.Diagnostics - if omc.SourceAddr != "" { + if omc.SourceSet { mc.SourceAddr = omc.SourceAddr mc.SourceAddrRange = omc.SourceAddrRange + mc.SourceSet = omc.SourceSet } if omc.Count != nil { @@ -145,9 +146,9 @@ func (r *ManagedResource) merge(or *ManagedResource) hcl.Diagnostics { if or.Count != nil { r.Count = or.Count } - if or.CreateBeforeDestroy { - // We can't distinguish false from unset here + if or.CreateBeforeDestroySet { r.CreateBeforeDestroy = or.CreateBeforeDestroy + r.CreateBeforeDestroySet = or.CreateBeforeDestroySet } if or.ForEach != nil { r.ForEach = or.ForEach @@ -155,9 +156,9 @@ func (r *ManagedResource) merge(or *ManagedResource) hcl.Diagnostics { if len(or.IgnoreChanges) != 0 { r.IgnoreChanges = or.IgnoreChanges } - if or.PreventDestroy { - // We can't distinguish false from unset here + if or.PreventDestroySet { r.PreventDestroy = or.PreventDestroy + r.PreventDestroySet = or.PreventDestroySet } if or.ProviderConfigRef != nil { r.ProviderConfigRef = or.ProviderConfigRef diff --git a/configs/module_merge_test.go b/configs/module_merge_test.go new file mode 100644 index 000000000..6e74fd402 --- /dev/null +++ b/configs/module_merge_test.go @@ -0,0 +1,61 @@ +package configs + +import ( + "testing" + + "github.com/hashicorp/hcl2/hcl" + "github.com/zclconf/go-cty/cty" +) + +func TestModuleOverrideVariable(t *testing.T) { + mod, diags := testModuleFromDir("test-fixtures/valid-modules/override-variable") + assertNoDiagnostics(t, diags) + if mod == nil { + t.Fatalf("module is nil") + } + + got := mod.Variables + want := map[string]*Variable{ + "fully_overridden": { + Name: "fully_overridden", + Description: "b_override description", + DescriptionSet: true, + Default: cty.StringVal("b_override"), + TypeHint: TypeHintString, + DeclRange: hcl.Range{ + Filename: "test-fixtures/valid-modules/override-variable/primary.tf", + Start: hcl.Pos{ + Line: 1, + Column: 1, + Byte: 0, + }, + End: hcl.Pos{ + Line: 1, + Column: 28, + Byte: 27, + }, + }, + }, + "partially_overridden": { + Name: "partially_overridden", + Description: "base description", + DescriptionSet: true, + Default: cty.StringVal("b_override partial"), + TypeHint: TypeHintString, + DeclRange: hcl.Range{ + Filename: "test-fixtures/valid-modules/override-variable/primary.tf", + Start: hcl.Pos{ + Line: 7, + Column: 1, + Byte: 103, + }, + End: hcl.Pos{ + Line: 7, + Column: 32, + Byte: 134, + }, + }, + }, + } + assertResultDeepEqual(t, got, want) +} diff --git a/configs/named_values.go b/configs/named_values.go index 052b04017..42ae4750d 100644 --- a/configs/named_values.go +++ b/configs/named_values.go @@ -19,6 +19,8 @@ type Variable struct { Default cty.Value TypeHint VariableTypeHint + DescriptionSet bool + DeclRange hcl.Range } @@ -56,6 +58,7 @@ func decodeVariableBlock(block *hcl.Block) (*Variable, hcl.Diagnostics) { if attr, exists := content.Attributes["description"]; exists { valDiags := gohcl.DecodeExpression(attr.Expr, nil, &v.Description) diags = append(diags, valDiags...) + v.DescriptionSet = true } if attr, exists := content.Attributes["default"]; exists { @@ -106,16 +109,24 @@ type Output struct { DependsOn []hcl.Traversal Sensitive bool + DescriptionSet bool + SensitiveSet bool + DeclRange hcl.Range } -func decodeOutputBlock(block *hcl.Block) (*Output, hcl.Diagnostics) { +func decodeOutputBlock(block *hcl.Block, override bool) (*Output, hcl.Diagnostics) { o := &Output{ Name: block.Labels[0], DeclRange: block.DefRange, } - content, diags := block.Body.Content(outputBlockSchema) + schema := outputBlockSchema + if override { + schema = schemaForOverrides(schema) + } + + content, diags := block.Body.Content(schema) if !hclsyntax.ValidIdentifier(o.Name) { diags = append(diags, &hcl.Diagnostic{ @@ -129,6 +140,7 @@ func decodeOutputBlock(block *hcl.Block) (*Output, hcl.Diagnostics) { if attr, exists := content.Attributes["description"]; exists { valDiags := gohcl.DecodeExpression(attr.Expr, nil, &o.Description) diags = append(diags, valDiags...) + o.DescriptionSet = true } if attr, exists := content.Attributes["value"]; exists { @@ -138,6 +150,7 @@ func decodeOutputBlock(block *hcl.Block) (*Output, hcl.Diagnostics) { if attr, exists := content.Attributes["sensitive"]; exists { valDiags := gohcl.DecodeExpression(attr.Expr, nil, &o.Sensitive) diags = append(diags, valDiags...) + o.SensitiveSet = true } if attr, exists := content.Attributes["depends_on"]; exists { diff --git a/configs/parser_config.go b/configs/parser_config.go index 16eb77c34..c7fc33fd9 100644 --- a/configs/parser_config.go +++ b/configs/parser_config.go @@ -19,6 +19,18 @@ import ( // This method wraps LoadHCLFile, and so it inherits the syntax selection // behaviors documented for that method. func (p *Parser) LoadConfigFile(path string) (*File, hcl.Diagnostics) { + return p.loadConfigFile(path, false) +} + +// LoadConfigFileOverride is the same as LoadConfigFile except that it relaxes +// certain required attribute constraints in order to interpret the given +// file as an overrides file. +func (p *Parser) LoadConfigFileOverride(path string) (*File, hcl.Diagnostics) { + return p.loadConfigFile(path, true) +} + +func (p *Parser) loadConfigFile(path string, override bool) (*File, hcl.Diagnostics) { + body, diags := p.LoadHCLFile(path) if body == nil { return nil, diags @@ -86,14 +98,14 @@ func (p *Parser) LoadConfigFile(path string) (*File, hcl.Diagnostics) { file.Locals = append(file.Locals, defs...) case "output": - cfg, cfgDiags := decodeOutputBlock(block) + cfg, cfgDiags := decodeOutputBlock(block, override) diags = append(diags, cfgDiags...) if cfg != nil { file.Outputs = append(file.Outputs, cfg) } case "module": - cfg, cfgDiags := decodeModuleBlock(block) + cfg, cfgDiags := decodeModuleBlock(block, override) diags = append(diags, cfgDiags...) if cfg != nil { file.ModuleCalls = append(file.ModuleCalls, cfg) diff --git a/configs/parser_config_dir.go b/configs/parser_config_dir.go index 57c72b604..fad3a9837 100644 --- a/configs/parser_config_dir.go +++ b/configs/parser_config_dir.go @@ -33,9 +33,9 @@ func (p *Parser) LoadConfigDir(path string) (*Module, hcl.Diagnostics) { return nil, diags } - primary, fDiags := p.loadFiles(primaryPaths) + primary, fDiags := p.loadFiles(primaryPaths, false) diags = append(diags, fDiags...) - override, fDiags := p.loadFiles(overridePaths) + override, fDiags := p.loadFiles(overridePaths, true) diags = append(diags, fDiags...) mod, modDiags := NewModule(primary, override) @@ -52,12 +52,18 @@ func (p *Parser) IsConfigDir(path string) bool { return (len(primaryPaths) + len(overridePaths)) > 0 } -func (p *Parser) loadFiles(paths []string) ([]*File, hcl.Diagnostics) { +func (p *Parser) loadFiles(paths []string, override bool) ([]*File, hcl.Diagnostics) { var files []*File var diags hcl.Diagnostics for _, path := range paths { - f, fDiags := p.LoadConfigFile(path) + var f *File + var fDiags hcl.Diagnostics + if override { + f, fDiags = p.LoadConfigFileOverride(path) + } else { + f, fDiags = p.LoadConfigFile(path) + } diags = append(diags, fDiags...) if f != nil { files = append(files, f) diff --git a/configs/parser_test.go b/configs/parser_test.go index 316a09e5d..f5f1edba2 100644 --- a/configs/parser_test.go +++ b/configs/parser_test.go @@ -3,7 +3,12 @@ package configs import ( "os" "path" + "reflect" + "testing" + "github.com/davecgh/go-spew/spew" + + "github.com/hashicorp/hcl2/hcl" "github.com/spf13/afero" ) @@ -29,3 +34,62 @@ func testParser(files map[string]string) *Parser { return NewParser(fs) } + +// testModuleFromFile reads a single file, wraps it in a module, and returns +// it. This is a helper for use in unit tests. +func testModuleFromFile(filename string) (*Module, hcl.Diagnostics) { + parser := NewParser(nil) + f, diags := parser.LoadConfigFile(filename) + mod, modDiags := NewModule([]*File{f}, nil) + diags = append(diags, modDiags...) + return mod, modDiags +} + +// testModuleFromDir reads configuration from the given directory path as +// a module and returns it. This is a helper for use in unit tests. +func testModuleFromDir(path string) (*Module, hcl.Diagnostics) { + parser := NewParser(nil) + return parser.LoadConfigDir(path) +} + +func assertNoDiagnostics(t *testing.T, diags hcl.Diagnostics) bool { + t.Helper() + return assertDiagnosticCount(t, diags, 0) +} + +func assertDiagnosticCount(t *testing.T, diags hcl.Diagnostics, want int) bool { + t.Helper() + if len(diags) != 0 { + t.Errorf("wrong number of diagnostics %d; want %d", len(diags), want) + for _, diag := range diags { + t.Logf("- %s", diag) + } + return true + } + return false +} + +func assertDiagnosticSummary(t *testing.T, diags hcl.Diagnostics, want string) bool { + t.Helper() + + for _, diag := range diags { + if diag.Summary == want { + return false + } + } + + t.Errorf("missing diagnostic summary %q", want) + for _, diag := range diags { + t.Logf("- %s", diag) + } + return true +} + +func assertResultDeepEqual(t *testing.T, got, want interface{}) bool { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Errorf("wrong result\ngot: %swant: %s", spew.Sdump(got), spew.Sdump(want)) + return true + } + return false +} diff --git a/configs/resource.go b/configs/resource.go index b66d12e38..3892c86b9 100644 --- a/configs/resource.go +++ b/configs/resource.go @@ -27,6 +27,9 @@ type ManagedResource struct { PreventDestroy bool IgnoreChanges []hcl.Traversal + CreateBeforeDestroySet bool + PreventDestroySet bool + DeclRange hcl.Range TypeRange hcl.Range } @@ -105,11 +108,13 @@ func decodeResourceBlock(block *hcl.Block) (*ManagedResource, hcl.Diagnostics) { if attr, exists := lcContent.Attributes["create_before_destroy"]; exists { valDiags := gohcl.DecodeExpression(attr.Expr, nil, &r.CreateBeforeDestroy) diags = append(diags, valDiags...) + r.CreateBeforeDestroySet = true } if attr, exists := lcContent.Attributes["prevent_destroy"]; exists { valDiags := gohcl.DecodeExpression(attr.Expr, nil, &r.PreventDestroy) diags = append(diags, valDiags...) + r.PreventDestroySet = true } if attr, exists := lcContent.Attributes["ignore_changes"]; exists { diff --git a/configs/test-fixtures/valid-modules/override-module/a_override.tf b/configs/test-fixtures/valid-modules/override-module/a_override.tf new file mode 100644 index 000000000..9e0403ff8 --- /dev/null +++ b/configs/test-fixtures/valid-modules/override-module/a_override.tf @@ -0,0 +1,5 @@ + +module "example" { + foo = "a_override foo" + new = "a_override new" +} diff --git a/configs/test-fixtures/valid-modules/override-module/b_override.tf b/configs/test-fixtures/valid-modules/override-module/b_override.tf new file mode 100644 index 000000000..98d5a3101 --- /dev/null +++ b/configs/test-fixtures/valid-modules/override-module/b_override.tf @@ -0,0 +1,5 @@ + +module "example" { + new = "b_override new" + newer = "b_override newer" +} diff --git a/configs/test-fixtures/valid-modules/override-module/primary.tf b/configs/test-fixtures/valid-modules/override-module/primary.tf new file mode 100644 index 000000000..0567e7867 --- /dev/null +++ b/configs/test-fixtures/valid-modules/override-module/primary.tf @@ -0,0 +1,7 @@ + +module "example" { + source = "./example2" + + kept = "primary kept" + foo = "primary foo" +} diff --git a/configs/test-fixtures/valid-modules/override-output-sensitive/override.tf b/configs/test-fixtures/valid-modules/override-output-sensitive/override.tf new file mode 100644 index 000000000..b965fc124 --- /dev/null +++ b/configs/test-fixtures/valid-modules/override-output-sensitive/override.tf @@ -0,0 +1,3 @@ +output "foo" { + sensitive = true +} diff --git a/configs/test-fixtures/valid-modules/override-output-sensitive/primary.tf b/configs/test-fixtures/valid-modules/override-output-sensitive/primary.tf new file mode 100644 index 000000000..13bd3a99b --- /dev/null +++ b/configs/test-fixtures/valid-modules/override-output-sensitive/primary.tf @@ -0,0 +1,3 @@ +output "foo" { + value = "Hello World" +} diff --git a/configs/util.go b/configs/util.go index 9594cbeee..002bb8cb8 100644 --- a/configs/util.go +++ b/configs/util.go @@ -16,3 +16,30 @@ func exprIsNativeQuotedString(expr hcl.Expression) bool { _, ok := expr.(*hclsyntax.TemplateExpr) return ok } + +// schemaForOverrides takes a *hcl.BodySchema and produces a new one that is +// equivalent except that any required attributes are forced to not be required. +// +// This is useful for dealing with "override" config files, which are allowed +// to omit things that they don't wish to override from the main configuration. +// +// The returned schema may have some pointers in common with the given schema, +// so neither the given schema nor the returned schema should be modified after +// using this function in order to avoid confusion. +// +// Overrides are rarely used, so it's recommended to just create the override +// schema on the fly only when it's needed, rather than storing it in a global +// variable as we tend to do for a primary schema. +func schemaForOverrides(schema *hcl.BodySchema) *hcl.BodySchema { + ret := &hcl.BodySchema{ + Attributes: make([]hcl.AttributeSchema, len(schema.Attributes)), + Blocks: schema.Blocks, + } + + for i, attrS := range schema.Attributes { + ret.Attributes[i] = attrS + ret.Attributes[i].Required = false + } + + return ret +} From cc38e91612b700dd3c44ef085d955267666f1833 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 6 Feb 2018 18:30:12 -0800 Subject: [PATCH 12/25] configs: Implementation of mergeBody mergeBody is a hcl.Body implementation that deals with our override file merging behavior for the portions of the configuration that are not processed until full eval time. Mimicking the behavior of our old config merge implementation from the "config" package, the rules here are: - Attributes in the override body hide attributes of the same name in the base body. - Any block in the override body hides all blocks with the same type name that appear in the base body. This is tested by a new test for the overriding of module arguments, which asserts the correct behavior of the merged body as part of its work. --- configs/module_merge_body.go | 80 ++++++++++++++++++- configs/module_merge_test.go | 75 +++++++++++++++++ configs/parser_test.go | 4 + .../override-module/a_override.tf | 2 + 4 files changed, 158 insertions(+), 3 deletions(-) diff --git a/configs/module_merge_body.go b/configs/module_merge_body.go index 166d20430..b1308e976 100644 --- a/configs/module_merge_body.go +++ b/configs/module_merge_body.go @@ -31,15 +31,89 @@ type mergeBody struct { var _ hcl.Body = mergeBody{} func (b mergeBody) Content(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Diagnostics) { - panic("mergeBody.Content not yet implemented") + var diags hcl.Diagnostics + oSchema := schemaForOverrides(schema) + + baseContent, cDiags := b.Base.Content(schema) + diags = append(diags, cDiags...) + overrideContent, cDiags := b.Override.Content(oSchema) + diags = append(diags, cDiags...) + + content := b.prepareContent(baseContent, overrideContent) + + return content, diags } func (b mergeBody) PartialContent(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Body, hcl.Diagnostics) { - panic("mergeBody.Content not yet implemented") + var diags hcl.Diagnostics + oSchema := schemaForOverrides(schema) + + baseContent, baseRemain, cDiags := b.Base.PartialContent(schema) + diags = append(diags, cDiags...) + overrideContent, overrideRemain, cDiags := b.Override.PartialContent(oSchema) + diags = append(diags, cDiags...) + + content := b.prepareContent(baseContent, overrideContent) + + remain := mergeBodies(baseRemain, overrideRemain) + + return content, remain, diags +} + +func (b mergeBody) prepareContent(base *hcl.BodyContent, override *hcl.BodyContent) *hcl.BodyContent { + content := &hcl.BodyContent{ + Attributes: make(hcl.Attributes), + } + + // For attributes we just assign from each map in turn and let the override + // map clobber any matching entries from base. + for k, a := range base.Attributes { + content.Attributes[k] = a + } + for k, a := range override.Attributes { + content.Attributes[k] = a + } + + // Things are a little more interesting for blocks because they arrive + // as a flat list. Our merging semantics call for us to suppress blocks + // from base if at least one block of the same type appears in override. + // We explicitly do not try to correlate and deeply merge nested blocks, + // since we don't have enough context here to infer user intent. + + overriddenBlockTypes := make(map[string]bool) + for _, block := range override.Blocks { + overriddenBlockTypes[block.Type] = true + } + for _, block := range base.Blocks { + if overriddenBlockTypes[block.Type] { + continue + } + content.Blocks = append(content.Blocks, block) + } + for _, block := range override.Blocks { + content.Blocks = append(content.Blocks, block) + } + + return content } func (b mergeBody) JustAttributes() (hcl.Attributes, hcl.Diagnostics) { - panic("mergeBody.JustAttributes not yet implemented") + var diags hcl.Diagnostics + ret := make(hcl.Attributes) + + baseAttrs, aDiags := b.Base.JustAttributes() + diags = append(diags, aDiags...) + overrideAttrs, aDiags := b.Override.JustAttributes() + diags = append(diags, aDiags...) + + for k, a := range baseAttrs { + ret[k] = a + } + for k, a := range overrideAttrs { + ret[k] = a + } + + return ret, diags } func (b mergeBody) MissingItemRange() hcl.Range { diff --git a/configs/module_merge_test.go b/configs/module_merge_test.go index 6e74fd402..14126afa4 100644 --- a/configs/module_merge_test.go +++ b/configs/module_merge_test.go @@ -3,6 +3,7 @@ package configs import ( "testing" + "github.com/hashicorp/hcl2/gohcl" "github.com/hashicorp/hcl2/hcl" "github.com/zclconf/go-cty/cty" ) @@ -59,3 +60,77 @@ func TestModuleOverrideVariable(t *testing.T) { } assertResultDeepEqual(t, got, want) } + +func TestModuleOverrideModule(t *testing.T) { + mod, diags := testModuleFromDir("test-fixtures/valid-modules/override-module") + assertNoDiagnostics(t, diags) + if mod == nil { + t.Fatalf("module is nil") + } + + if _, exists := mod.ModuleCalls["example"]; !exists { + t.Fatalf("no module 'example'") + } + if len(mod.ModuleCalls) != 1 { + t.Fatalf("wrong number of module calls in result %d; want 1", len(mod.ModuleCalls)) + } + + got := mod.ModuleCalls["example"] + want := &ModuleCall{ + Name: "example", + SourceAddr: "./example2-a_override", + SourceAddrRange: hcl.Range{ + Filename: "test-fixtures/valid-modules/override-module/a_override.tf", + Start: hcl.Pos{ + Line: 3, + Column: 12, + Byte: 31, + }, + End: hcl.Pos{ + Line: 3, + Column: 35, + Byte: 54, + }, + }, + SourceSet: true, + DeclRange: hcl.Range{ + Filename: "test-fixtures/valid-modules/override-module/primary.tf", + Start: hcl.Pos{ + Line: 2, + Column: 1, + Byte: 1, + }, + End: hcl.Pos{ + Line: 2, + Column: 17, + Byte: 17, + }, + }, + } + + // We're going to extract and nil out our hcl.Body here because DeepEqual + // is not a useful way to assert on that. + gotConfig := got.Config + got.Config = nil + + assertResultDeepEqual(t, got, want) + + type content struct { + Kept *string `hcl:"kept"` + Foo *string `hcl:"foo"` + New *string `hcl:"new"` + Newer *string `hcl:"newer"` + } + var gotArgs content + diags = gohcl.DecodeBody(gotConfig, nil, &gotArgs) + assertNoDiagnostics(t, diags) + + wantArgs := content{ + Kept: stringPtr("primary kept"), + Foo: stringPtr("a_override foo"), + New: stringPtr("b_override new"), + Newer: stringPtr("b_override newer"), + } + + assertResultDeepEqual(t, gotArgs, wantArgs) +} diff --git a/configs/parser_test.go b/configs/parser_test.go index f5f1edba2..93939fd88 100644 --- a/configs/parser_test.go +++ b/configs/parser_test.go @@ -93,3 +93,7 @@ func assertResultDeepEqual(t *testing.T, got, want interface{}) bool { } return false } + +func stringPtr(s string) *string { + return &s +} diff --git a/configs/test-fixtures/valid-modules/override-module/a_override.tf b/configs/test-fixtures/valid-modules/override-module/a_override.tf index 9e0403ff8..c1b8d7cba 100644 --- a/configs/test-fixtures/valid-modules/override-module/a_override.tf +++ b/configs/test-fixtures/valid-modules/override-module/a_override.tf @@ -1,5 +1,7 @@ module "example" { + source = "./example2-a_override" + foo = "a_override foo" new = "a_override new" } From 8929eca405d353da1347c3a7a5c1835871e32b71 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 7 Feb 2018 16:40:58 -0800 Subject: [PATCH 13/25] configs: BuildConfig function BuildConfig creates a module tree by recursively walking through module calls in the root module and any descendent modules. This is intended to be used both for the simple case of loading already-installed modules and the more complex case of installing modules inside "terraform init", both of which will be dealt with in a separate package. --- configs/config.go | 120 ++++++++++++++++- configs/config_build.go | 127 ++++++++++++++++++ configs/config_build_test.go | 71 ++++++++++ .../config-build/child_a/child_a.tf | 4 + .../config-build/child_b/child_b.tf | 7 + .../config-build/child_c/child_c.tf | 3 + configs/test-fixtures/config-build/root.tf | 9 ++ 7 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 configs/config_build.go create mode 100644 configs/config_build_test.go create mode 100644 configs/test-fixtures/config-build/child_a/child_a.tf create mode 100644 configs/test-fixtures/config-build/child_b/child_b.tf create mode 100644 configs/test-fixtures/config-build/child_c/child_c.tf create mode 100644 configs/test-fixtures/config-build/root.tf diff --git a/configs/config.go b/configs/config.go index 07f6ed0a5..0c07ba143 100644 --- a/configs/config.go +++ b/configs/config.go @@ -1,5 +1,12 @@ package configs +import ( + "fmt" + + version "github.com/hashicorp/go-version" + "github.com/hashicorp/hcl2/hcl" +) + // A Config is a node in the tree of modules within a configuration. // // The module tree is constructed by following ModuleCall instances recursively @@ -24,7 +31,116 @@ type Config struct { // Module.ModuleCalls. Children map[string]*Config - // Elements points to the object describing the configuration for the + // Module points to the object describing the configuration for the // various elements (variables, resources, etc) defined by this module. - Elements *Module + Module *Module + + // SourceAddr is the source address that the referenced module was requested + // from, as specified in configuration. + // + // This field is meaningless for the root module, where its contents are undefined. + SourceAddr string + + // SourceAddrRange is the location in the configuration source where the + // SourceAddr value was set, for use in diagnostic messages. + // + // This field is meaningless for the root module, where its contents are undefined. + SourceAddrRange hcl.Range + + // Version is the specific version that was selected for this module, + // based on version constraints given in configuration. + // + // This field is meaningless for the root module, where its contents are undefined. + Version *version.Version +} + +// Path returns the path of logical names that lead to this Config from its +// root. +// +// This function should not be used to display a path to the end-user, since +// our UI conventions call for us to return a module address string in that +// case, and a module address string ought to be built from the dynamic +// module tree (resulting from evaluating "count" and "for_each" arguments +// on our calls to produce potentially multiple child instances per call) +// rather than from our static module tree. +// +// This function will panic if called on a config that is not part of a +// wholesome config tree, e.g. because it has incorrectly-built Children +// maps, missing node pointers, etc. However, it should work as expected +// for any tree constructed by BuildConfig and not subsequently modified. +func (c *Config) Path() []string { + // The implementation here is not especially efficient, but we don't + // care too much because module trees are shallow and narrow in all + // reasonable configurations. + + // We'll build our path in reverse here, since we're starting at the + // leafiest node, and then we'll flip it before we return. + path := make([]string, 0, c.Depth()) + + this := c + for this.Parent != nil { + parent := this.Parent + var name string + for candidate, ref := range parent.Children { + if ref == this { + name = candidate + } + } + if name == "" { + panic(fmt.Errorf( + "Config %p does not appear in the child table for its parent %p: %#v", + this, parent, parent.Children, + )) + } + path = append(path, name) + this = parent + } + + // reverse the items + for i := 0; i < len(path)/2; i++ { + j := len(path) - i - 1 + path[i], path[j] = path[j], path[i] + } + return path +} + +// Depth returns the number of "hops" the receiver is from the root of its +// module tree, with the root module having a depth of zero. +func (c *Config) Depth() int { + ret := 0 + this := c + for this.Parent != nil { + ret++ + this = this.Parent + } + return ret +} + +// DeepEach calls the given function once for each module in the tree, starting +// with the receiver. +// +// A parent is always called before its children and children of a particular +// node are visited in lexicographic order by their names. +func (c *Config) DeepEach(cb func(c *Config)) { + cb(c) + + names := make([]string, 0, len(c.Children)) + for name := range c.Children { + names = append(names, name) + } + + for _, name := range names { + c.Children[name].DeepEach(cb) + } +} + +// AllModules returns a slice of all the receiver and all of its descendent +// nodes in the module tree, in the same order they would be visited by +// DeepEach. +func (c *Config) AllModules() []*Config { + var ret []*Config + c.DeepEach(func(c *Config) { + ret = append(ret, c) + }) + return ret } diff --git a/configs/config_build.go b/configs/config_build.go new file mode 100644 index 000000000..ecbf8bc47 --- /dev/null +++ b/configs/config_build.go @@ -0,0 +1,127 @@ +package configs + +import ( + version "github.com/hashicorp/go-version" + "github.com/hashicorp/hcl2/hcl" +) + +// BuildConfig constructs a Config from a root module by loading all of its +// descendent modules via the given ModuleWalker. +// +// The result is a module tree that has so far only had basic module- and +// file-level invariants validated. If the returned diagnostics contains errors, +// the returned module tree may be incomplete but can still be used carefully +// for static analysis. +func BuildConfig(root *Module, walker ModuleWalker) (*Config, hcl.Diagnostics) { + var diags hcl.Diagnostics + cfg := &Config{ + Module: root, + } + cfg.Root = cfg // Root module is self-referential. + cfg.Children, diags = buildChildModules(cfg, walker) + return cfg, diags +} + +func buildChildModules(parent *Config, walker ModuleWalker) (map[string]*Config, hcl.Diagnostics) { + var diags hcl.Diagnostics + ret := map[string]*Config{} + + calls := parent.Module.ModuleCalls + + for _, call := range calls { + req := ModuleRequest{ + Name: call.Name, + SourceAddr: call.SourceAddr, + SourceAddrRange: call.SourceAddrRange, + VersionConstraints: []VersionConstraint{call.Version}, + Parent: parent, + } + + mod, ver, modDiags := walker.LoadModule(&req) + diags = append(diags, modDiags...) + if mod == nil { + // nil can be returned if the source address was invalid and so + // nothing could be loaded whatsoever. LoadModule should've + // returned at least one error diagnostic in that case. + continue + } + + child := &Config{ + Parent: parent, + Root: parent.Root, + Module: mod, + SourceAddr: call.SourceAddr, + SourceAddrRange: call.SourceAddrRange, + Version: ver, + } + + child.Children, modDiags = buildChildModules(child, walker) + + ret[call.Name] = child + } + + return ret, diags +} + +// A ModuleWalker knows how to find and load a child module given details about +// the module to be loaded and a reference to its partially-loaded parent +// Config. +type ModuleWalker interface { + // LoadModule finds and loads a requested child module. + // + // If errors are detected during loading, implementations should return them + // in the diagnostics object. If the diagnostics object contains any errors + // then the caller will tolerate the returned module being nil or incomplete. + // If no errors are returned, it should be non-nil and complete. + // + // Full validation need not have been performed but an implementation should + // ensure that the basic file- and module-validations performed by the + // LoadConfigDir function (valid syntax, no namespace collisions, etc) have + // been performed before returning a module. + LoadModule(req *ModuleRequest) (*Module, *version.Version, hcl.Diagnostics) +} + +// ModuleWalkerFunc is an implementation of ModuleWalker that directly wraps +// a callback function, for more convenient use of that interface. +type ModuleWalkerFunc func(req *ModuleRequest) (*Module, *version.Version, hcl.Diagnostics) + +// LoadModule implements ModuleWalker. +func (f ModuleWalkerFunc) LoadModule(req *ModuleRequest) (*Module, *version.Version, hcl.Diagnostics) { + return f(req) +} + +// ModuleRequest is used with the ModuleWalker interface to describe a child +// module that must be loaded. +type ModuleRequest struct { + // Name is the "logical name" of the module call within configuration. + // This is provided in case the name is used as part of a storage key + // for the module, but implementations must otherwise treat it as an + // opaque string. It is guaranteed to have already been validated as an + // HCL identifier and UTF-8 encoded. + Name string + + // SourceAddr is the source address string provided by the user in + // configuration. + SourceAddr string + + // SourceAddrRange is the source range for the SourceAddr value as it + // was provided in configuration. This can and should be used to generate + // diagnostics about the source address having invalid syntax, referring + // to a non-existent object, etc. + SourceAddrRange hcl.Range + + // VersionConstraints are the constraints applied to the module in + // configuration. This data structure includes the source range for + // each constraint, which can and should be used to generate diagnostics + // about constraint-related issues, such as constraints that eliminate all + // available versions of a module whose source is otherwise valid. + VersionConstraints []VersionConstraint + + // Parent is the partially-constructed module tree node that the loaded + // module will be added to. Callers may refer to any field of this + // structure except Children, which is still under construction when + // ModuleRequest objects are created and thus has undefined content. + // The main reason this is provided is so that full module paths can + // be constructed for uniqueness. + Parent *Config +} diff --git a/configs/config_build_test.go b/configs/config_build_test.go new file mode 100644 index 000000000..3ac46e35c --- /dev/null +++ b/configs/config_build_test.go @@ -0,0 +1,71 @@ +package configs + +import ( + "fmt" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + + "github.com/davecgh/go-spew/spew" + + version "github.com/hashicorp/go-version" + "github.com/hashicorp/hcl2/hcl" +) + +func TestBuildConfig(t *testing.T) { + parser := NewParser(nil) + mod, diags := parser.LoadConfigDir("test-fixtures/config-build") + assertNoDiagnostics(t, diags) + if mod == nil { + t.Fatal("got nil root module; want non-nil") + } + + versionI := 0 + cfg, diags := BuildConfig(mod, ModuleWalkerFunc( + func(req *ModuleRequest) (*Module, *version.Version, hcl.Diagnostics) { + // For the sake of this test we're going to just treat our + // SourceAddr as a path relative to our fixture directory. + // A "real" implementation of ModuleWalker should accept the + // various different source address syntaxes Terraform supports. + sourcePath := filepath.Join("test-fixtures/config-build", req.SourceAddr) + + mod, diags := parser.LoadConfigDir(sourcePath) + version, _ := version.NewVersion(fmt.Sprintf("1.0.%d", versionI)) + versionI++ + return mod, version, diags + }, + )) + assertNoDiagnostics(t, diags) + if cfg == nil { + t.Fatal("got nil config; want non-nil") + } + + var got []string + cfg.DeepEach(func(c *Config) { + got = append(got, fmt.Sprintf("%s %s", strings.Join(c.Path(), "."), c.Version)) + }) + sort.Strings(got) + want := []string{ + " ", + "child_a 1.0.0", + "child_a.child_c 1.0.1", + "child_b 1.0.2", + "child_b.child_c 1.0.3", + } + + if !reflect.DeepEqual(got, want) { + t.Fatalf("wrong result\ngot: %swant: %s", spew.Sdump(got), spew.Sdump(want)) + } + + if _, exists := cfg.Children["child_a"].Children["child_c"].Module.Outputs["hello"]; !exists { + t.Fatalf("missing output 'hello' in child_a.child_c") + } + if _, exists := cfg.Children["child_b"].Children["child_c"].Module.Outputs["hello"]; !exists { + t.Fatalf("missing output 'hello' in child_b.child_c") + } + if cfg.Children["child_a"].Children["child_c"].Module == cfg.Children["child_b"].Children["child_c"].Module { + t.Fatalf("child_a.child_c is same object as child_b.child_c; should not be") + } +} diff --git a/configs/test-fixtures/config-build/child_a/child_a.tf b/configs/test-fixtures/config-build/child_a/child_a.tf new file mode 100644 index 000000000..6c5759701 --- /dev/null +++ b/configs/test-fixtures/config-build/child_a/child_a.tf @@ -0,0 +1,4 @@ + +module "child_c" { + source = "child_c" +} diff --git a/configs/test-fixtures/config-build/child_b/child_b.tf b/configs/test-fixtures/config-build/child_b/child_b.tf new file mode 100644 index 000000000..1adcb0650 --- /dev/null +++ b/configs/test-fixtures/config-build/child_b/child_b.tf @@ -0,0 +1,7 @@ + +module "child_c" { + # In the unit test where this fixture is used, we treat the source strings + # as absolute paths rather than as source addresses as we would in a real + # module walker. + source = "child_c" +} diff --git a/configs/test-fixtures/config-build/child_c/child_c.tf b/configs/test-fixtures/config-build/child_c/child_c.tf new file mode 100644 index 000000000..16020207c --- /dev/null +++ b/configs/test-fixtures/config-build/child_c/child_c.tf @@ -0,0 +1,3 @@ +output "hello" { + value = "hello" +} diff --git a/configs/test-fixtures/config-build/root.tf b/configs/test-fixtures/config-build/root.tf new file mode 100644 index 000000000..7b3c4a75a --- /dev/null +++ b/configs/test-fixtures/config-build/root.tf @@ -0,0 +1,9 @@ + +module "child_a" { + source = "child_a" +} + +module "child_b" { + source = "child_b" +} + From 9153bb448edd83471bcc8666f1bd89235e5a3fb3 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 8 Feb 2018 18:56:48 -0800 Subject: [PATCH 14/25] configs: include the module call source range in our module tree --- configs/config.go | 12 +++++++++++- configs/config_build.go | 24 ++++++++++++++++-------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/configs/config.go b/configs/config.go index 0c07ba143..07040fa8b 100644 --- a/configs/config.go +++ b/configs/config.go @@ -35,6 +35,12 @@ type Config struct { // various elements (variables, resources, etc) defined by this module. Module *Module + // CallRange is the source range for the header of the module block that + // requested this module. + // + // This field is meaningless for the root module, where its contents are undefined. + CallRange hcl.Range + // SourceAddr is the source address that the referenced module was requested // from, as specified in configuration. // @@ -50,7 +56,11 @@ type Config struct { // Version is the specific version that was selected for this module, // based on version constraints given in configuration. // - // This field is meaningless for the root module, where its contents are undefined. + // This field is nil if the module was loaded from a non-registry source, + // since versions are not supported for other sources. + // + // This field is meaningless for the root module, where it will always + // be nil. Version *version.Version } diff --git a/configs/config_build.go b/configs/config_build.go index ecbf8bc47..0182426c1 100644 --- a/configs/config_build.go +++ b/configs/config_build.go @@ -30,11 +30,12 @@ func buildChildModules(parent *Config, walker ModuleWalker) (map[string]*Config, for _, call := range calls { req := ModuleRequest{ - Name: call.Name, - SourceAddr: call.SourceAddr, - SourceAddrRange: call.SourceAddrRange, - VersionConstraints: []VersionConstraint{call.Version}, - Parent: parent, + Name: call.Name, + SourceAddr: call.SourceAddr, + SourceAddrRange: call.SourceAddrRange, + VersionConstraint: call.Version, + Parent: parent, + CallRange: call.DeclRange, } mod, ver, modDiags := walker.LoadModule(&req) @@ -50,6 +51,7 @@ func buildChildModules(parent *Config, walker ModuleWalker) (map[string]*Config, Parent: parent, Root: parent.Root, Module: mod, + CallRange: call.DeclRange, SourceAddr: call.SourceAddr, SourceAddrRange: call.SourceAddrRange, Version: ver, @@ -110,12 +112,12 @@ type ModuleRequest struct { // to a non-existent object, etc. SourceAddrRange hcl.Range - // VersionConstraints are the constraints applied to the module in + // VersionConstraint is the version constraint applied to the module in // configuration. This data structure includes the source range for - // each constraint, which can and should be used to generate diagnostics + // the constraint, which can and should be used to generate diagnostics // about constraint-related issues, such as constraints that eliminate all // available versions of a module whose source is otherwise valid. - VersionConstraints []VersionConstraint + VersionConstraint VersionConstraint // Parent is the partially-constructed module tree node that the loaded // module will be added to. Callers may refer to any field of this @@ -124,4 +126,10 @@ type ModuleRequest struct { // The main reason this is provided is so that full module paths can // be constructed for uniqueness. Parent *Config + + // CallRange is the source range for the header of the "module" block + // in configuration that prompted this request. This can be used as the + // subject of an error diagnostic that relates to the module call itself, + // rather than to either its source address or its version number. + CallRange hcl.Range } From 72ad927c4d499fb71807d38bff4772c9fd4bd91a Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 9 Feb 2018 15:32:49 -0800 Subject: [PATCH 15/25] configs/configload: package for loading configurations Previously the behavior for loading and installing modules was included in the same package as the representation of the module tree (in the config/module package). In our new world, the model of a module tree (now called a "Config") is included in "configs" along with the Module and File structs. This new package replaces the loading and installation functionality previously in config/module with new equivalents that work with the model objects in "configs". As of this commit, only the loading functionality is implemented. The installation functionality will follow in subsequent commits. --- configs/config.go | 63 ++------- configs/config_build.go | 12 ++ configs/config_build_test.go | 2 +- configs/configload/doc.go | 4 + configs/configload/loader.go | 91 +++++++++++++ configs/configload/loader_load.go | 97 +++++++++++++ configs/configload/loader_load_test.go | 60 ++++++++ configs/configload/loader_test.go | 61 +++++++++ configs/configload/module_manifest.go | 128 ++++++++++++++++++ configs/configload/module_mgr.go | 35 +++++ .../.terraform/modules/child_a/child_a.tf | 4 + .../modules/child_a/child_c/child_c.tf | 4 + .../modules/child_b.child_d/child_d.tf | 4 + .../.terraform/modules/child_b/child_b.tf | 5 + .../.terraform/modules/modules.json | 1 + .../test-fixtures/already-installed/root.tf | 10 ++ 16 files changed, 528 insertions(+), 53 deletions(-) create mode 100644 configs/configload/doc.go create mode 100644 configs/configload/loader.go create mode 100644 configs/configload/loader_load.go create mode 100644 configs/configload/loader_load_test.go create mode 100644 configs/configload/loader_test.go create mode 100644 configs/configload/module_manifest.go create mode 100644 configs/configload/module_mgr.go create mode 100644 configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_a.tf create mode 100644 configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_c/child_c.tf create mode 100644 configs/configload/test-fixtures/already-installed/.terraform/modules/child_b.child_d/child_d.tf create mode 100644 configs/configload/test-fixtures/already-installed/.terraform/modules/child_b/child_b.tf create mode 100644 configs/configload/test-fixtures/already-installed/.terraform/modules/modules.json create mode 100644 configs/configload/test-fixtures/already-installed/root.tf diff --git a/configs/config.go b/configs/config.go index 07040fa8b..e9b23b7a7 100644 --- a/configs/config.go +++ b/configs/config.go @@ -1,8 +1,6 @@ package configs import ( - "fmt" - version "github.com/hashicorp/go-version" "github.com/hashicorp/hcl2/hcl" ) @@ -26,6 +24,17 @@ type Config struct { // this module. If this is the root module then this field is nil. Parent *Config + // Path is a sequence of module logical names that traverse from the root + // module to this config. Path is empty for the root module. + // + // This should not be used to display a path to the end-user, since + // our UI conventions call for us to return a module address string in that + // case, and a module address string ought to be built from the dynamic + // module tree (resulting from evaluating "count" and "for_each" arguments + // on our calls to produce potentially multiple child instances per call) + // rather than from our static module tree. + Path []string + // ChildModules points to the Config for each of the direct child modules // called from this module. The keys in this map match the keys in // Module.ModuleCalls. @@ -64,56 +73,6 @@ type Config struct { Version *version.Version } -// Path returns the path of logical names that lead to this Config from its -// root. -// -// This function should not be used to display a path to the end-user, since -// our UI conventions call for us to return a module address string in that -// case, and a module address string ought to be built from the dynamic -// module tree (resulting from evaluating "count" and "for_each" arguments -// on our calls to produce potentially multiple child instances per call) -// rather than from our static module tree. -// -// This function will panic if called on a config that is not part of a -// wholesome config tree, e.g. because it has incorrectly-built Children -// maps, missing node pointers, etc. However, it should work as expected -// for any tree constructed by BuildConfig and not subsequently modified. -func (c *Config) Path() []string { - // The implementation here is not especially efficient, but we don't - // care too much because module trees are shallow and narrow in all - // reasonable configurations. - - // We'll build our path in reverse here, since we're starting at the - // leafiest node, and then we'll flip it before we return. - path := make([]string, 0, c.Depth()) - - this := c - for this.Parent != nil { - parent := this.Parent - var name string - for candidate, ref := range parent.Children { - if ref == this { - name = candidate - } - } - if name == "" { - panic(fmt.Errorf( - "Config %p does not appear in the child table for its parent %p: %#v", - this, parent, parent.Children, - )) - } - path = append(path, name) - this = parent - } - - // reverse the items - for i := 0; i < len(path)/2; i++ { - j := len(path) - i - 1 - path[i], path[j] = path[j], path[i] - } - return path -} - // Depth returns the number of "hops" the receiver is from the root of its // module tree, with the root module having a depth of zero. func (c *Config) Depth() int { diff --git a/configs/config_build.go b/configs/config_build.go index 0182426c1..7aca4f1c5 100644 --- a/configs/config_build.go +++ b/configs/config_build.go @@ -29,8 +29,13 @@ func buildChildModules(parent *Config, walker ModuleWalker) (map[string]*Config, calls := parent.Module.ModuleCalls for _, call := range calls { + path := make([]string, len(parent.Path)+1) + copy(path, parent.Path) + path[len(path)-1] = call.Name + req := ModuleRequest{ Name: call.Name, + Path: path, SourceAddr: call.SourceAddr, SourceAddrRange: call.SourceAddrRange, VersionConstraint: call.Version, @@ -50,6 +55,7 @@ func buildChildModules(parent *Config, walker ModuleWalker) (map[string]*Config, child := &Config{ Parent: parent, Root: parent.Root, + Path: path, Module: mod, CallRange: call.DeclRange, SourceAddr: call.SourceAddr, @@ -102,6 +108,12 @@ type ModuleRequest struct { // HCL identifier and UTF-8 encoded. Name string + // Path is a list of logical names that traverse from the root module to + // this module. This can be used, for example, to form a lookup key for + // each distinct module call in a configuration, allowing for multiple + // calls with the same name at different points in the tree. + Path []string + // SourceAddr is the source address string provided by the user in // configuration. SourceAddr string diff --git a/configs/config_build_test.go b/configs/config_build_test.go index 3ac46e35c..1e409d846 100644 --- a/configs/config_build_test.go +++ b/configs/config_build_test.go @@ -44,7 +44,7 @@ func TestBuildConfig(t *testing.T) { var got []string cfg.DeepEach(func(c *Config) { - got = append(got, fmt.Sprintf("%s %s", strings.Join(c.Path(), "."), c.Version)) + got = append(got, fmt.Sprintf("%s %s", strings.Join(c.Path, "."), c.Version)) }) sort.Strings(got) want := []string{ diff --git a/configs/configload/doc.go b/configs/configload/doc.go new file mode 100644 index 000000000..8b615f902 --- /dev/null +++ b/configs/configload/doc.go @@ -0,0 +1,4 @@ +// Package configload knows how to install modules into the .terraform/modules +// directory and to load modules from those installed locations. It is used +// in conjunction with the LoadConfig function in the parent package. +package configload diff --git a/configs/configload/loader.go b/configs/configload/loader.go new file mode 100644 index 000000000..fba244173 --- /dev/null +++ b/configs/configload/loader.go @@ -0,0 +1,91 @@ +package configload + +import ( + "fmt" + + "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry" + "github.com/hashicorp/terraform/svchost/auth" + "github.com/hashicorp/terraform/svchost/disco" + "github.com/spf13/afero" +) + +// A Loader instance is the main entry-point for loading configurations via +// this package. +// +// It extends the general config-loading functionality in the parent package +// "configs" to support installation of modules from remote sources and +// loading full configurations using modules that were previously installed. +type Loader struct { + // parser is used to read configuration + parser *configs.Parser + + // modules is used to install and locate descendent modules that are + // referenced (directly or indirectly) from the root module. + modules moduleMgr +} + +// Config is used with NewLoader to specify configuration arguments for the +// loader. +type Config struct { + // ModulesDir is a path to a directory where descendent modules are + // (or should be) installed. (This is usually the + // .terraform/modules directory, in the common case where this package + // is being loaded from the main Terraform CLI package.) + ModulesDir string + + // Services is the service discovery client to use when locating remote + // module registry endpoints. If this is nil then registry sources are + // not supported, which should be true only in specialized circumstances + // such as in tests. + Services *disco.Disco + + // Creds is a credentials store for communicating with remote module + // registry endpoints. If this is nil then no credentials will be used. + Creds auth.CredentialsSource +} + +// NewLoader creates and returns a loader that reads configuration from the +// real OS filesystem. +// +// The loader has some internal state about the modules that are currently +// installed, which is read from disk as part of this function. If that +// manifest cannot be read then an error will be returned. +func NewLoader(config *Config) (*Loader, error) { + fs := afero.NewOsFs() + parser := configs.NewParser(fs) + reg := registry.NewClient(config.Services, config.Creds, nil) + + ret := &Loader{ + parser: parser, + modules: moduleMgr{ + FS: afero.Afero{fs}, + Dir: config.ModulesDir, + Services: config.Services, + Creds: config.Creds, + Registry: reg, + }, + } + + err := ret.modules.readModuleManifestSnapshot() + if err != nil { + return nil, fmt.Errorf("failed to read module manifest: %s", err) + } + + return ret, nil +} + +// Parser returns the underlying parser for this loader. +// +// This is useful for loading other sorts of files than the module directories +// that a loader deals with, since then they will share the source code cache +// for this loader and can thus be shown as snippets in diagnostic messages. +func (l *Loader) Parser() *configs.Parser { + return l.parser +} + +// Sources returns the source code cache for the underlying parser of this +// loader. This is a shorthand for l.Parser().Sources(). +func (l *Loader) Sources() map[string][]byte { + return l.parser.Sources() +} diff --git a/configs/configload/loader_load.go b/configs/configload/loader_load.go new file mode 100644 index 000000000..104a31d20 --- /dev/null +++ b/configs/configload/loader_load.go @@ -0,0 +1,97 @@ +package configload + +import ( + "fmt" + + version "github.com/hashicorp/go-version" + "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/terraform/configs" +) + +// LoadConfig reads the Terraform module in the given directory and uses it as the +// root module to build the static module tree that represents a configuration, +// assuming that all required descendent modules have already been installed. +// +// If error diagnostics are returned, the returned configuration may be either +// nil or incomplete. In the latter case, cautious static analysis is possible +// in spite of the errors. +// +// LoadConfig performs the basic syntax and uniqueness validations that are +// required to process the individual modules, and also detects +func (l *Loader) LoadConfig(rootDir string) (*configs.Config, hcl.Diagnostics) { + rootMod, diags := l.parser.LoadConfigDir(rootDir) + if rootMod == nil { + return nil, diags + } + + cfg, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc(l.moduleWalkerLoad)) + diags = append(diags, cDiags...) + + return cfg, diags +} + +// moduleWalkerLoad is a configs.ModuleWalkerFunc for loading modules that +// are presumed to have already been installed. A different function +// (moduleWalkerInstall) is used for installation. +func (l *Loader) moduleWalkerLoad(req *configs.ModuleRequest) (*configs.Module, *version.Version, hcl.Diagnostics) { + // Since we're just loading here, we expect that all referenced modules + // will be already installed and described in our manifest. However, we + // do verify that the manifest and the configuration are in agreement + // so that we can prompt the user to run "terraform init" if not. + + key := manifestKey(req.Path) + record, exists := l.modules.manifest[key] + + if !exists { + return nil, nil, hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: "Module not installed", + Detail: "This module is not yet installed. Run \"terraform init\" to install all modules required by this configuration.", + Subject: &req.CallRange, + }, + } + } + + var diags hcl.Diagnostics + + // Check for inconsistencies between manifest and config + if req.SourceAddr != record.SourceAddr { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Module source has changed", + Detail: "The source address was changed since this module was installed. Run \"terraform init\" to install all modules required by this configuration.", + Subject: &req.SourceAddrRange, + }) + } + if !req.VersionConstraint.Required.Check(record.Version) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Module version requirements have changed", + Detail: fmt.Sprintf( + "The version requirements have changed since this module was installed and the installed version (%s) is no longer acceptable. Run \"terraform init\" to install all modules required by this configuration.", + record.Version, + ), + Subject: &req.SourceAddrRange, + }) + } + + mod, mDiags := l.parser.LoadConfigDir(record.Dir) + diags = append(diags, mDiags...) + if mod == nil { + // nil specifically indicates that the directory does not exist or + // cannot be read, so in this case we'll discard any generic diagnostics + // returned from LoadConfigDir and produce our own context-sensitive + // error message. + return nil, nil, hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: "Module not installed", + Detail: fmt.Sprintf("This module's local cache directory %s could not be read. Run \"terraform init\" to install all modules required by this configuration.", record.Dir), + Subject: &req.CallRange, + }, + } + } + + return mod, record.Version, diags +} diff --git a/configs/configload/loader_load_test.go b/configs/configload/loader_load_test.go new file mode 100644 index 000000000..6c3a0a940 --- /dev/null +++ b/configs/configload/loader_load_test.go @@ -0,0 +1,60 @@ +package configload + +import ( + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/zclconf/go-cty/cty" + + "github.com/hashicorp/terraform/configs" +) + +func TestLoaderLoadConfig_okay(t *testing.T) { + fixtureDir := filepath.Clean("test-fixtures/already-installed") + loader, err := NewLoader(&Config{ + ModulesDir: filepath.Join(fixtureDir, ".terraform/modules"), + }) + if err != nil { + t.Fatalf("unexpected error from NewLoader: %s", err) + } + + cfg, diags := loader.LoadConfig(fixtureDir) + assertNoDiagnostics(t, diags) + if cfg == nil { + t.Fatalf("config is nil; want non-nil") + } + + var gotPaths []string + cfg.DeepEach(func(c *configs.Config) { + gotPaths = append(gotPaths, strings.Join(c.Path, ".")) + }) + sort.Strings(gotPaths) + wantPaths := []string{ + "", // root module + "child_a", + "child_a.child_c", + "child_b", + "child_b.child_d", + } + + if !reflect.DeepEqual(gotPaths, wantPaths) { + t.Fatalf("wrong module paths\ngot: %swant %s", spew.Sdump(gotPaths), spew.Sdump(wantPaths)) + } + + t.Run("child_a.child_c output", func(t *testing.T) { + output := cfg.Children["child_a"].Children["child_c"].Module.Outputs["hello"] + got, diags := output.Expr.Value(nil) + assertNoDiagnostics(t, diags) + assertResultCtyEqual(t, got, cty.StringVal("Hello from child_c")) + }) + t.Run("child_b.child_d output", func(t *testing.T) { + output := cfg.Children["child_b"].Children["child_d"].Module.Outputs["hello"] + got, diags := output.Expr.Value(nil) + assertNoDiagnostics(t, diags) + assertResultCtyEqual(t, got, cty.StringVal("Hello from child_d")) + }) +} diff --git a/configs/configload/loader_test.go b/configs/configload/loader_test.go new file mode 100644 index 000000000..512b99f3f --- /dev/null +++ b/configs/configload/loader_test.go @@ -0,0 +1,61 @@ +package configload + +import ( + "reflect" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/hashicorp/hcl2/hcl" + "github.com/zclconf/go-cty/cty" +) + +func assertNoDiagnostics(t *testing.T, diags hcl.Diagnostics) bool { + t.Helper() + return assertDiagnosticCount(t, diags, 0) +} + +func assertDiagnosticCount(t *testing.T, diags hcl.Diagnostics, want int) bool { + t.Helper() + if len(diags) != 0 { + t.Errorf("wrong number of diagnostics %d; want %d", len(diags), want) + for _, diag := range diags { + t.Logf("- %s", diag) + } + return true + } + return false +} + +func assertDiagnosticSummary(t *testing.T, diags hcl.Diagnostics, want string) bool { + t.Helper() + + for _, diag := range diags { + if diag.Summary == want { + return false + } + } + + t.Errorf("missing diagnostic summary %q", want) + for _, diag := range diags { + t.Logf("- %s", diag) + } + return true +} + +func assertResultDeepEqual(t *testing.T, got, want interface{}) bool { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Errorf("wrong result\ngot: %swant: %s", spew.Sdump(got), spew.Sdump(want)) + return true + } + return false +} + +func assertResultCtyEqual(t *testing.T, got, want cty.Value) bool { + t.Helper() + if !got.RawEquals(want) { + t.Errorf("wrong result\ngot: %#v\nwant: %#v", got, want) + return true + } + return false +} diff --git a/configs/configload/module_manifest.go b/configs/configload/module_manifest.go new file mode 100644 index 000000000..8ecc720a7 --- /dev/null +++ b/configs/configload/module_manifest.go @@ -0,0 +1,128 @@ +package configload + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + version "github.com/hashicorp/go-version" +) + +// moduleRecord represents some metadata about an installed module, as part +// of a moduleManifest. +type moduleRecord struct { + // Key is a unique identifier for this particular module, based on its + // position within the static module tree. + Key string `json:"Key"` + + // SourceAddr is the source address given for this module in configuration. + // This is used only to detect if the source was changed in configuration + // since the module was last installed, which means that the installer + // must re-install it. + SourceAddr string `json:"Source"` + + // Version is the exact version of the module, which results from parsing + // VersionStr. nil for un-versioned modules. + Version *version.Version `json:"-"` + + // VersionStr is the version specifier string. This is used only for + // serialization in snapshots and should not be accessed or updated + // by any other codepaths; use "Version" instead. + VersionStr string `json:"Version"` + + // Dir is the path to the local directory where the module is installed. + Dir string `json:"Dir"` +} + +// moduleManifest is a map used to keep track of the filesystem locations +// and other metadata about installed modules. +// +// The configuration loader refers to this, while the module installer updates +// it to reflect any changes to the installed modules. +type moduleManifest map[string]moduleRecord + +func manifestKey(path []string) string { + return strings.Join(path, ".") +} + +// manifestSnapshotFile is an internal struct used only to assist in our JSON +// serializtion of manifest snapshots. It should not be used for any other +// purposes. +type manifestSnapshotFile struct { + Records []moduleRecord `json:"Modules"` +} + +const manifestFilename = "modules.json" + +func (m *moduleMgr) manifestSnapshotPath() string { + return filepath.Join(m.Dir, manifestFilename) +} + +// readModuleManifestSnapshot loads a manifest snapshot from the filesystem. +func (m *moduleMgr) readModuleManifestSnapshot() error { + src, err := m.FS.ReadFile(m.manifestSnapshotPath()) + if err != nil { + if os.IsNotExist(err) { + // We'll treat a missing file as an empty manifest + m.manifest = make(moduleManifest) + return nil + } + return err + } + + if len(src) == 0 { + // This should never happen, but we'll tolerate it as if it were + // a valid empty JSON object. + m.manifest = make(moduleManifest) + return nil + } + + var read manifestSnapshotFile + err = json.Unmarshal(src, &read) + + new := make(moduleManifest) + for _, record := range read.Records { + if record.VersionStr != "" { + record.Version, err = version.NewVersion(record.VersionStr) + if err != nil { + return fmt.Errorf("invalid version %q for %s: %s", record.VersionStr, record.Key, err) + } + } + if _, exists := new[record.Key]; exists { + // This should never happen in any valid file, so we'll catch it + // and report it to avoid confusing/undefined behavior if the + // snapshot file was edited incorrectly outside of Terraform. + return fmt.Errorf("snapshot file contains two records for path %s", record.Key) + } + new[record.Key] = record + } + + m.manifest = new + + return nil +} + +// writeModuleManifestSnapshot writes a snapshot of the current manifest +// to the filesystem. +// +// The caller must guarantee no concurrent modifications of the manifest for +// the duration of a call to this function, or the behavior is undefined. +func (m *moduleMgr) writeModuleManifestSnapshot() error { + var write manifestSnapshotFile + + for _, record := range m.manifest { + // Make sure VersionStr is in sync with Version, since we encourage + // callers to manipulate Version and ignore VersionStr. + record.VersionStr = record.Version.String() + write.Records = append(write.Records, record) + } + + src, err := json.Marshal(write) + if err != nil { + return err + } + + return m.FS.WriteFile(m.manifestSnapshotPath(), src, os.ModePerm) +} diff --git a/configs/configload/module_mgr.go b/configs/configload/module_mgr.go new file mode 100644 index 000000000..5856c2ec0 --- /dev/null +++ b/configs/configload/module_mgr.go @@ -0,0 +1,35 @@ +package configload + +import ( + "github.com/hashicorp/terraform/registry" + "github.com/hashicorp/terraform/svchost/auth" + "github.com/hashicorp/terraform/svchost/disco" + "github.com/spf13/afero" +) + +type moduleMgr struct { + FS afero.Afero + + // Dir is the path where descendent modules are (or will be) installed. + Dir string + + // Services is a service discovery client that will be used to find + // remote module registry endpoints. This object may be pre-loaded with + // cached discovery information. + Services *disco.Disco + + // Creds provides optional credentials for communicating with service hosts. + Creds auth.CredentialsSource + + // Registry is a client for the module registry protocol, which is used + // when a module is requested from a registry source. + Registry *registry.Client + + // manifest tracks the currently-installed modules for this manager. + // + // The loader may read this. Only the installer may write to it, and + // after a set of updates are completed the installer must call + // writeModuleManifestSnapshot to persist a snapshot of the manifest + // to disk for use on subsequent runs. + manifest moduleManifest +} diff --git a/configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_a.tf b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_a.tf new file mode 100644 index 000000000..2f4d0f1a0 --- /dev/null +++ b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_a.tf @@ -0,0 +1,4 @@ + +module "child_c" { + source = "./child_c" +} diff --git a/configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_c/child_c.tf b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_c/child_c.tf new file mode 100644 index 000000000..785d98d98 --- /dev/null +++ b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_c/child_c.tf @@ -0,0 +1,4 @@ + +output "hello" { + value = "Hello from child_c" +} diff --git a/configs/configload/test-fixtures/already-installed/.terraform/modules/child_b.child_d/child_d.tf b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_b.child_d/child_d.tf new file mode 100644 index 000000000..145576a36 --- /dev/null +++ b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_b.child_d/child_d.tf @@ -0,0 +1,4 @@ + +output "hello" { + value = "Hello from child_d" +} diff --git a/configs/configload/test-fixtures/already-installed/.terraform/modules/child_b/child_b.tf b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_b/child_b.tf new file mode 100644 index 000000000..4a1b247d3 --- /dev/null +++ b/configs/configload/test-fixtures/already-installed/.terraform/modules/child_b/child_b.tf @@ -0,0 +1,5 @@ + +module "child_d" { + source = "example.com/foo/bar_d/baz" + # Intentionally no version here +} diff --git a/configs/configload/test-fixtures/already-installed/.terraform/modules/modules.json b/configs/configload/test-fixtures/already-installed/.terraform/modules/modules.json new file mode 100644 index 000000000..454e719ab --- /dev/null +++ b/configs/configload/test-fixtures/already-installed/.terraform/modules/modules.json @@ -0,0 +1 @@ +{"Modules":[{"Key":"","Source":"","Dir":"test-fixtures/already-installed"},{"Key":"child_a","Source":"example.com/foo/bar_a/baz","Version":"1.0.1","Dir":"test-fixtures/already-installed/.terraform/modules/child_a"},{"Key":"child_b","Source":"example.com/foo/bar_b/baz","Version":"1.0.0","Dir":"test-fixtures/already-installed/.terraform/modules/child_b"},{"Key":"child_a.child_c","Source":"./child_c","Dir":"test-fixtures/already-installed/.terraform/modules/child_a/child_c"},{"Key":"child_b.child_d","Source":"example.com/foo/bar_d/baz","Version":"1.2.0","Dir":"test-fixtures/already-installed/.terraform/modules/child_b.child_d"}]} \ No newline at end of file diff --git a/configs/configload/test-fixtures/already-installed/root.tf b/configs/configload/test-fixtures/already-installed/root.tf new file mode 100644 index 000000000..8a4473942 --- /dev/null +++ b/configs/configload/test-fixtures/already-installed/root.tf @@ -0,0 +1,10 @@ + +module "child_a" { + source = "example.com/foo/bar_a/baz" + version = ">= 1.0.0" +} + +module "child_b" { + source = "example.com/foo/bar_b/baz" + version = ">= 1.0.0" +} From 7feef98517b80238e8fa7ba734d05e63ea63c901 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 13 Feb 2018 14:40:53 -0800 Subject: [PATCH 16/25] configs/configload: installation of local modules Enough of the InstallModules method to install local modules (those with relative paths). "Install" is actually a bit of an exaggeration for these since we actually just record them in our manifest after verifying that the source directory exists. This is a change of behavior relative to the old module installer since we no longer create a symlink to the module directory inside the .terraform/modules directory. Instead, we record the module's true location in our manifest so that the loader will find it later. The use of a symlink here predated the manifest file. Now that we have a manifest file the symlinks are redundant. Using the "natural" location of the module leads to more helpful error messages, since we'll refer to the module path as the user expects it, rather than to an internal alias. --- configs/configload/loader_install.go | 237 ++++++++++++++++++ configs/configload/loader_install_hooks.go | 34 +++ configs/configload/loader_install_test.go | 65 +++++ configs/configload/loader_test.go | 31 +++ configs/configload/module_manifest.go | 8 +- configs/configload/source_addr.go | 28 +++ .../local-modules/child_a/child_a.tf | 4 + .../local-modules/child_a/child_b/child_b.tf | 4 + .../test-fixtures/local-modules/root.tf | 4 + 9 files changed, 413 insertions(+), 2 deletions(-) create mode 100644 configs/configload/loader_install.go create mode 100644 configs/configload/loader_install_hooks.go create mode 100644 configs/configload/loader_install_test.go create mode 100644 configs/configload/source_addr.go create mode 100644 configs/configload/test-fixtures/local-modules/child_a/child_a.tf create mode 100644 configs/configload/test-fixtures/local-modules/child_a/child_b/child_b.tf create mode 100644 configs/configload/test-fixtures/local-modules/root.tf diff --git a/configs/configload/loader_install.go b/configs/configload/loader_install.go new file mode 100644 index 000000000..2ab6529d3 --- /dev/null +++ b/configs/configload/loader_install.go @@ -0,0 +1,237 @@ +package configload + +import ( + "fmt" + "log" + "os" + "path/filepath" + "strings" + + version "github.com/hashicorp/go-version" + "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry/regsrc" +) + +// InstallModules analyses the root module in the given directory and installs +// all of its direct and transitive dependencies into the loader's modules +// directory, which must already exist. +// +// Since InstallModules makes possibly-time-consuming calls to remote services, +// a hook interface is supported to allow the caller to be notified when +// each module is installed and, for remote modules, when downloading begins. +// LoadConfig guarantees that two hook calls will not happen concurrently but +// it does not guarantee any particular ordering of hook calls. This mechanism +// is for UI feedback only and does not give the caller any control over the +// process. +// +// If modules are already installed in the target directory, they will be +// skipped unless their source address or version have changed or unless +// the upgrade flag is set. +// +// InstallModules never deletes any directory, except in the case where it +// needs to replace a directory that is already present with a newly-extracted +// package. +// +// If the returned diagnostics contains errors then the module installation +// may have wholly or partially completed. Modules must be loaded in order +// to find their dependencies, so this function does many of the same checks +// as LoadConfig as a side-effect. +func (l *Loader) InstallModules(rootDir string, upgrade bool, hooks InstallHooks) hcl.Diagnostics { + rootMod, diags := l.parser.LoadConfigDir(rootDir) + if rootMod == nil { + return diags + } + + if hooks == nil { + // Use our no-op implementation as a placeholder + hooks = InstallHooksImpl{} + } + + // Create a manifest record for the root module. This will be used if + // there are any relative-pathed modules in the root. + l.modules.manifest[""] = moduleRecord{ + Key: "", + Dir: rootDir, + } + + _, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc( + func(req *configs.ModuleRequest) (*configs.Module, *version.Version, hcl.Diagnostics) { + + key := manifestKey(req.Path) + instPath := l.packageInstallPath(req.Path) + + log.Printf("[DEBUG] Module installer: begin %s", key) + + // First we'll check if we need to upgrade/replace an existing + // installed module, and delete it out of the way if so. + replace := upgrade + if !replace { + record, recorded := l.modules.manifest[key] + switch { + case !recorded: + log.Printf("[TRACE] %s is not yet installed", key) + replace = true + case record.SourceAddr != req.SourceAddr: + log.Printf("[TRACE] %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr) + replace = true + case record.Version != nil && !req.VersionConstraint.Required.Check(record.Version): + log.Printf("[TRACE] %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraint.Required) + replace = true + } + } + + // If we _are_ planning to replace this module, then we'll remove + // it now so our installation code below won't conflict with any + // existing remnants. + if replace { + if _, recorded := l.modules.manifest[key]; recorded { + log.Printf("[TRACE] discarding previous record of %s prior to reinstall", key) + } + delete(l.modules.manifest, key) + // Deleting a module invalidates all of its descendent modules too. + keyPrefix := key + "." + for subKey := range l.modules.manifest { + if strings.HasPrefix(subKey, keyPrefix) { + if _, recorded := l.modules.manifest[subKey]; recorded { + log.Printf("[TRACE] also discarding downstream %s", subKey) + } + delete(l.modules.manifest, subKey) + } + } + } + + record, recorded := l.modules.manifest[key] + if !recorded { + // Clean up any stale cache directory that might be present. + // If this is a local (relative) source then the dir will + // not exist, but we'll ignore that. + log.Printf("[TRACE] cleaning directory %s prior to install of %s", instPath, key) + err := l.modules.FS.RemoveAll(instPath) + if err != nil && !os.IsNotExist(err) { + log.Printf("[TRACE] failed to remove %s: %s", key, err) + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to remove local module cache", + Detail: fmt.Sprintf( + "Terraform tried to remove %s in order to reinstall this module, but encountered an error: %s", + instPath, err, + ), + Subject: &req.CallRange, + }) + return nil, nil, diags + } + } else { + // If this module is already recorded and its root directory + // exists then we will just load what's already there and + // keep our existing record. + info, err := l.modules.FS.Stat(record.Dir) + if err == nil && info.IsDir() { + mod, mDiags := l.parser.LoadConfigDir(record.Dir) + diags = append(diags, mDiags...) + + log.Printf("[TRACE] Module installer: %s %s already installed in %s", key, record.Version, record.Dir) + return mod, record.Version, diags + } + } + + // If we get down here then it's finally time to actually install + // the module. There are some variants to this process depending + // on what type of module source address we have. + switch { + + case isLocalSourceAddr(req.SourceAddr): + log.Printf("[TRACE] %s has local path %q", key, req.SourceAddr) + mod, mDiags := l.installLocalModule(req, key, hooks) + diags = append(diags, mDiags...) + return mod, nil, diags + + case isRegistrySourceAddr(req.SourceAddr): + addr, err := regsrc.ParseModuleSource(req.SourceAddr) + if err != nil { + // Should never happen because isRegistrySourceAddr already validated + panic(err) + } + log.Printf("[TRACE] %s is a registry module at %s", key, addr) + + // TODO: Implement + panic("registry source installation not yet implemented") + + default: + log.Printf("[TRACE] %s address %q will be interpreted with go-getter", key, req.SourceAddr) + + // TODO: Implement + panic("fallback source installation not yet implemented") + + } + + }, + )) + diags = append(diags, cDiags...) + + err := l.modules.writeModuleManifestSnapshot() + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to update module manifest", + Detail: fmt.Sprintf("Unable to write the module manifest file: %s", err), + }) + } + + return diags +} + +func (l *Loader) installLocalModule(req *configs.ModuleRequest, key string, hooks InstallHooks) (*configs.Module, hcl.Diagnostics) { + var diags hcl.Diagnostics + + parentKey := manifestKey(req.Parent.Path) + parentRecord, recorded := l.modules.manifest[parentKey] + if !recorded { + // This is indicative of a bug rather than a user-actionable error + panic(fmt.Errorf("missing manifest record for parent module %s", parentKey)) + } + + if len(req.VersionConstraint.Required) != 0 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid version constraint", + Detail: "A version constraint cannot be applied to a module at a relative local path.", + Subject: &req.VersionConstraint.DeclRange, + }) + } + + // For local sources we don't actually need to modify the + // filesystem at all because the parent already wrote + // the files we need, and so we just load up what's already here. + newDir := filepath.Join(parentRecord.Dir, req.SourceAddr) + log.Printf("[TRACE] %s uses directory from parent: %s", key, newDir) + mod, mDiags := l.parser.LoadConfigDir(newDir) + if mod == nil { + // nil indicates missing or unreadable directory, so we'll + // discard the returned diags and return a more specific + // error message here. + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unreadable module directory", + Detail: fmt.Sprintf("The directory %s could not be read.", newDir), + Subject: &req.SourceAddrRange, + }) + } else { + diags = append(diags, mDiags...) + } + + // Note the local location in our manifest. + l.modules.manifest[key] = moduleRecord{ + Key: key, + Dir: newDir, + SourceAddr: req.SourceAddr, + } + log.Printf("[TRACE] Module installer: %s installed at %s", key, newDir) + hooks.Install(key, nil, newDir) + + return mod, diags +} + +func (l *Loader) packageInstallPath(modulePath []string) string { + return filepath.Join(l.modules.Dir, strings.Join(modulePath, ".")) +} diff --git a/configs/configload/loader_install_hooks.go b/configs/configload/loader_install_hooks.go new file mode 100644 index 000000000..058369414 --- /dev/null +++ b/configs/configload/loader_install_hooks.go @@ -0,0 +1,34 @@ +package configload + +import version "github.com/hashicorp/go-version" + +// InstallHooks is an interface used to provide notifications about the +// installation process being orchestrated by InstallModules. +// +// This interface may have new methods added in future, so implementers should +// embed InstallHooksImpl to get no-op implementations of any unimplemented +// methods. +type InstallHooks interface { + // Download is called for modules that are retrieved from a remote source + // before that download begins, to allow a caller to give feedback + // on progress through a possibly-long sequence of downloads. + Download(moduleAddr, packageAddr string, version *version.Version) + + // Install is called for each module that is installed, even if it did + // not need to be downloaded from a remote source. + Install(moduleAddr string, version *version.Version, localPath string) +} + +// InstallHooksImpl is a do-nothing implementation of InstallHooks that +// can be embedded in another implementation struct to allow only partial +// implementation of the interface. +type InstallHooksImpl struct { +} + +func (h InstallHooksImpl) Download(moduleAddr, packageAddr string, version *version.Version) { +} + +func (h InstallHooksImpl) Install(moduleAddr string, version *version.Version, localPath string) { +} + +var _ InstallHooks = InstallHooksImpl{} diff --git a/configs/configload/loader_install_test.go b/configs/configload/loader_install_test.go new file mode 100644 index 000000000..608b08a00 --- /dev/null +++ b/configs/configload/loader_install_test.go @@ -0,0 +1,65 @@ +package configload + +import ( + "path/filepath" + "testing" + + version "github.com/hashicorp/go-version" +) + +func TestLoaderInstallModules_local(t *testing.T) { + fixtureDir := filepath.Clean("test-fixtures/local-modules") + loader := newTestLoader(filepath.Join(fixtureDir, ".terraform/modules")) + + hooks := &testInstallHooks{} + + diags := loader.InstallModules(fixtureDir, false, hooks) + assertNoDiagnostics(t, diags) + + wantCalls := []testInstallHookCall{ + { + Name: "Install", + ModuleAddr: "child_a", + PackageAddr: "", + LocalPath: "test-fixtures/local-modules/child_a", + }, + { + Name: "Install", + ModuleAddr: "child_a.child_b", + PackageAddr: "", + LocalPath: "test-fixtures/local-modules/child_a/child_b", + }, + } + + assertResultDeepEqual(t, hooks.Calls, wantCalls) +} + +type testInstallHooks struct { + Calls []testInstallHookCall +} + +type testInstallHookCall struct { + Name string + ModuleAddr string + PackageAddr string + Version *version.Version + LocalPath string +} + +func (h *testInstallHooks) Download(moduleAddr, packageAddr string, version *version.Version) { + h.Calls = append(h.Calls, testInstallHookCall{ + Name: "Download", + ModuleAddr: moduleAddr, + PackageAddr: packageAddr, + Version: version, + }) +} + +func (h *testInstallHooks) Install(moduleAddr string, version *version.Version, localPath string) { + h.Calls = append(h.Calls, testInstallHookCall{ + Name: "Install", + ModuleAddr: moduleAddr, + Version: version, + LocalPath: localPath, + }) +} diff --git a/configs/configload/loader_test.go b/configs/configload/loader_test.go index 512b99f3f..6ab1fc8c5 100644 --- a/configs/configload/loader_test.go +++ b/configs/configload/loader_test.go @@ -4,11 +4,42 @@ import ( "reflect" "testing" + "github.com/spf13/afero" + "github.com/davecgh/go-spew/spew" "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry" "github.com/zclconf/go-cty/cty" ) +// newTestLoader is like NewLoader but it uses a copy-on-write overlay filesystem +// over the real filesystem so that any files that are created cannot persist +// between test runs. +// +// It will also panic if there are any errors creating the loader, since +// these should never happen in a testing scenario. +func newTestLoader(dir string) *Loader { + realFS := afero.NewOsFs() + overlayFS := afero.NewMemMapFs() + fs := afero.NewCopyOnWriteFs(realFS, overlayFS) + parser := configs.NewParser(fs) + reg := registry.NewClient(nil, nil, nil) + ret := &Loader{ + parser: parser, + modules: moduleMgr{ + FS: afero.Afero{fs}, + Dir: dir, + Registry: reg, + }, + } + err := ret.modules.readModuleManifestSnapshot() + if err != nil { + panic(err) + } + return ret +} + func assertNoDiagnostics(t *testing.T, diags hcl.Diagnostics) bool { t.Helper() return assertDiagnosticCount(t, diags, 0) diff --git a/configs/configload/module_manifest.go b/configs/configload/module_manifest.go index 8ecc720a7..777f3a282 100644 --- a/configs/configload/module_manifest.go +++ b/configs/configload/module_manifest.go @@ -30,7 +30,7 @@ type moduleRecord struct { // VersionStr is the version specifier string. This is used only for // serialization in snapshots and should not be accessed or updated // by any other codepaths; use "Version" instead. - VersionStr string `json:"Version"` + VersionStr string `json:"Version,omitempty"` // Dir is the path to the local directory where the module is installed. Dir string `json:"Dir"` @@ -115,7 +115,11 @@ func (m *moduleMgr) writeModuleManifestSnapshot() error { for _, record := range m.manifest { // Make sure VersionStr is in sync with Version, since we encourage // callers to manipulate Version and ignore VersionStr. - record.VersionStr = record.Version.String() + if record.Version != nil { + record.VersionStr = record.Version.String() + } else { + record.VersionStr = "" + } write.Records = append(write.Records, record) } diff --git a/configs/configload/source_addr.go b/configs/configload/source_addr.go new file mode 100644 index 000000000..7767859c5 --- /dev/null +++ b/configs/configload/source_addr.go @@ -0,0 +1,28 @@ +package configload + +import ( + "strings" + + "github.com/hashicorp/terraform/registry/regsrc" +) + +var localSourcePrefixes = []string{ + "./", + "../", + ".\\", + "..\\", +} + +func isLocalSourceAddr(addr string) bool { + for _, prefix := range localSourcePrefixes { + if strings.HasPrefix(addr, prefix) { + return true + } + } + return false +} + +func isRegistrySourceAddr(addr string) bool { + _, err := regsrc.ParseModuleSource(addr) + return err == nil +} diff --git a/configs/configload/test-fixtures/local-modules/child_a/child_a.tf b/configs/configload/test-fixtures/local-modules/child_a/child_a.tf new file mode 100644 index 000000000..eb2c0044f --- /dev/null +++ b/configs/configload/test-fixtures/local-modules/child_a/child_a.tf @@ -0,0 +1,4 @@ + +module "child_b" { + source = "./child_b" +} diff --git a/configs/configload/test-fixtures/local-modules/child_a/child_b/child_b.tf b/configs/configload/test-fixtures/local-modules/child_a/child_b/child_b.tf new file mode 100644 index 000000000..4900e9796 --- /dev/null +++ b/configs/configload/test-fixtures/local-modules/child_a/child_b/child_b.tf @@ -0,0 +1,4 @@ + +output "hello" { + value = "Hello from child_b!" +} diff --git a/configs/configload/test-fixtures/local-modules/root.tf b/configs/configload/test-fixtures/local-modules/root.tf new file mode 100644 index 000000000..1a6721b5d --- /dev/null +++ b/configs/configload/test-fixtures/local-modules/root.tf @@ -0,0 +1,4 @@ + +module "child_a" { + source = "./child_a" +} From 51e5f7a56b098aaa8a53d61e2f12a1546bf22709 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 14 Feb 2018 12:37:30 -0800 Subject: [PATCH 17/25] govendor fetch github.com/go-test/deep This will be used to produce readable diffs for deep data structures in unit tests. --- vendor/github.com/go-test/deep/CHANGES.md | 9 + vendor/github.com/go-test/deep/LICENSE | 21 ++ vendor/github.com/go-test/deep/README.md | 51 ++++ vendor/github.com/go-test/deep/deep.go | 352 ++++++++++++++++++++++ vendor/vendor.json | 6 + 5 files changed, 439 insertions(+) create mode 100644 vendor/github.com/go-test/deep/CHANGES.md create mode 100644 vendor/github.com/go-test/deep/LICENSE create mode 100644 vendor/github.com/go-test/deep/README.md create mode 100644 vendor/github.com/go-test/deep/deep.go diff --git a/vendor/github.com/go-test/deep/CHANGES.md b/vendor/github.com/go-test/deep/CHANGES.md new file mode 100644 index 000000000..4351819d6 --- /dev/null +++ b/vendor/github.com/go-test/deep/CHANGES.md @@ -0,0 +1,9 @@ +# go-test/deep Changelog + +## v1.0.1 released 2018-01-28 + +* Fixed #12: Arrays are not properly compared (samlitowitz) + +## v1.0.0 releaesd 2017-10-27 + +* First release diff --git a/vendor/github.com/go-test/deep/LICENSE b/vendor/github.com/go-test/deep/LICENSE new file mode 100644 index 000000000..228ef16f7 --- /dev/null +++ b/vendor/github.com/go-test/deep/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright 2015-2017 Daniel Nichter + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/go-test/deep/README.md b/vendor/github.com/go-test/deep/README.md new file mode 100644 index 000000000..3b78eac7c --- /dev/null +++ b/vendor/github.com/go-test/deep/README.md @@ -0,0 +1,51 @@ +# Deep Variable Equality for Humans + +[![Go Report Card](https://goreportcard.com/badge/github.com/go-test/deep)](https://goreportcard.com/report/github.com/go-test/deep) [![Build Status](https://travis-ci.org/go-test/deep.svg?branch=master)](https://travis-ci.org/go-test/deep) [![Coverage Status](https://coveralls.io/repos/github/go-test/deep/badge.svg?branch=master)](https://coveralls.io/github/go-test/deep?branch=master) [![GoDoc](https://godoc.org/github.com/go-test/deep?status.svg)](https://godoc.org/github.com/go-test/deep) + +This package provides a single function: `deep.Equal`. It's like [reflect.DeepEqual](http://golang.org/pkg/reflect/#DeepEqual) but much friendlier to humans (or any sentient being) for two reason: + +* `deep.Equal` returns a list of differences +* `deep.Equal` does not compare unexported fields (by default) + +`reflect.DeepEqual` is good (like all things Golang!), but it's a game of [Hunt the Wumpus](https://en.wikipedia.org/wiki/Hunt_the_Wumpus). For large maps, slices, and structs, finding the difference is difficult. + +`deep.Equal` doesn't play games with you, it lists the differences: + +```go +package main_test + +import ( + "testing" + "github.com/go-test/deep" +) + +type T struct { + Name string + Numbers []float64 +} + +func TestDeepEqual(t *testing.T) { + // Can you spot the difference? + t1 := T{ + Name: "Isabella", + Numbers: []float64{1.13459, 2.29343, 3.010100010}, + } + t2 := T{ + Name: "Isabella", + Numbers: []float64{1.13459, 2.29843, 3.010100010}, + } + + if diff := deep.Equal(t1, t2); diff != nil { + t.Error(diff) + } +} +``` + + +``` +$ go test +--- FAIL: TestDeepEqual (0.00s) + main_test.go:25: [Numbers.slice[1]: 2.29343 != 2.29843] +``` + +The difference is in `Numbers.slice[1]`: the two values aren't equal using Go `==`. diff --git a/vendor/github.com/go-test/deep/deep.go b/vendor/github.com/go-test/deep/deep.go new file mode 100644 index 000000000..4ea14cb04 --- /dev/null +++ b/vendor/github.com/go-test/deep/deep.go @@ -0,0 +1,352 @@ +// Package deep provides function deep.Equal which is like reflect.DeepEqual but +// returns a list of differences. This is helpful when comparing complex types +// like structures and maps. +package deep + +import ( + "errors" + "fmt" + "log" + "reflect" + "strings" +) + +var ( + // FloatPrecision is the number of decimal places to round float values + // to when comparing. + FloatPrecision = 10 + + // MaxDiff specifies the maximum number of differences to return. + MaxDiff = 10 + + // MaxDepth specifies the maximum levels of a struct to recurse into. + MaxDepth = 10 + + // LogErrors causes errors to be logged to STDERR when true. + LogErrors = false + + // CompareUnexportedFields causes unexported struct fields, like s in + // T{s int}, to be comparsed when true. + CompareUnexportedFields = false +) + +var ( + // ErrMaxRecursion is logged when MaxDepth is reached. + ErrMaxRecursion = errors.New("recursed to MaxDepth") + + // ErrTypeMismatch is logged when Equal passed two different types of values. + ErrTypeMismatch = errors.New("variables are different reflect.Type") + + // ErrNotHandled is logged when a primitive Go kind is not handled. + ErrNotHandled = errors.New("cannot compare the reflect.Kind") +) + +type cmp struct { + diff []string + buff []string + floatFormat string +} + +var errorType = reflect.TypeOf((*error)(nil)).Elem() + +// Equal compares variables a and b, recursing into their structure up to +// MaxDepth levels deep, and returns a list of differences, or nil if there are +// none. Some differences may not be found if an error is also returned. +// +// If a type has an Equal method, like time.Equal, it is called to check for +// equality. +func Equal(a, b interface{}) []string { + aVal := reflect.ValueOf(a) + bVal := reflect.ValueOf(b) + c := &cmp{ + diff: []string{}, + buff: []string{}, + floatFormat: fmt.Sprintf("%%.%df", FloatPrecision), + } + if a == nil && b == nil { + return nil + } else if a == nil && b != nil { + c.saveDiff(b, "") + } else if a != nil && b == nil { + c.saveDiff(a, "") + } + if len(c.diff) > 0 { + return c.diff + } + + c.equals(aVal, bVal, 0) + if len(c.diff) > 0 { + return c.diff // diffs + } + return nil // no diffs +} + +func (c *cmp) equals(a, b reflect.Value, level int) { + if level > MaxDepth { + logError(ErrMaxRecursion) + return + } + + // Check if one value is nil, e.g. T{x: *X} and T.x is nil + if !a.IsValid() || !b.IsValid() { + if a.IsValid() && !b.IsValid() { + c.saveDiff(a.Type(), "") + } else if !a.IsValid() && b.IsValid() { + c.saveDiff("", b.Type()) + } + return + } + + // If differenet types, they can't be equal + aType := a.Type() + bType := b.Type() + if aType != bType { + c.saveDiff(aType, bType) + logError(ErrTypeMismatch) + return + } + + // Primitive https://golang.org/pkg/reflect/#Kind + aKind := a.Kind() + bKind := b.Kind() + + // If both types implement the error interface, compare the error strings. + // This must be done before dereferencing because the interface is on a + // pointer receiver. + if aType.Implements(errorType) && bType.Implements(errorType) { + if a.Elem().IsValid() && b.Elem().IsValid() { // both err != nil + aString := a.MethodByName("Error").Call(nil)[0].String() + bString := b.MethodByName("Error").Call(nil)[0].String() + if aString != bString { + c.saveDiff(aString, bString) + } + return + } + } + + // Dereference pointers and interface{} + if aElem, bElem := (aKind == reflect.Ptr || aKind == reflect.Interface), + (bKind == reflect.Ptr || bKind == reflect.Interface); aElem || bElem { + + if aElem { + a = a.Elem() + } + + if bElem { + b = b.Elem() + } + + c.equals(a, b, level+1) + return + } + + // Types with an Equal(), like time.Time. + eqFunc := a.MethodByName("Equal") + if eqFunc.IsValid() { + retVals := eqFunc.Call([]reflect.Value{b}) + if !retVals[0].Bool() { + c.saveDiff(a, b) + } + return + } + + switch aKind { + + ///////////////////////////////////////////////////////////////////// + // Iterable kinds + ///////////////////////////////////////////////////////////////////// + + case reflect.Struct: + /* + The variables are structs like: + type T struct { + FirstName string + LastName string + } + Type = .T, Kind = reflect.Struct + + Iterate through the fields (FirstName, LastName), recurse into their values. + */ + for i := 0; i < a.NumField(); i++ { + if aType.Field(i).PkgPath != "" && !CompareUnexportedFields { + continue // skip unexported field, e.g. s in type T struct {s string} + } + + c.push(aType.Field(i).Name) // push field name to buff + + // Get the Value for each field, e.g. FirstName has Type = string, + // Kind = reflect.String. + af := a.Field(i) + bf := b.Field(i) + + // Recurse to compare the field values + c.equals(af, bf, level+1) + + c.pop() // pop field name from buff + + if len(c.diff) >= MaxDiff { + break + } + } + case reflect.Map: + /* + The variables are maps like: + map[string]int{ + "foo": 1, + "bar": 2, + } + Type = map[string]int, Kind = reflect.Map + + Or: + type T map[string]int{} + Type = .T, Kind = reflect.Map + + Iterate through the map keys (foo, bar), recurse into their values. + */ + + if a.IsNil() || b.IsNil() { + if a.IsNil() && !b.IsNil() { + c.saveDiff("", b) + } else if !a.IsNil() && b.IsNil() { + c.saveDiff(a, "") + } + return + } + + if a.Pointer() == b.Pointer() { + return + } + + for _, key := range a.MapKeys() { + c.push(fmt.Sprintf("map[%s]", key)) + + aVal := a.MapIndex(key) + bVal := b.MapIndex(key) + if bVal.IsValid() { + c.equals(aVal, bVal, level+1) + } else { + c.saveDiff(aVal, "") + } + + c.pop() + + if len(c.diff) >= MaxDiff { + return + } + } + + for _, key := range b.MapKeys() { + if aVal := a.MapIndex(key); aVal.IsValid() { + continue + } + + c.push(fmt.Sprintf("map[%s]", key)) + c.saveDiff("", b.MapIndex(key)) + c.pop() + if len(c.diff) >= MaxDiff { + return + } + } + case reflect.Array: + n := a.Len() + for i := 0; i < n; i++ { + c.push(fmt.Sprintf("array[%d]", i)) + c.equals(a.Index(i), b.Index(i), level+1) + c.pop() + if len(c.diff) >= MaxDiff { + break + } + } + case reflect.Slice: + if a.IsNil() || b.IsNil() { + if a.IsNil() && !b.IsNil() { + c.saveDiff("", b) + } else if !a.IsNil() && b.IsNil() { + c.saveDiff(a, "") + } + return + } + + if a.Pointer() == b.Pointer() { + return + } + + aLen := a.Len() + bLen := b.Len() + n := aLen + if bLen > aLen { + n = bLen + } + for i := 0; i < n; i++ { + c.push(fmt.Sprintf("slice[%d]", i)) + if i < aLen && i < bLen { + c.equals(a.Index(i), b.Index(i), level+1) + } else if i < aLen { + c.saveDiff(a.Index(i), "") + } else { + c.saveDiff("", b.Index(i)) + } + c.pop() + if len(c.diff) >= MaxDiff { + break + } + } + + ///////////////////////////////////////////////////////////////////// + // Primitive kinds + ///////////////////////////////////////////////////////////////////// + + case reflect.Float32, reflect.Float64: + // Avoid 0.04147685731961082 != 0.041476857319611 + // 6 decimal places is close enough + aval := fmt.Sprintf(c.floatFormat, a.Float()) + bval := fmt.Sprintf(c.floatFormat, b.Float()) + if aval != bval { + c.saveDiff(a.Float(), b.Float()) + } + case reflect.Bool: + if a.Bool() != b.Bool() { + c.saveDiff(a.Bool(), b.Bool()) + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if a.Int() != b.Int() { + c.saveDiff(a.Int(), b.Int()) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + if a.Uint() != b.Uint() { + c.saveDiff(a.Uint(), b.Uint()) + } + case reflect.String: + if a.String() != b.String() { + c.saveDiff(a.String(), b.String()) + } + + default: + logError(ErrNotHandled) + } +} + +func (c *cmp) push(name string) { + c.buff = append(c.buff, name) +} + +func (c *cmp) pop() { + if len(c.buff) > 0 { + c.buff = c.buff[0 : len(c.buff)-1] + } +} + +func (c *cmp) saveDiff(aval, bval interface{}) { + if len(c.buff) > 0 { + varName := strings.Join(c.buff, ".") + c.diff = append(c.diff, fmt.Sprintf("%s: %v != %v", varName, aval, bval)) + } else { + c.diff = append(c.diff, fmt.Sprintf("%v != %v", aval, bval)) + } +} + +func logError(err error) { + if LogErrors { + log.Println(err) + } +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 3465886dd..a753fb261 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -1158,6 +1158,12 @@ "revision": "766e555c68dc8bda90d197ee8946c37519c19409", "revisionTime": "2017-01-17T13:00:17Z" }, + { + "checksumSHA1": "dni8Puy96l7QbQlJwLtrZvM5rxM=", + "path": "github.com/go-test/deep", + "revision": "6592d9cc0a499ad2d5f574fde80a2b5c5cc3b4f5", + "revisionTime": "2018-01-28T22:55:04Z" + }, { "checksumSHA1": "q3Bc7JpLWBqhZ4M7oreGo34RSkc=", "path": "github.com/golang/protobuf/proto", From 3d551e25e01bd95c190766869bf0260debe07ead Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 14 Feb 2018 12:46:13 -0800 Subject: [PATCH 18/25] configs: BuildConfig sorts child modules by name This is not strictly necessary, but since this is not a performance-critical codepath we'll do this because it makes life easier for callers that want to print out user-facing logs about build process, or who are logging actions taken as part of a unit test. --- configs/config_build.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/configs/config_build.go b/configs/config_build.go index 7aca4f1c5..b33d121e4 100644 --- a/configs/config_build.go +++ b/configs/config_build.go @@ -1,6 +1,8 @@ package configs import ( + "sort" + version "github.com/hashicorp/go-version" "github.com/hashicorp/hcl2/hcl" ) @@ -28,7 +30,16 @@ func buildChildModules(parent *Config, walker ModuleWalker) (map[string]*Config, calls := parent.Module.ModuleCalls - for _, call := range calls { + // We'll sort the calls by their local names so that they'll appear in a + // predictable order in any logging that's produced during the walk. + callNames := make([]string, 0, len(calls)) + for k := range calls { + callNames = append(callNames, k) + } + sort.Strings(callNames) + + for _, callName := range callNames { + call := calls[callName] path := make([]string, len(parent.Path)+1) copy(path, parent.Path) path[len(path)-1] = call.Name From 59939cf320aa19c50c05a1ed668f26088906fa95 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 13 Feb 2018 16:36:36 -0800 Subject: [PATCH 19/25] configs/configload: installation from registry and go-getter Unlike the old installer in config/module, this uses new-style installation directories that include the static module path so that paths we show in diagnostics will be more meaningful to the user. As before, we retrieve the entire "package" associated with the given source string, rather than any given subdirectory directly, because the retrieved module may contain ../ references into parent directories which must be resolvable after extraction. --- configs/configload/copy_dir.go | 114 ++++++++ configs/configload/getter.go | 122 ++++++++ configs/configload/inode.go | 21 ++ configs/configload/inode_freebsd.go | 21 ++ configs/configload/inode_windows.go | 8 + configs/configload/loader_install.go | 271 +++++++++++++++++- configs/configload/loader_install_test.go | 205 ++++++++++++- configs/configload/loader_test.go | 109 +++++-- configs/configload/source_addr.go | 17 ++ .../go-getter-modules/.gitignore | 1 + .../test-fixtures/go-getter-modules/root.tf | 16 ++ .../test-fixtures/registry-modules/.gitignore | 1 + .../test-fixtures/registry-modules/root.tf | 27 ++ registry/client.go | 2 +- registry/errors.go | 23 ++ 15 files changed, 915 insertions(+), 43 deletions(-) create mode 100644 configs/configload/copy_dir.go create mode 100644 configs/configload/getter.go create mode 100644 configs/configload/inode.go create mode 100644 configs/configload/inode_freebsd.go create mode 100644 configs/configload/inode_windows.go create mode 100644 configs/configload/test-fixtures/go-getter-modules/.gitignore create mode 100644 configs/configload/test-fixtures/go-getter-modules/root.tf create mode 100644 configs/configload/test-fixtures/registry-modules/.gitignore create mode 100644 configs/configload/test-fixtures/registry-modules/root.tf create mode 100644 registry/errors.go diff --git a/configs/configload/copy_dir.go b/configs/configload/copy_dir.go new file mode 100644 index 000000000..ad34a3204 --- /dev/null +++ b/configs/configload/copy_dir.go @@ -0,0 +1,114 @@ +package configload + +import ( + "io" + "os" + "path/filepath" + "strings" +) + +// copyDir copies the src directory contents into dst. Both directories +// should already exist. +func copyDir(dst, src string) error { + src, err := filepath.EvalSymlinks(src) + if err != nil { + return err + } + + walkFn := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if path == src { + return nil + } + + if strings.HasPrefix(filepath.Base(path), ".") { + // Skip any dot files + if info.IsDir() { + return filepath.SkipDir + } else { + return nil + } + } + + // The "path" has the src prefixed to it. We need to join our + // destination with the path without the src on it. + dstPath := filepath.Join(dst, path[len(src):]) + + // we don't want to try and copy the same file over itself. + if eq, err := sameFile(path, dstPath); eq { + return nil + } else if err != nil { + return err + } + + // If we have a directory, make that subdirectory, then continue + // the walk. + if info.IsDir() { + if path == filepath.Join(src, dst) { + // dst is in src; don't walk it. + return nil + } + + if err := os.MkdirAll(dstPath, 0755); err != nil { + return err + } + + return nil + } + + // If we have a file, copy the contents. + srcF, err := os.Open(path) + if err != nil { + return err + } + defer srcF.Close() + + dstF, err := os.Create(dstPath) + if err != nil { + return err + } + defer dstF.Close() + + if _, err := io.Copy(dstF, srcF); err != nil { + return err + } + + // Chmod it + return os.Chmod(dstPath, info.Mode()) + } + + return filepath.Walk(src, walkFn) +} + +// sameFile tried to determine if to paths are the same file. +// If the paths don't match, we lookup the inode on supported systems. +func sameFile(a, b string) (bool, error) { + if a == b { + return true, nil + } + + aIno, err := inode(a) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + bIno, err := inode(b) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + if aIno > 0 && aIno == bIno { + return true, nil + } + + return false, nil +} diff --git a/configs/configload/getter.go b/configs/configload/getter.go new file mode 100644 index 000000000..fffc48c04 --- /dev/null +++ b/configs/configload/getter.go @@ -0,0 +1,122 @@ +package configload + +import ( + "log" + "path/filepath" + + cleanhttp "github.com/hashicorp/go-cleanhttp" + getter "github.com/hashicorp/go-getter" +) + +// We configure our own go-getter detector and getter sets here, because +// the set of sources we support is part of Terraform's documentation and +// so we don't want any new sources introduced in go-getter to sneak in here +// and work even though they aren't documented. This also insulates us from +// any meddling that might be done by other go-getter callers linked into our +// executable. + +var goGetterDetectors = []getter.Detector{ + new(getter.GitHubDetector), + new(getter.BitBucketDetector), + new(getter.S3Detector), + new(getter.FileDetector), +} + +var goGetterNoDetectors = []getter.Detector{} + +var goGetterDecompressors = map[string]getter.Decompressor{ + "bz2": new(getter.Bzip2Decompressor), + "gz": new(getter.GzipDecompressor), + "xz": new(getter.XzDecompressor), + "zip": new(getter.ZipDecompressor), + + "tar.bz2": new(getter.TarBzip2Decompressor), + "tar.tbz2": new(getter.TarBzip2Decompressor), + + "tar.gz": new(getter.TarGzipDecompressor), + "tgz": new(getter.TarGzipDecompressor), + + "tar.xz": new(getter.TarXzDecompressor), + "txz": new(getter.TarXzDecompressor), +} + +var goGetterGetters = map[string]getter.Getter{ + "file": new(getter.FileGetter), + "git": new(getter.GitGetter), + "hg": new(getter.HgGetter), + "s3": new(getter.S3Getter), + "http": getterHTTPGetter, + "https": getterHTTPGetter, +} + +var getterHTTPClient = cleanhttp.DefaultClient() + +var getterHTTPGetter = &getter.HttpGetter{ + Client: getterHTTPClient, + Netrc: true, +} + +// getWithGoGetter retrieves the package referenced in the given address +// into the installation path and then returns the full path to any subdir +// indicated in the address. +// +// The errors returned by this function are those surfaced by the underlying +// go-getter library, which have very inconsistent quality as +// end-user-actionable error messages. At this time we do not have any +// reasonable way to improve these error messages at this layer because +// the underlying errors are not separatelyr recognizable. +func getWithGoGetter(instPath, addr string) (string, error) { + packageAddr, subDir := splitAddrSubdir(addr) + + log.Printf("[DEBUG] will download %q to %s", packageAddr, instPath) + + realAddr, err := getter.Detect(packageAddr, instPath, getter.Detectors) + if err != nil { + return "", err + } + + var realSubDir string + realAddr, realSubDir = splitAddrSubdir(realAddr) + if realSubDir != "" { + subDir = filepath.Join(realSubDir, subDir) + } + + if realAddr != packageAddr { + log.Printf("[TRACE] go-getter detectors rewrote %q to %q", packageAddr, realAddr) + } + + client := getter.Client{ + Src: realAddr, + Dst: instPath, + Pwd: instPath, + + Mode: getter.ClientModeDir, + + Detectors: goGetterNoDetectors, // we already did detection above + Decompressors: goGetterDecompressors, + Getters: goGetterGetters, + } + err = client.Get() + if err != nil { + return "", err + } + + // Our subDir string can contain wildcards until this point, so that + // e.g. a subDir of * can expand to one top-level directory in a .tar.gz + // archive. Now that we've expanded the archive successfully we must + // resolve that into a concrete path. + var finalDir string + if subDir != "" { + finalDir, err = getter.SubdirGlob(instPath, subDir) + log.Printf("[TRACE] expanded %q to %q", subDir, finalDir) + if err != nil { + return "", err + } + } else { + finalDir = instPath + } + + // If we got this far then we have apparently succeeded in downloading + // the requested object! + return filepath.Clean(finalDir), nil +} diff --git a/configs/configload/inode.go b/configs/configload/inode.go new file mode 100644 index 000000000..57df04145 --- /dev/null +++ b/configs/configload/inode.go @@ -0,0 +1,21 @@ +// +build linux darwin openbsd netbsd solaris dragonfly + +package configload + +import ( + "fmt" + "os" + "syscall" +) + +// lookup the inode of a file on posix systems +func inode(path string) (uint64, error) { + stat, err := os.Stat(path) + if err != nil { + return 0, err + } + if st, ok := stat.Sys().(*syscall.Stat_t); ok { + return st.Ino, nil + } + return 0, fmt.Errorf("could not determine file inode") +} diff --git a/configs/configload/inode_freebsd.go b/configs/configload/inode_freebsd.go new file mode 100644 index 000000000..4dc28eaa8 --- /dev/null +++ b/configs/configload/inode_freebsd.go @@ -0,0 +1,21 @@ +// +build freebsd + +package configload + +import ( + "fmt" + "os" + "syscall" +) + +// lookup the inode of a file on posix systems +func inode(path string) (uint64, error) { + stat, err := os.Stat(path) + if err != nil { + return 0, err + } + if st, ok := stat.Sys().(*syscall.Stat_t); ok { + return uint64(st.Ino), nil + } + return 0, fmt.Errorf("could not determine file inode") +} diff --git a/configs/configload/inode_windows.go b/configs/configload/inode_windows.go new file mode 100644 index 000000000..0d22e6726 --- /dev/null +++ b/configs/configload/inode_windows.go @@ -0,0 +1,8 @@ +// +build windows + +package configload + +// no syscall.Stat_t on windows, return 0 for inodes +func inode(path string) (uint64, error) { + return 0, nil +} diff --git a/configs/configload/loader_install.go b/configs/configload/loader_install.go index 2ab6529d3..41ef4224d 100644 --- a/configs/configload/loader_install.go +++ b/configs/configload/loader_install.go @@ -10,6 +10,7 @@ import ( version "github.com/hashicorp/go-version" "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry" "github.com/hashicorp/terraform/registry/regsrc" ) @@ -154,15 +155,16 @@ func (l *Loader) InstallModules(rootDir string, upgrade bool, hooks InstallHooks } log.Printf("[TRACE] %s is a registry module at %s", key, addr) - // TODO: Implement - panic("registry source installation not yet implemented") + mod, v, mDiags := l.installRegistryModule(req, key, instPath, addr, hooks) + diags = append(diags, mDiags...) + return mod, v, diags default: - log.Printf("[TRACE] %s address %q will be interpreted with go-getter", key, req.SourceAddr) - - // TODO: Implement - panic("fallback source installation not yet implemented") + log.Printf("[TRACE] %s address %q will be handled by go-getter", key, req.SourceAddr) + mod, mDiags := l.installGoGetterModule(req, key, instPath, hooks) + diags = append(diags, mDiags...) + return mod, nil, diags } }, @@ -226,12 +228,267 @@ func (l *Loader) installLocalModule(req *configs.ModuleRequest, key string, hook Dir: newDir, SourceAddr: req.SourceAddr, } - log.Printf("[TRACE] Module installer: %s installed at %s", key, newDir) + log.Printf("[DEBUG] Module installer: %s installed at %s", key, newDir) hooks.Install(key, nil, newDir) return mod, diags } +func (l *Loader) installRegistryModule(req *configs.ModuleRequest, key string, instPath string, addr *regsrc.Module, hooks InstallHooks) (*configs.Module, *version.Version, hcl.Diagnostics) { + var diags hcl.Diagnostics + + hostname, err := addr.SvcHost() + if err != nil { + // If it looks like the user was trying to use punycode then we'll generate + // a specialized error for that case. We require the unicode form of + // hostname so that hostnames are always human-readable in configuration + // and punycode can't be used to hide a malicious module hostname. + if strings.HasPrefix(addr.RawHost.Raw, "xn--") { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid module registry hostname", + Detail: "The hostname portion of this source address is not an acceptable hostname. Internationalized domain names must be given in unicode form rather than ASCII (\"punycode\") form.", + Subject: &req.SourceAddrRange, + }) + } else { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid module registry hostname", + Detail: "The hostname portion of this source address is not a valid hostname.", + Subject: &req.SourceAddrRange, + }) + } + return nil, nil, diags + } + + reg := l.modules.Registry + + log.Printf("[DEBUG] %s listing available versions of %s at %s", key, addr, hostname) + resp, err := reg.Versions(addr) + if err != nil { + if registry.IsModuleNotFound(err) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Module not found", + Detail: fmt.Sprintf("The specified module could not be found in the module registry at %s.", hostname), + Subject: &req.SourceAddrRange, + }) + } else { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Error accessing remote module registry", + Detail: fmt.Sprintf("Failed to retrieve available versions for this module from %s: %s.", hostname, err), + Subject: &req.SourceAddrRange, + }) + } + return nil, nil, diags + } + + // The response might contain information about dependencies to allow us + // to potentially optimize future requests, but we don't currently do that + // and so for now we'll just take the first item which is guaranteed to + // be the address we requested. + if len(resp.Modules) < 1 { + // Should never happen, but since this is a remote service that may + // be implemented by third-parties we will handle it gracefully. + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid response from remote module registry", + Detail: fmt.Sprintf("The registry at %s returned an invalid response when Terraform requested available versions for this module.", hostname), + Subject: &req.SourceAddrRange, + }) + return nil, nil, diags + } + + modMeta := resp.Modules[0] + + var latestMatch *version.Version + var latestVersion *version.Version + for _, mv := range modMeta.Versions { + v, err := version.NewVersion(mv.Version) + if err != nil { + // Should never happen if the registry server is compliant with + // the protocol, but we'll warn if not to assist someone who + // might be developing a module registry server. + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "Invalid response from remote module registry", + Detail: fmt.Sprintf("The registry at %s returned an invalid version string %q for this module, which Terraform ignored.", hostname, mv.Version), + Subject: &req.SourceAddrRange, + }) + continue + } + + // If we've found a pre-release version then we'll ignore it unless + // it was exactly requested. + if v.Prerelease() != "" && req.VersionConstraint.Required.String() != v.String() { + log.Printf("[TRACE] %s ignoring %s because it is a pre-release and was not requested exactly", key, v) + continue + } + + if latestVersion == nil || v.GreaterThan(latestVersion) { + latestVersion = v + } + + if req.VersionConstraint.Required.Check(v) { + if latestMatch == nil || v.GreaterThan(latestMatch) { + latestMatch = v + } + } + } + + if latestVersion == nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Module has no versions", + Detail: fmt.Sprintf("The specified module does not have any available versions."), + Subject: &req.SourceAddrRange, + }) + return nil, nil, diags + } + + if latestMatch == nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unresolvable module version constraint", + Detail: fmt.Sprintf("There is no available version of %q that matches the given version constraint. The newest available version is %s.", addr, latestVersion), + Subject: &req.VersionConstraint.DeclRange, + }) + return nil, nil, diags + } + + // Report up to the caller that we're about to start downloading. + packageAddr, _ := splitAddrSubdir(req.SourceAddr) + hooks.Download(key, packageAddr, latestMatch) + + // If we manage to get down here then we've found a suitable version to + // install, so we need to ask the registry where we should download it from. + // The response to this is a go-getter-style address string. + dlAddr, err := reg.Location(addr, latestMatch.String()) + if err != nil { + log.Printf("[ERROR] %s from %s %s: %s", key, addr, latestMatch, err) + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid response from remote module registry", + Detail: fmt.Sprintf("The remote registry at %s failed to return a download URL for %s %s.", hostname, addr, latestMatch), + Subject: &req.VersionConstraint.DeclRange, + }) + return nil, nil, diags + } + + log.Printf("[TRACE] %s %s %s is available at %q", key, addr, latestMatch, dlAddr) + + modDir, err := getWithGoGetter(instPath, dlAddr) + if err != nil { + // Errors returned by go-getter have very inconsistent quality as + // end-user error messages, but for now we're accepting that because + // we have no way to recognize any specific errors to improve them + // and masking the error entirely would hide valuable diagnostic + // information from the user. + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to download module", + Detail: fmt.Sprintf("Error attempting to download module source code from %q: %s", dlAddr, err), + Subject: &req.CallRange, + }) + return nil, nil, diags + } + + log.Printf("[TRACE] %s %q was downloaded to %s", key, dlAddr, modDir) + + if addr.RawSubmodule != "" { + // Append the user's requested subdirectory to any subdirectory that + // was implied by any of the nested layers we expanded within go-getter. + modDir = filepath.Join(modDir, addr.RawSubmodule) + } + + log.Printf("[TRACE] %s should now be at %s", key, modDir) + + // Finally we are ready to try actually loading the module. + mod, mDiags := l.parser.LoadConfigDir(modDir) + if mod == nil { + // nil indicates missing or unreadable directory, so we'll + // discard the returned diags and return a more specific + // error message here. For registry modules this actually + // indicates a bug in the code above, since it's not the + // user's responsibility to create the directory in this case. + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unreadable module directory", + Detail: fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir), + Subject: &req.CallRange, + }) + } else { + diags = append(diags, mDiags...) + } + + // Note the local location in our manifest. + l.modules.manifest[key] = moduleRecord{ + Key: key, + Version: latestMatch, + Dir: modDir, + SourceAddr: req.SourceAddr, + } + log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir) + hooks.Install(key, latestMatch, modDir) + + return mod, latestMatch, diags +} + +func (l *Loader) installGoGetterModule(req *configs.ModuleRequest, key string, instPath string, hooks InstallHooks) (*configs.Module, hcl.Diagnostics) { + var diags hcl.Diagnostics + + // Report up to the caller that we're about to start downloading. + packageAddr, _ := splitAddrSubdir(req.SourceAddr) + hooks.Download(key, packageAddr, nil) + + modDir, err := getWithGoGetter(instPath, req.SourceAddr) + if err != nil { + // Errors returned by go-getter have very inconsistent quality as + // end-user error messages, but for now we're accepting that because + // we have no way to recognize any specific errors to improve them + // and masking the error entirely would hide valuable diagnostic + // information from the user. + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to download module", + Detail: fmt.Sprintf("Error attempting to download module source code from %q: %s", packageAddr, err), + Subject: &req.SourceAddrRange, + }) + return nil, diags + } + + log.Printf("[TRACE] %s %q was downloaded to %s", key, req.SourceAddr, modDir) + + mod, mDiags := l.parser.LoadConfigDir(modDir) + if mod == nil { + // nil indicates missing or unreadable directory, so we'll + // discard the returned diags and return a more specific + // error message here. For registry modules this actually + // indicates a bug in the code above, since it's not the + // user's responsibility to create the directory in this case. + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unreadable module directory", + Detail: fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir), + Subject: &req.CallRange, + }) + } else { + diags = append(diags, mDiags...) + } + + // Note the local location in our manifest. + l.modules.manifest[key] = moduleRecord{ + Key: key, + Dir: modDir, + SourceAddr: req.SourceAddr, + } + log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir) + hooks.Install(key, nil, modDir) + + return mod, diags +} + func (l *Loader) packageInstallPath(modulePath []string) string { return filepath.Join(l.modules.Dir, strings.Join(modulePath, ".")) } diff --git a/configs/configload/loader_install_test.go b/configs/configload/loader_install_test.go index 608b08a00..2be85a6ba 100644 --- a/configs/configload/loader_install_test.go +++ b/configs/configload/loader_install_test.go @@ -1,6 +1,7 @@ package configload import ( + "os" "path/filepath" "testing" @@ -9,11 +10,12 @@ import ( func TestLoaderInstallModules_local(t *testing.T) { fixtureDir := filepath.Clean("test-fixtures/local-modules") - loader := newTestLoader(filepath.Join(fixtureDir, ".terraform/modules")) + loader, done := tempChdirLoader(t, fixtureDir) + defer done() hooks := &testInstallHooks{} - diags := loader.InstallModules(fixtureDir, false, hooks) + diags := loader.InstallModules(".", false, hooks) assertNoDiagnostics(t, diags) wantCalls := []testInstallHookCall{ @@ -21,17 +23,210 @@ func TestLoaderInstallModules_local(t *testing.T) { Name: "Install", ModuleAddr: "child_a", PackageAddr: "", - LocalPath: "test-fixtures/local-modules/child_a", + LocalPath: "child_a", }, { Name: "Install", ModuleAddr: "child_a.child_b", PackageAddr: "", - LocalPath: "test-fixtures/local-modules/child_a/child_b", + LocalPath: "child_a/child_b", }, } - assertResultDeepEqual(t, hooks.Calls, wantCalls) + if assertResultDeepEqual(t, hooks.Calls, wantCalls) { + return + } + + // Make sure the configuration is loadable now. + // (This ensures that correct information is recorded in the manifest.) + _, loadDiags := loader.LoadConfig(".") + assertNoDiagnostics(t, loadDiags) +} + +func TestLoaderInstallModules_registry(t *testing.T) { + if os.Getenv("TF_ACC") == "" { + t.Skip("this test accesses registry.terraform.io and github.com; set TF_ACC=1 to run it") + } + + fixtureDir := filepath.Clean("test-fixtures/registry-modules") + loader, done := tempChdirLoader(t, fixtureDir) + defer done() + + hooks := &testInstallHooks{} + + diags := loader.InstallModules(".", false, hooks) + assertNoDiagnostics(t, diags) + + v := version.Must(version.NewVersion("0.0.1")) + + wantCalls := []testInstallHookCall{ + // the configuration builder visits each level of calls in lexicographical + // order by name, so the following list is kept in the same order. + + // acctest_child_a accesses //modules/child_a directly + { + Name: "Download", + ModuleAddr: "acctest_child_a", + PackageAddr: "hashicorp/module-installer-acctest/aws", // intentionally excludes the subdir because we're downloading the whole package here + Version: v, + }, + { + Name: "Install", + ModuleAddr: "acctest_child_a", + Version: v, + LocalPath: ".terraform/modules/acctest_child_a/hashicorp-terraform-aws-module-installer-acctest-853d038/modules/child_a", + }, + + // acctest_child_a.child_b + // (no download because it's a relative path inside acctest_child_a) + { + Name: "Install", + ModuleAddr: "acctest_child_a.child_b", + LocalPath: ".terraform/modules/acctest_child_a/hashicorp-terraform-aws-module-installer-acctest-853d038/modules/child_b", + }, + + // acctest_child_b accesses //modules/child_b directly + { + Name: "Download", + ModuleAddr: "acctest_child_b", + PackageAddr: "hashicorp/module-installer-acctest/aws", // intentionally excludes the subdir because we're downloading the whole package here + Version: v, + }, + { + Name: "Install", + ModuleAddr: "acctest_child_b", + Version: v, + LocalPath: ".terraform/modules/acctest_child_b/hashicorp-terraform-aws-module-installer-acctest-853d038/modules/child_b", + }, + + // acctest_root + { + Name: "Download", + ModuleAddr: "acctest_root", + PackageAddr: "hashicorp/module-installer-acctest/aws", + Version: v, + }, + { + Name: "Install", + ModuleAddr: "acctest_root", + Version: v, + LocalPath: ".terraform/modules/acctest_root/hashicorp-terraform-aws-module-installer-acctest-853d038", + }, + + // acctest_root.child_a + // (no download because it's a relative path inside acctest_root) + { + Name: "Install", + ModuleAddr: "acctest_root.child_a", + LocalPath: ".terraform/modules/acctest_root/hashicorp-terraform-aws-module-installer-acctest-853d038/modules/child_a", + }, + + // acctest_root.child_a.child_b + // (no download because it's a relative path inside acctest_root, via acctest_root.child_a) + { + Name: "Install", + ModuleAddr: "acctest_root.child_a.child_b", + LocalPath: ".terraform/modules/acctest_root/hashicorp-terraform-aws-module-installer-acctest-853d038/modules/child_b", + }, + } + + if assertResultDeepEqual(t, hooks.Calls, wantCalls) { + return + } + + // Make sure the configuration is loadable now. + // (This ensures that correct information is recorded in the manifest.) + _, loadDiags := loader.LoadConfig(".") + assertNoDiagnostics(t, loadDiags) +} + +func TestLoaderInstallModules_goGetter(t *testing.T) { + if os.Getenv("TF_ACC") == "" { + t.Skip("this test accesses github.com; set TF_ACC=1 to run it") + } + + fixtureDir := filepath.Clean("test-fixtures/go-getter-modules") + loader, done := tempChdirLoader(t, fixtureDir) + defer done() + + hooks := &testInstallHooks{} + + diags := loader.InstallModules(".", false, hooks) + assertNoDiagnostics(t, diags) + + wantCalls := []testInstallHookCall{ + // the configuration builder visits each level of calls in lexicographical + // order by name, so the following list is kept in the same order. + + // acctest_child_a accesses //modules/child_a directly + { + Name: "Download", + ModuleAddr: "acctest_child_a", + PackageAddr: "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1", // intentionally excludes the subdir because we're downloading the whole repo here + }, + { + Name: "Install", + ModuleAddr: "acctest_child_a", + LocalPath: ".terraform/modules/acctest_child_a/modules/child_a", + }, + + // acctest_child_a.child_b + // (no download because it's a relative path inside acctest_child_a) + { + Name: "Install", + ModuleAddr: "acctest_child_a.child_b", + LocalPath: ".terraform/modules/acctest_child_a/modules/child_b", + }, + + // acctest_child_b accesses //modules/child_b directly + { + Name: "Download", + ModuleAddr: "acctest_child_b", + PackageAddr: "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1", // intentionally excludes the subdir because we're downloading the whole package here + }, + { + Name: "Install", + ModuleAddr: "acctest_child_b", + LocalPath: ".terraform/modules/acctest_child_b/modules/child_b", + }, + + // acctest_root + { + Name: "Download", + ModuleAddr: "acctest_root", + PackageAddr: "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1", + }, + { + Name: "Install", + ModuleAddr: "acctest_root", + LocalPath: ".terraform/modules/acctest_root", + }, + + // acctest_root.child_a + // (no download because it's a relative path inside acctest_root) + { + Name: "Install", + ModuleAddr: "acctest_root.child_a", + LocalPath: ".terraform/modules/acctest_root/modules/child_a", + }, + + // acctest_root.child_a.child_b + // (no download because it's a relative path inside acctest_root, via acctest_root.child_a) + { + Name: "Install", + ModuleAddr: "acctest_root.child_a.child_b", + LocalPath: ".terraform/modules/acctest_root/modules/child_b", + }, + } + + if assertResultDeepEqual(t, hooks.Calls, wantCalls) { + return + } + + // Make sure the configuration is loadable now. + // (This ensures that correct information is recorded in the manifest.) + _, loadDiags := loader.LoadConfig(".") + assertNoDiagnostics(t, loadDiags) } type testInstallHooks struct { diff --git a/configs/configload/loader_test.go b/configs/configload/loader_test.go index 6ab1fc8c5..239599dc1 100644 --- a/configs/configload/loader_test.go +++ b/configs/configload/loader_test.go @@ -1,43 +1,90 @@ package configload import ( - "reflect" + "fmt" + "io/ioutil" + "os" + "path/filepath" "testing" - "github.com/spf13/afero" - - "github.com/davecgh/go-spew/spew" + "github.com/go-test/deep" "github.com/hashicorp/hcl2/hcl" - "github.com/hashicorp/terraform/configs" - "github.com/hashicorp/terraform/registry" "github.com/zclconf/go-cty/cty" ) -// newTestLoader is like NewLoader but it uses a copy-on-write overlay filesystem -// over the real filesystem so that any files that are created cannot persist -// between test runs. +// tempChdir copies the contents of the given directory to a temporary +// directory and changes the test process's current working directory to +// point to that directory. Also returned is a function that should be +// called at the end of the test (e.g. via "defer") to restore the previous +// working directory. // -// It will also panic if there are any errors creating the loader, since -// these should never happen in a testing scenario. -func newTestLoader(dir string) *Loader { - realFS := afero.NewOsFs() - overlayFS := afero.NewMemMapFs() - fs := afero.NewCopyOnWriteFs(realFS, overlayFS) - parser := configs.NewParser(fs) - reg := registry.NewClient(nil, nil, nil) - ret := &Loader{ - parser: parser, - modules: moduleMgr{ - FS: afero.Afero{fs}, - Dir: dir, - Registry: reg, - }, - } - err := ret.modules.readModuleManifestSnapshot() +// Tests using this helper cannot safely be run in parallel with other tests. +func tempChdir(t *testing.T, sourceDir string) (string, func()) { + t.Helper() + + tmpDir, err := ioutil.TempDir("", "terraform-configload") if err != nil { - panic(err) + t.Fatalf("failed to create temporary directory: %s", err) + return "", nil } - return ret + + if err := copyDir(tmpDir, sourceDir); err != nil { + t.Fatalf("failed to copy fixture to temporary directory: %s", err) + return "", nil + } + + oldDir, err := os.Getwd() + if err != nil { + t.Fatalf("failed to determine current working directory: %s", err) + return "", nil + } + + err = os.Chdir(tmpDir) + if err != nil { + t.Fatalf("failed to switch to temp dir %s: %s", tmpDir, err) + return "", nil + } + + t.Logf("tempChdir switched to %s after copying from %s", tmpDir, sourceDir) + + return tmpDir, func() { + err := os.Chdir(oldDir) + if err != nil { + panic(fmt.Errorf("failed to restore previous working directory %s: %s", oldDir, err)) + } + + if os.Getenv("TF_CONFIGLOAD_TEST_KEEP_TMP") == "" { + os.RemoveAll(tmpDir) + } + } +} + +// tempChdirLoader is a wrapper around tempChdir that also returns a Loader +// whose modules directory is at the conventional location within the +// created temporary directory. +func tempChdirLoader(t *testing.T, sourceDir string) (*Loader, func()) { + t.Helper() + + _, done := tempChdir(t, sourceDir) + modulesDir := filepath.Clean(".terraform/modules") + + err := os.MkdirAll(modulesDir, os.ModePerm) + if err != nil { + done() // undo the chdir in tempChdir so we can safely run other tests + t.Fatalf("failed to create modules directory: %s", err) + return nil, nil + } + + loader, err := NewLoader(&Config{ + ModulesDir: modulesDir, + }) + if err != nil { + done() // undo the chdir in tempChdir so we can safely run other tests + t.Fatalf("failed to create loader: %s", err) + return nil, nil + } + + return loader, done } func assertNoDiagnostics(t *testing.T, diags hcl.Diagnostics) bool { @@ -75,8 +122,10 @@ func assertDiagnosticSummary(t *testing.T, diags hcl.Diagnostics, want string) b func assertResultDeepEqual(t *testing.T, got, want interface{}) bool { t.Helper() - if !reflect.DeepEqual(got, want) { - t.Errorf("wrong result\ngot: %swant: %s", spew.Sdump(got), spew.Sdump(want)) + if diff := deep.Equal(got, want); diff != nil { + for _, problem := range diff { + t.Errorf("%s", problem) + } return true } return false diff --git a/configs/configload/source_addr.go b/configs/configload/source_addr.go index 7767859c5..594cf6406 100644 --- a/configs/configload/source_addr.go +++ b/configs/configload/source_addr.go @@ -3,6 +3,8 @@ package configload import ( "strings" + "github.com/hashicorp/go-getter" + "github.com/hashicorp/terraform/registry/regsrc" ) @@ -26,3 +28,18 @@ func isRegistrySourceAddr(addr string) bool { _, err := regsrc.ParseModuleSource(addr) return err == nil } + +// splitAddrSubdir splits the given address (which is assumed to be a +// registry address or go-getter-style address) into a package portion +// and a sub-directory portion. +// +// The package portion defines what should be downloaded and then the +// sub-directory portion, if present, specifies a sub-directory within +// the downloaded object (an archive, VCS repository, etc) that contains +// the module's configuration files. +// +// The subDir portion will be returned as empty if no subdir separator +// ("//") is present in the address. +func splitAddrSubdir(addr string) (packageAddr, subDir string) { + return getter.SourceDirSubdir(addr) +} diff --git a/configs/configload/test-fixtures/go-getter-modules/.gitignore b/configs/configload/test-fixtures/go-getter-modules/.gitignore new file mode 100644 index 000000000..6e0db03a8 --- /dev/null +++ b/configs/configload/test-fixtures/go-getter-modules/.gitignore @@ -0,0 +1 @@ +.terraform/* diff --git a/configs/configload/test-fixtures/go-getter-modules/root.tf b/configs/configload/test-fixtures/go-getter-modules/root.tf new file mode 100644 index 000000000..3c92ef1db --- /dev/null +++ b/configs/configload/test-fixtures/go-getter-modules/root.tf @@ -0,0 +1,16 @@ +# This fixture depends on a github repo at: +# https://github.com/hashicorp/terraform-aws-module-installer-acctest +# ...and expects its v0.0.1 tag to be pointing at the following commit: +# d676ab2559d4e0621d59e3c3c4cbb33958ac4608 + +module "acctest_root" { + source = "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1" +} + +module "acctest_child_a" { + source = "github.com/hashicorp/terraform-aws-module-installer-acctest//modules/child_a?ref=v0.0.1" +} + +module "acctest_child_b" { + source = "github.com/hashicorp/terraform-aws-module-installer-acctest//modules/child_b?ref=v0.0.1" +} diff --git a/configs/configload/test-fixtures/registry-modules/.gitignore b/configs/configload/test-fixtures/registry-modules/.gitignore new file mode 100644 index 000000000..6e0db03a8 --- /dev/null +++ b/configs/configload/test-fixtures/registry-modules/.gitignore @@ -0,0 +1 @@ +.terraform/* diff --git a/configs/configload/test-fixtures/registry-modules/root.tf b/configs/configload/test-fixtures/registry-modules/root.tf new file mode 100644 index 000000000..786966494 --- /dev/null +++ b/configs/configload/test-fixtures/registry-modules/root.tf @@ -0,0 +1,27 @@ +# This fixture indirectly depends on a github repo at: +# https://github.com/hashicorp/terraform-aws-module-installer-acctest +# ...and expects its v0.0.1 tag to be pointing at the following commit: +# d676ab2559d4e0621d59e3c3c4cbb33958ac4608 +# +# This repository is accessed indirectly via: +# https://registry.terraform.io/modules/hashicorp/module-installer-acctest/aws/0.0.1 +# +# Since the tag's id is included in a downloaded archive, it is expected to +# have the following id: +# 853d03855b3290a3ca491d4c3a7684572dd42237 +# (this particular assumption is encoded in the tests that use this fixture) + +module "acctest_root" { + source = "hashicorp/module-installer-acctest/aws" + version = "0.0.1" +} + +module "acctest_child_a" { + source = "hashicorp/module-installer-acctest/aws//modules/child_a" + version = "0.0.1" +} + +module "acctest_child_b" { + source = "hashicorp/module-installer-acctest/aws//modules/child_b" + version = "0.0.1" +} diff --git a/registry/client.go b/registry/client.go index c69d31b15..6525c5410 100644 --- a/registry/client.go +++ b/registry/client.go @@ -115,7 +115,7 @@ func (c *Client) Versions(module *regsrc.Module) (*response.ModuleVersions, erro case http.StatusOK: // OK case http.StatusNotFound: - return nil, fmt.Errorf("module %q not found", module.String()) + return nil, &errModuleNotFound{addr: module} default: return nil, fmt.Errorf("error looking up module versions: %s", resp.Status) } diff --git a/registry/errors.go b/registry/errors.go new file mode 100644 index 000000000..b8dcd31e3 --- /dev/null +++ b/registry/errors.go @@ -0,0 +1,23 @@ +package registry + +import ( + "fmt" + + "github.com/hashicorp/terraform/registry/regsrc" +) + +type errModuleNotFound struct { + addr *regsrc.Module +} + +func (e *errModuleNotFound) Error() string { + return fmt.Sprintf("module %s not found", e.addr) +} + +// IsModuleNotFound returns true only if the given error is a "module not found" +// error. This allows callers to recognize this particular error condition +// as distinct from operational errors such as poor network connectivity. +func IsModuleNotFound(err error) bool { + _, ok := err.(*errModuleNotFound) + return ok +} From 74afcb4a7fc7876a0f430ba3f5d6aa1232036d2e Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 14 Feb 2018 14:35:03 -0800 Subject: [PATCH 20/25] configs/configload: some loaders can't install modules Originally the hope was to use the afero filesystem abstraction for all loader operations, but since we install modules using go-getter we cannot (without a lot of refactoring) support vfs for installation. The vfs use-case is for reading configuration from plan zip files anyway, and so we have no real reason to support installation into a vfs. For now at least we will just add the possibility that a loader might not be install-capable. At the moment we have no non-install-capable loaders, but we'll add one later once we get to loading configuration from plan files. --- configs/configload/loader.go | 11 ++++++----- configs/configload/loader_install.go | 19 +++++++++++++++++++ configs/configload/module_mgr.go | 7 +++++++ 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/configs/configload/loader.go b/configs/configload/loader.go index fba244173..06ff27400 100644 --- a/configs/configload/loader.go +++ b/configs/configload/loader.go @@ -59,11 +59,12 @@ func NewLoader(config *Config) (*Loader, error) { ret := &Loader{ parser: parser, modules: moduleMgr{ - FS: afero.Afero{fs}, - Dir: config.ModulesDir, - Services: config.Services, - Creds: config.Creds, - Registry: reg, + FS: afero.Afero{Fs: fs}, + CanInstall: true, + Dir: config.ModulesDir, + Services: config.Services, + Creds: config.Creds, + Registry: reg, }, } diff --git a/configs/configload/loader_install.go b/configs/configload/loader_install.go index 41ef4224d..b3e015239 100644 --- a/configs/configload/loader_install.go +++ b/configs/configload/loader_install.go @@ -38,7 +38,16 @@ import ( // may have wholly or partially completed. Modules must be loaded in order // to find their dependencies, so this function does many of the same checks // as LoadConfig as a side-effect. +// +// This function will panic if called on a loader that cannot install modules. +// Use CanInstallModules to determine if a loader can install modules, or +// refer to the documentation for that method for situations where module +// installation capability is guaranteed. func (l *Loader) InstallModules(rootDir string, upgrade bool, hooks InstallHooks) hcl.Diagnostics { + if !l.CanInstallModules() { + panic(fmt.Errorf("InstallModules called on loader that cannot install modules")) + } + rootMod, diags := l.parser.LoadConfigDir(rootDir) if rootMod == nil { return diags @@ -183,6 +192,16 @@ func (l *Loader) InstallModules(rootDir string, upgrade bool, hooks InstallHooks return diags } +// CanInstallModules returns true if InstallModules can be used with this +// loader. +// +// Loaders created with NewLoader can always install modules. Loaders created +// from plan files (where the configuration is embedded in the plan file itself) +// cannot install modules, because the plan file is read-only. +func (l *Loader) CanInstallModules() bool { + return l.modules.CanInstall +} + func (l *Loader) installLocalModule(req *configs.ModuleRequest, key string, hooks InstallHooks) (*configs.Module, hcl.Diagnostics) { var diags hcl.Diagnostics diff --git a/configs/configload/module_mgr.go b/configs/configload/module_mgr.go index 5856c2ec0..ef17fda7a 100644 --- a/configs/configload/module_mgr.go +++ b/configs/configload/module_mgr.go @@ -10,6 +10,13 @@ import ( type moduleMgr struct { FS afero.Afero + // CanInstall is true for a module manager that can support installation. + // + // This must be set only if FS is an afero.OsFs, because the installer + // (which uses go-getter) is not aware of the virtual filesystem + // abstraction and will always write into the "real" filesystem. + CanInstall bool + // Dir is the path where descendent modules are (or will be) installed. Dir string From d859bacbddee2689c0c2e0e46ae6aedb173a21a6 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 14 Feb 2018 20:13:35 -0800 Subject: [PATCH 21/25] configs/configload: InitDirFromModule This will provide the functionality of "terraform init -from-module=...", which uses the contents of a given module to populate the working directory. This mechanism is intended for installing e.g. examples from Terraform Registry or elsewhere. It's not fully-general since it can't reasonably install a module from a subdir that refers up to a parent directory, but that isn't an issue for all reasonable uses of this option. --- configs/configload/loader_init_from_module.go | 372 ++++++++++++++++++ .../loader_init_from_module_test.go | 92 +++++ configs/configload/loader_install.go | 9 + .../configload/test-fixtures/empty/.gitignore | 0 4 files changed, 473 insertions(+) create mode 100644 configs/configload/loader_init_from_module.go create mode 100644 configs/configload/loader_init_from_module_test.go create mode 100644 configs/configload/test-fixtures/empty/.gitignore diff --git a/configs/configload/loader_init_from_module.go b/configs/configload/loader_init_from_module.go new file mode 100644 index 000000000..473024487 --- /dev/null +++ b/configs/configload/loader_init_from_module.go @@ -0,0 +1,372 @@ +package configload + +import ( + "fmt" + "log" + "os" + "path/filepath" + "sort" + "strings" + + version "github.com/hashicorp/go-version" + "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/terraform/configs" +) + +const initFromModuleRootCallName = "root" +const initFromModuleRootKeyPrefix = initFromModuleRootCallName + "." + +// InitDirFromModule populates the given directory (which must exist and be +// empty) with the contents of the module at the given source address. +// +// It does this by installing the given module and all of its descendent +// modules in a temporary root directory and then copying the installed +// files into suitable locations. As a consequence, any diagnostics it +// generates will reveal the location of this temporary directory to the +// user. +// +// This rather roundabout installation approach is taken to ensure that +// installation proceeds in a manner identical to normal module installation. +// +// If the given source address specifies a sub-directory of the given +// package then only the sub-directory and its descendents will be copied +// into the given root directory, which will cause any relative module +// references using ../ from that module to be unresolvable. Error diagnostics +// are produced in that case, to prompt the user to rewrite the source strings +// to be absolute references to the original remote module. +// +// This can be installed only on a loder that can install modules, and will +// panic otherwise. Use CanInstallModules to determine if this method can be +// used, or refer to the documentation of that method for situations where +// install ability is guaranteed. +func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHooks) hcl.Diagnostics { + var diags hcl.Diagnostics + + // The way this function works is pretty ugly, but we accept it because + // -from-module is a less important case than normal module installation + // and so it's better to keep this ugly complexity out here rather than + // adding even more complexity to the normal module installer. + + // The target directory must exist but be empty. + { + entries, err := l.modules.FS.ReadDir(rootDir) + if err != nil { + if os.IsNotExist(err) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Target directory does not exist", + Detail: fmt.Sprintf("Cannot initialize non-existent directory %s.", rootDir), + }) + } else { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to read target directory", + Detail: fmt.Sprintf("Error reading %s to ensure it is empty: %s.", rootDir, err), + }) + } + return diags + } + haveEntries := false + for _, entry := range entries { + if entry.Name() == "." || entry.Name() == ".." || entry.Name() == ".terraform" { + continue + } + haveEntries = true + } + if haveEntries { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Can't populate non-empty directory", + Detail: fmt.Sprintf("The target directory %s is not empty, so it cannot be initialized with the -from-module=... option.", rootDir), + }) + return diags + } + } + + // We use a hidden sub-loader to manage our inner installation directory, + // but it shares dependencies with the receiver to allow it to access the + // same remote resources and ensure it populates the same source code + // cache in case . + subLoader := &Loader{ + parser: l.parser, + modules: l.modules, // this is a shallow copy, so we can safely mutate below + } + + // Our sub-loader will have its own independent manifest and install + // directory, so we can install with it and know we won't interfere + // with the receiver. + subLoader.modules.manifest = make(moduleManifest) + subLoader.modules.Dir = filepath.Join(rootDir, ".terraform/init-from-module") + + log.Printf("[DEBUG] using a child module loader in %s to initialize working directory from %q", subLoader.modules.Dir, sourceAddr) + + subLoader.modules.FS.RemoveAll(subLoader.modules.Dir) // if this fails then we'll fail on MkdirAll below too + + err := subLoader.modules.FS.MkdirAll(subLoader.modules.Dir, os.ModePerm) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to create temporary directory", + Detail: fmt.Sprintf("Failed to create temporary directory %s: %s.", subLoader.modules.Dir, err), + }) + return diags + } + + fakeFilename := fmt.Sprintf("-from-module=%q", sourceAddr) + fakeRange := hcl.Range{ + Filename: fakeFilename, + Start: hcl.Pos{ + Line: 1, + Column: 1, + Byte: 0, + }, + End: hcl.Pos{ + Line: 1, + Column: len(fakeFilename) + 1, // not accurate if the address contains unicode, but irrelevant since we have no source cache for this anyway + Byte: len(fakeFilename), + }, + } + + // Now we need to create an artificial root module that will seed our + // installation process. + fakeRootModule := &configs.Module{ + ModuleCalls: map[string]*configs.ModuleCall{ + initFromModuleRootCallName: &configs.ModuleCall{ + Name: initFromModuleRootCallName, + + SourceAddr: sourceAddr, + SourceAddrRange: fakeRange, + SourceSet: true, + + DeclRange: fakeRange, + }, + }, + } + + // wrapHooks filters hook notifications to only include Download calls + // and to trim off the initFromModuleRootCallName prefix. We'll produce + // our own Install notifications directly below. + wrapHooks := installHooksInitDir{ + Wrapped: hooks, + } + instDiags := subLoader.installDescendentModules(fakeRootModule, rootDir, true, wrapHooks) + diags = append(diags, instDiags...) + if instDiags.HasErrors() { + return diags + } + + // If all of that succeeded then we'll now migrate what was installed + // into the final directory structure. + modulesDir := l.modules.Dir + err = subLoader.modules.FS.MkdirAll(modulesDir, os.ModePerm) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to create local modules directory", + Detail: fmt.Sprintf("Failed to create modules directory %s: %s.", modulesDir, err), + }) + return diags + } + + manifest := subLoader.modules.manifest + recordKeys := make([]string, 0, len(manifest)) + for k := range manifest { + recordKeys = append(recordKeys, k) + } + sort.Strings(recordKeys) + + for _, recordKey := range recordKeys { + record := manifest[recordKey] + + if record.Key == initFromModuleRootCallName { + // We've found the module the user requested, which we must + // now copy into rootDir so it can be used directly. + log.Printf("[TRACE] copying new root module from %s to %s", record.Dir, rootDir) + err := copyDir(rootDir, record.Dir) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to copy root module", + Detail: fmt.Sprintf("Error copying root module %q from %s to %s: %s.", sourceAddr, record.Dir, rootDir, err), + }) + continue + } + + // We'll try to load the newly-copied module here just so we can + // sniff for any module calls that ../ out of the root directory + // and must thus be rewritten to be absolute addresses again. + // For now we can't do this rewriting automatically, but we'll + // generate an error to help the user do it manually. + mod, _ := l.parser.LoadConfigDir(rootDir) // ignore diagnostics since we're just doing value-add here anyway + for _, mc := range mod.ModuleCalls { + if pathTraversesUp(sourceAddr) { + packageAddr, givenSubdir := splitAddrSubdir(sourceAddr) + newSubdir := filepath.Join(givenSubdir, mc.SourceAddr) + if pathTraversesUp(newSubdir) { + // This should never happen in any reasonable + // configuration since this suggests a path that + // traverses up out of the package root. We'll just + // ignore this, since we'll fail soon enough anyway + // trying to resolve this path when this module is + // loaded. + continue + } + + var newAddr = packageAddr + if newSubdir != "" { + newAddr = fmt.Sprintf("%s//%s", newAddr, filepath.ToSlash(newSubdir)) + } + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Root module references parent directory", + Detail: fmt.Sprintf("The requested module %q refers to a module via its parent directory. To use this as a new root module this source string must be rewritten as a remote source address, such as %q.", sourceAddr, newAddr), + Subject: &mc.SourceAddrRange, + }) + continue + } + } + + l.modules.manifest[""] = moduleRecord{ + Key: "", + Dir: rootDir, + } + continue + } + + if !strings.HasPrefix(record.Key, initFromModuleRootKeyPrefix) { + // Ignore the *real* root module, whose key is empty, since + // we're only interested in the module named "root" and its + // descendents. + continue + } + + newKey := record.Key[len(initFromModuleRootKeyPrefix):] + instPath := filepath.Join(l.modules.Dir, newKey) + tempPath := filepath.Join(subLoader.modules.Dir, record.Key) + + // tempPath won't be present for a module that was installed from + // a relative path, so in that case we just record the installation + // directory and assume it was already copied into place as part + // of its parent. + if _, err := os.Stat(tempPath); err != nil { + if !os.IsNotExist(err) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to stat temporary module install directory", + Detail: fmt.Sprintf("Error from stat %s for module %s: %s.", instPath, newKey, err), + }) + continue + } + + var parentKey string + if lastDot := strings.LastIndexByte(newKey, '.'); lastDot != -1 { + parentKey = newKey[:lastDot] + } else { + parentKey = "" // parent is the root module + } + + parentOld := manifest[initFromModuleRootKeyPrefix+parentKey] + parentNew := l.modules.manifest[parentKey] + + // We need to figure out which portion of our directory is the + // parent package path and which portion is the subdirectory + // under that. + baseDirRel, err := filepath.Rel(parentOld.Dir, record.Dir) + if err != nil { + // Should never happen, because we constructed both directories + // from the same base and so they must have a common prefix. + panic(err) + } + + newDir := filepath.Join(parentNew.Dir, baseDirRel) + log.Printf("[TRACE] relative reference for %s rewritten from %s to %s", newKey, record.Dir, newDir) + newRecord := record // shallow copy + newRecord.Dir = newDir + newRecord.Key = newKey + l.modules.manifest[newKey] = newRecord + hooks.Install(newRecord.Key, newRecord.Version, newRecord.Dir) + continue + } + + err = subLoader.modules.FS.MkdirAll(instPath, os.ModePerm) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to create module install directory", + Detail: fmt.Sprintf("Error creating directory %s for module %s: %s.", instPath, newKey, err), + }) + continue + } + + // We copy rather than "rename" here because renaming between directories + // can be tricky in edge-cases like network filesystems, etc. + log.Printf("[TRACE] copying new module %s from %s to %s", newKey, record.Dir, instPath) + err := copyDir(instPath, tempPath) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to copy descendent module", + Detail: fmt.Sprintf("Error copying module %q from %s to %s: %s.", newKey, tempPath, rootDir, err), + }) + continue + } + + subDir, err := filepath.Rel(tempPath, record.Dir) + if err != nil { + // Should never happen, because we constructed both directories + // from the same base and so they must have a common prefix. + panic(err) + } + + newRecord := record // shallow copy + newRecord.Dir = filepath.Join(instPath, subDir) + newRecord.Key = newKey + l.modules.manifest[newKey] = newRecord + hooks.Install(newRecord.Key, newRecord.Version, newRecord.Dir) + } + + err = l.modules.writeModuleManifestSnapshot() + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to write module manifest", + Detail: fmt.Sprintf("Error writing module manifest: %s.", err), + }) + } + + if !diags.HasErrors() { + // Try to clean up our temporary directory, but don't worry if we don't + // succeed since it shouldn't hurt anything. + subLoader.modules.FS.RemoveAll(subLoader.modules.Dir) + } + + return diags +} + +func pathTraversesUp(path string) bool { + return strings.HasPrefix(filepath.ToSlash(path), "../") +} + +// installHooksInitDir is an adapter wrapper for an InstallHooks that +// does some fakery to make downloads look like they are happening in their +// final locations, rather than in the temporary loader we use. +// +// It also suppresses "Install" calls entirely, since InitDirFromModule +// does its own installation steps after the initial installation pass +// has completed. +type installHooksInitDir struct { + Wrapped InstallHooks + InstallHooksImpl +} + +func (h installHooksInitDir) Download(moduleAddr, packageAddr string, version *version.Version) { + if !strings.HasPrefix(moduleAddr, initFromModuleRootKeyPrefix) { + // We won't announce the root module, since hook implementations + // don't expect to see that and the caller will usually have produced + // its own user-facing notification about what it's doing anyway. + return + } + + trimAddr := moduleAddr[len(initFromModuleRootKeyPrefix):] + h.Wrapped.Download(trimAddr, packageAddr, version) +} diff --git a/configs/configload/loader_init_from_module_test.go b/configs/configload/loader_init_from_module_test.go new file mode 100644 index 000000000..f08540297 --- /dev/null +++ b/configs/configload/loader_init_from_module_test.go @@ -0,0 +1,92 @@ +package configload + +import ( + "os" + "path/filepath" + "strings" + "testing" + + version "github.com/hashicorp/go-version" + "github.com/hashicorp/terraform/configs" +) + +func TestLoaderInitDirFromModule_registry(t *testing.T) { + if os.Getenv("TF_ACC") == "" { + t.Skip("this test accesses registry.terraform.io and github.com; set TF_ACC=1 to run it") + } + + fixtureDir := filepath.Clean("test-fixtures/empty") + loader, done := tempChdirLoader(t, fixtureDir) + defer done() + + hooks := &testInstallHooks{} + + diags := loader.InitDirFromModule(".", "hashicorp/module-installer-acctest/aws//examples/main", hooks) + assertNoDiagnostics(t, diags) + + v := version.Must(version.NewVersion("0.0.1")) + + wantCalls := []testInstallHookCall{ + // The module specified to populate the root directory is not mentioned + // here, because the hook mechanism is defined to talk about descendent + // modules only and so a caller to InitDirFromModule is expected to + // produce its own user-facing announcement about the root module being + // installed. + + // Note that "root" in the following examples is, confusingly, the + // label on the module block in the example we've installed here: + // module "root" { + + { + Name: "Download", + ModuleAddr: "root", + PackageAddr: "hashicorp/module-installer-acctest/aws", + Version: v, + }, + { + Name: "Install", + ModuleAddr: "root", + Version: v, + LocalPath: ".terraform/modules/root/hashicorp-terraform-aws-module-installer-acctest-5e87aff", + }, + { + Name: "Install", + ModuleAddr: "root.child_a", + LocalPath: ".terraform/modules/root/hashicorp-terraform-aws-module-installer-acctest-5e87aff/modules/child_a", + }, + { + Name: "Install", + ModuleAddr: "root.child_a.child_b", + LocalPath: ".terraform/modules/root/hashicorp-terraform-aws-module-installer-acctest-5e87aff/modules/child_b", + }, + } + + if assertResultDeepEqual(t, hooks.Calls, wantCalls) { + return + } + + // Make sure the configuration is loadable now. + // (This ensures that correct information is recorded in the manifest.) + config, loadDiags := loader.LoadConfig(".") + if assertNoDiagnostics(t, loadDiags) { + return + } + + wantTraces := map[string]string{ + "": "in example", + "root": "in root module", + "root.child_a": "in child_a module", + "root.child_a.child_b": "in child_b module", + } + gotTraces := map[string]string{} + config.DeepEach(func(c *configs.Config) { + path := strings.Join(c.Path, ".") + if c.Module.Variables["v"] == nil { + gotTraces[path] = "" + return + } + varDesc := c.Module.Variables["v"].Description + gotTraces[path] = varDesc + }) + assertResultDeepEqual(t, gotTraces, wantTraces) +} diff --git a/configs/configload/loader_install.go b/configs/configload/loader_install.go index b3e015239..b9f6be8dd 100644 --- a/configs/configload/loader_install.go +++ b/configs/configload/loader_install.go @@ -53,6 +53,15 @@ func (l *Loader) InstallModules(rootDir string, upgrade bool, hooks InstallHooks return diags } + instDiags := l.installDescendentModules(rootMod, rootDir, upgrade, hooks) + diags = append(diags, instDiags...) + + return diags +} + +func (l *Loader) installDescendentModules(rootMod *configs.Module, rootDir string, upgrade bool, hooks InstallHooks) hcl.Diagnostics { + var diags hcl.Diagnostics + if hooks == nil { // Use our no-op implementation as a placeholder hooks = InstallHooksImpl{} diff --git a/configs/configload/test-fixtures/empty/.gitignore b/configs/configload/test-fixtures/empty/.gitignore new file mode 100644 index 000000000..e69de29bb From ea868026b85ddeae36b4254c2d1cbcba42fe77a0 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 15 Feb 2018 09:23:07 -0800 Subject: [PATCH 22/25] configs/configload: installer tests inspect their result Previously we were just loading the module and asserting no diagnostics, but that is not really good enough since if we install modules incorrectly it's possible that we are still able to load an empty configuration successfully. Now we'll do some basic inspecetion of the module tree that results from loading what we installed, to ensure that all of the expected modules are present at the right locations in the tree. --- configs/configload/loader_install_test.go | 69 ++++++++++++++++++- .../test-fixtures/go-getter-modules/root.tf | 5 ++ .../local-modules/child_a/child_a.tf | 5 ++ .../local-modules/child_a/child_b/child_b.tf | 5 ++ .../test-fixtures/local-modules/root.tf | 5 ++ .../test-fixtures/registry-modules/root.tf | 6 ++ 6 files changed, 92 insertions(+), 3 deletions(-) diff --git a/configs/configload/loader_install_test.go b/configs/configload/loader_install_test.go index 2be85a6ba..e0e6bdf98 100644 --- a/configs/configload/loader_install_test.go +++ b/configs/configload/loader_install_test.go @@ -3,9 +3,11 @@ package configload import ( "os" "path/filepath" + "strings" "testing" version "github.com/hashicorp/go-version" + "github.com/hashicorp/terraform/configs" ) func TestLoaderInstallModules_local(t *testing.T) { @@ -39,8 +41,25 @@ func TestLoaderInstallModules_local(t *testing.T) { // Make sure the configuration is loadable now. // (This ensures that correct information is recorded in the manifest.) - _, loadDiags := loader.LoadConfig(".") + config, loadDiags := loader.LoadConfig(".") assertNoDiagnostics(t, loadDiags) + + wantTraces := map[string]string{ + "": "in root module", + "child_a": "in child_a module", + "child_a.child_b": "in child_b module", + } + gotTraces := map[string]string{} + config.DeepEach(func(c *configs.Config) { + path := strings.Join(c.Path, ".") + if c.Module.Variables["v"] == nil { + gotTraces[path] = "" + return + } + varDesc := c.Module.Variables["v"].Description + gotTraces[path] = varDesc + }) + assertResultDeepEqual(t, gotTraces, wantTraces) } func TestLoaderInstallModules_registry(t *testing.T) { @@ -136,8 +155,30 @@ func TestLoaderInstallModules_registry(t *testing.T) { // Make sure the configuration is loadable now. // (This ensures that correct information is recorded in the manifest.) - _, loadDiags := loader.LoadConfig(".") + config, loadDiags := loader.LoadConfig(".") assertNoDiagnostics(t, loadDiags) + + wantTraces := map[string]string{ + "": "in local caller for registry-modules", + "acctest_root": "in root module", + "acctest_root.child_a": "in child_a module", + "acctest_root.child_a.child_b": "in child_b module", + "acctest_child_a": "in child_a module", + "acctest_child_a.child_b": "in child_b module", + "acctest_child_b": "in child_b module", + } + gotTraces := map[string]string{} + config.DeepEach(func(c *configs.Config) { + path := strings.Join(c.Path, ".") + if c.Module.Variables["v"] == nil { + gotTraces[path] = "" + return + } + varDesc := c.Module.Variables["v"].Description + gotTraces[path] = varDesc + }) + assertResultDeepEqual(t, gotTraces, wantTraces) + } func TestLoaderInstallModules_goGetter(t *testing.T) { @@ -225,8 +266,30 @@ func TestLoaderInstallModules_goGetter(t *testing.T) { // Make sure the configuration is loadable now. // (This ensures that correct information is recorded in the manifest.) - _, loadDiags := loader.LoadConfig(".") + config, loadDiags := loader.LoadConfig(".") assertNoDiagnostics(t, loadDiags) + + wantTraces := map[string]string{ + "": "in local caller for go-getter-modules", + "acctest_root": "in root module", + "acctest_root.child_a": "in child_a module", + "acctest_root.child_a.child_b": "in child_b module", + "acctest_child_a": "in child_a module", + "acctest_child_a.child_b": "in child_b module", + "acctest_child_b": "in child_b module", + } + gotTraces := map[string]string{} + config.DeepEach(func(c *configs.Config) { + path := strings.Join(c.Path, ".") + if c.Module.Variables["v"] == nil { + gotTraces[path] = "" + return + } + varDesc := c.Module.Variables["v"].Description + gotTraces[path] = varDesc + }) + assertResultDeepEqual(t, gotTraces, wantTraces) + } type testInstallHooks struct { diff --git a/configs/configload/test-fixtures/go-getter-modules/root.tf b/configs/configload/test-fixtures/go-getter-modules/root.tf index 3c92ef1db..9b174a7a5 100644 --- a/configs/configload/test-fixtures/go-getter-modules/root.tf +++ b/configs/configload/test-fixtures/go-getter-modules/root.tf @@ -3,6 +3,11 @@ # ...and expects its v0.0.1 tag to be pointing at the following commit: # d676ab2559d4e0621d59e3c3c4cbb33958ac4608 +variable "v" { + description = "in local caller for go-getter-modules" + default = "" +} + module "acctest_root" { source = "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1" } diff --git a/configs/configload/test-fixtures/local-modules/child_a/child_a.tf b/configs/configload/test-fixtures/local-modules/child_a/child_a.tf index eb2c0044f..68ebb8e40 100644 --- a/configs/configload/test-fixtures/local-modules/child_a/child_a.tf +++ b/configs/configload/test-fixtures/local-modules/child_a/child_a.tf @@ -1,4 +1,9 @@ +variable "v" { + description = "in child_a module" + default = "" +} + module "child_b" { source = "./child_b" } diff --git a/configs/configload/test-fixtures/local-modules/child_a/child_b/child_b.tf b/configs/configload/test-fixtures/local-modules/child_a/child_b/child_b.tf index 4900e9796..e2e220916 100644 --- a/configs/configload/test-fixtures/local-modules/child_a/child_b/child_b.tf +++ b/configs/configload/test-fixtures/local-modules/child_a/child_b/child_b.tf @@ -1,4 +1,9 @@ +variable "v" { + description = "in child_b module" + default = "" +} + output "hello" { value = "Hello from child_b!" } diff --git a/configs/configload/test-fixtures/local-modules/root.tf b/configs/configload/test-fixtures/local-modules/root.tf index 1a6721b5d..3b4c6416d 100644 --- a/configs/configload/test-fixtures/local-modules/root.tf +++ b/configs/configload/test-fixtures/local-modules/root.tf @@ -1,4 +1,9 @@ +variable "v" { + description = "in root module" + default = "" +} + module "child_a" { source = "./child_a" } diff --git a/configs/configload/test-fixtures/registry-modules/root.tf b/configs/configload/test-fixtures/registry-modules/root.tf index 786966494..4b5ad1f1e 100644 --- a/configs/configload/test-fixtures/registry-modules/root.tf +++ b/configs/configload/test-fixtures/registry-modules/root.tf @@ -11,6 +11,12 @@ # 853d03855b3290a3ca491d4c3a7684572dd42237 # (this particular assumption is encoded in the tests that use this fixture) + +variable "v" { + description = "in local caller for registry-modules" + default = "" +} + module "acctest_root" { source = "hashicorp/module-installer-acctest/aws" version = "0.0.1" From 36fb5b52e79c66f6476c0e1ee93da9dea438f47a Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 15 Feb 2018 10:17:36 -0800 Subject: [PATCH 23/25] configs: quoted keywords/references are warnings, not errors In our new loader we are changing certain values in configuration to be naked keywords or references rather than quoted strings as before. Since many of these have been shown in books, tutorials, and our own documentation we will make the old forms generate deprecation warnings rather than errors so that newcomers starting from older documentation can be eased into the new syntax, rather than getting blocked. This will also avoid creating a hard compatibility wall for reusable modules that are already published, allowing them to still be used in spite of these warnings and then fixed when the maintainer is able. --- configs/compat_shim.go | 81 +++++++++++++++++++ configs/depends_on.go | 18 +---- configs/named_values.go | 30 +++---- configs/parser_config_dir_test.go | 4 +- configs/parser_config_test.go | 59 ++++++++++---- configs/provisioner.go | 52 +++++------- configs/resource.go | 13 ++- .../valid-files/resources-dependson-quoted.tf | 8 ++ .../resources-ignorechanges-quoted.tf | 7 ++ .../resources-provisioner-onfailure-quoted.tf | 6 ++ .../resources-provisioner-when-quoted.tf | 6 ++ .../variable-type-quoted.tf | 0 12 files changed, 195 insertions(+), 89 deletions(-) create mode 100644 configs/compat_shim.go create mode 100644 configs/test-fixtures/valid-files/resources-dependson-quoted.tf create mode 100644 configs/test-fixtures/valid-files/resources-ignorechanges-quoted.tf create mode 100644 configs/test-fixtures/valid-files/resources-provisioner-onfailure-quoted.tf create mode 100644 configs/test-fixtures/valid-files/resources-provisioner-when-quoted.tf rename configs/test-fixtures/{invalid-files => valid-files}/variable-type-quoted.tf (100%) diff --git a/configs/compat_shim.go b/configs/compat_shim.go new file mode 100644 index 000000000..69d8b0d0f --- /dev/null +++ b/configs/compat_shim.go @@ -0,0 +1,81 @@ +package configs + +import ( + "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hcl/hclsyntax" +) + +// ------------------------------------------------------------------------- +// Functions in this file are compatibility shims intended to ease conversion +// from the old configuration loader. Any use of these functions that makes +// a change should generate a deprecation warning explaining to the user how +// to update their code for new patterns. +// +// Shims are particularly important for any patterns that have been widely +// documented in books, tutorials, etc. Users will still be starting from +// these examples and we want to help them adopt the latest patterns rather +// than leave them stranded. +// ------------------------------------------------------------------------- + +// shimTraversalInString takes any arbitrary expression and checks if it is +// a quoted string in the native syntax. If it _is_, then it is parsed as a +// traversal and re-wrapped into a synthetic traversal expression and a +// warning is generated. Otherwise, the given expression is just returned +// verbatim. +// +// This function has no effect on expressions from the JSON syntax, since +// traversals in strings are the required pattern in that syntax. +// +// If wantKeyword is set, the generated warning diagnostic will talk about +// keywords rather than references. The behavior is otherwise unchanged, and +// the caller remains responsible for checking that the result is indeed +// a keyword, e.g. using hcl.ExprAsKeyword. +func shimTraversalInString(expr hcl.Expression, wantKeyword bool) (hcl.Expression, hcl.Diagnostics) { + if !exprIsNativeQuotedString(expr) { + return expr, nil + } + + strVal, diags := expr.Value(nil) + if diags.HasErrors() || strVal.IsNull() || !strVal.IsKnown() { + // Since we're not even able to attempt a shim here, we'll discard + // the diagnostics we saw so far and let the caller's own error + // handling take care of reporting the invalid expression. + return expr, nil + } + + // The position handling here isn't _quite_ right because it won't + // take into account any escape sequences in the literal string, but + // it should be close enough for any error reporting to make sense. + srcRange := expr.Range() + startPos := srcRange.Start // copy + startPos.Column++ // skip initial quote + startPos.Byte++ // skip initial quote + + traversal, tDiags := hclsyntax.ParseTraversalAbs( + []byte(strVal.AsString()), + srcRange.Filename, + startPos, + ) + diags = append(diags, tDiags...) + + if wantKeyword { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "Quoted keywords are deprecated", + Detail: "In this context, keywords are expected literally rather than in quotes. Previous versions of Terraform required quotes, but that usage is now deprecated. Remove the quotes surrounding this keyword to silence this warning.", + Subject: &srcRange, + }) + } else { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "Quoted references are deprecated", + Detail: "In this context, references are expected literally rather than in quotes. Previous versions of Terraform required quotes, but that usage is now deprecated. Remove the quotes surrounding this reference to silence this warning.", + Subject: &srcRange, + }) + } + + return &hclsyntax.ScopeTraversalExpr{ + Traversal: traversal, + SrcRange: srcRange, + }, diags +} diff --git a/configs/depends_on.go b/configs/depends_on.go index f20c58167..b1984768f 100644 --- a/configs/depends_on.go +++ b/configs/depends_on.go @@ -1,8 +1,6 @@ package configs import ( - "fmt" - "github.com/hashicorp/hcl2/hcl" ) @@ -11,20 +9,8 @@ func decodeDependsOn(attr *hcl.Attribute) ([]hcl.Traversal, hcl.Diagnostics) { exprs, diags := hcl.ExprList(attr.Expr) for _, expr := range exprs { - // A dependency reference was given as a string literal in the legacy - // configuration language and there are lots of examples out there - // showing that usage, so we'll sniff for that situation here and - // produce a specialized error message for it to help users find - // the new correct form. - if exprIsNativeQuotedString(expr) { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid explicit dependency reference", - Detail: fmt.Sprintf("%s elements must not be given in quotes.", attr.Name), - Subject: attr.Expr.Range().Ptr(), - }) - continue - } + expr, shimDiags := shimTraversalInString(expr, false) + diags = append(diags, shimDiags...) traversal, travDiags := hcl.AbsTraversalForExpr(expr) diags = append(diags, travDiags...) diff --git a/configs/named_values.go b/configs/named_values.go index 42ae4750d..d1d93a57d 100644 --- a/configs/named_values.go +++ b/configs/named_values.go @@ -68,7 +68,10 @@ func decodeVariableBlock(block *hcl.Block) (*Variable, hcl.Diagnostics) { } if attr, exists := content.Attributes["type"]; exists { - switch hcl.ExprAsKeyword(attr.Expr) { + expr, shimDiags := shimTraversalInString(attr.Expr, true) + diags = append(diags, shimDiags...) + + switch hcl.ExprAsKeyword(expr) { case "string": v.TypeHint = TypeHintString case "list": @@ -76,25 +79,12 @@ func decodeVariableBlock(block *hcl.Block) (*Variable, hcl.Diagnostics) { case "map": v.TypeHint = TypeHintMap default: - // In our legacy configuration format these keywords would've been - // provided as quoted strings, so we'll generate a special error - // message for that to help those who find outdated examples and - // would otherwise be confused. - if exprIsNativeQuotedString(attr.Expr) { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid variable type hint", - Detail: "The type hint keyword must not be given in quotes.", - Subject: attr.Expr.Range().Ptr(), - }) - } else { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid variable type hint", - Detail: "The type argument requires one of the following keywords: string, list, or map.", - Subject: attr.Expr.Range().Ptr(), - }) - } + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid variable type hint", + Detail: "The type argument requires one of the following keywords: string, list, or map.", + Subject: expr.Range().Ptr(), + }) } } diff --git a/configs/parser_config_dir_test.go b/configs/parser_config_dir_test.go index 703fc122d..b35f30127 100644 --- a/configs/parser_config_dir_test.go +++ b/configs/parser_config_dir_test.go @@ -60,8 +60,8 @@ func TestParserLoadConfigDirSuccess(t *testing.T) { }) _, diags := parser.LoadConfigDir("mod") - if len(diags) != 0 { - t.Errorf("unexpected diagnostics") + if diags.HasErrors() { + t.Errorf("unexpected error diagnostics") for _, diag := range diags { t.Logf("- %s", diag) } diff --git a/configs/parser_config_test.go b/configs/parser_config_test.go index 2f118933c..ae7c8856d 100644 --- a/configs/parser_config_test.go +++ b/configs/parser_config_test.go @@ -34,8 +34,8 @@ func TestParserLoadConfigFileSuccess(t *testing.T) { }) _, diags := parser.LoadConfigFile(name) - if len(diags) != 0 { - t.Errorf("unexpected diagnostics") + if diags.HasErrors() { + t.Errorf("unexpected error diagnostics") for _, diag := range diags { t.Logf("- %s", diag) } @@ -85,38 +85,65 @@ func TestParserLoadConfigFileFailure(t *testing.T) { // file produces the expected diagnostic summary. func TestParserLoadConfigFileFailureMessages(t *testing.T) { tests := []struct { - Filename string - WantError string + Filename string + WantSeverity hcl.DiagnosticSeverity + WantDiag string }{ { - "data-resource-lifecycle.tf", + "invalid-files/data-resource-lifecycle.tf", + hcl.DiagError, "Unsupported lifecycle block", }, { - "variable-type-unknown.tf", + "invalid-files/variable-type-unknown.tf", + hcl.DiagError, "Invalid variable type hint", }, { - "variable-type-quoted.tf", - "Invalid variable type hint", + "valid-files/variable-type-quoted.tf", + hcl.DiagWarning, + "Quoted keywords are deprecated", }, { - "unexpected-attr.tf", + "invalid-files/unexpected-attr.tf", + hcl.DiagError, "Unsupported attribute", }, { - "unexpected-block.tf", + "invalid-files/unexpected-block.tf", + hcl.DiagError, "Unsupported block type", }, { - "resource-lifecycle-badbool.tf", + "invalid-files/resource-lifecycle-badbool.tf", + hcl.DiagError, "Unsuitable value type", }, + { + "valid-files/resources-dependson-quoted.tf", + hcl.DiagWarning, + "Quoted references are deprecated", + }, + { + "valid-files/resources-ignorechanges-quoted.tf", + hcl.DiagWarning, + "Quoted references are deprecated", + }, + { + "valid-files/resources-provisioner-when-quoted.tf", + hcl.DiagWarning, + "Quoted keywords are deprecated", + }, + { + "valid-files/resources-provisioner-onfailure-quoted.tf", + hcl.DiagWarning, + "Quoted keywords are deprecated", + }, } for _, test := range tests { t.Run(test.Filename, func(t *testing.T) { - src, err := ioutil.ReadFile(filepath.Join("test-fixtures/invalid-files", test.Filename)) + src, err := ioutil.ReadFile(filepath.Join("test-fixtures", test.Filename)) if err != nil { t.Fatal(err) } @@ -133,11 +160,11 @@ func TestParserLoadConfigFileFailureMessages(t *testing.T) { } return } - if diags[0].Severity != hcl.DiagError { - t.Errorf("Wrong diagnostic severity %s; want %s", diags[0].Severity, hcl.DiagError) + if diags[0].Severity != test.WantSeverity { + t.Errorf("Wrong diagnostic severity %#v; want %#v", diags[0].Severity, test.WantSeverity) } - if diags[0].Summary != test.WantError { - t.Errorf("Wrong diagnostic summary\ngot: %s\nwant: %s", diags[0].Summary, test.WantError) + if diags[0].Summary != test.WantDiag { + t.Errorf("Wrong diagnostic summary\ngot: %s\nwant: %s", diags[0].Summary, test.WantDiag) } }) } diff --git a/configs/provisioner.go b/configs/provisioner.go index 03c14e16c..843f4c0f9 100644 --- a/configs/provisioner.go +++ b/configs/provisioner.go @@ -33,52 +33,40 @@ func decodeProvisionerBlock(block *hcl.Block) (*Provisioner, hcl.Diagnostics) { pv.Config = config if attr, exists := content.Attributes["when"]; exists { - switch hcl.ExprAsKeyword(attr.Expr) { + expr, shimDiags := shimTraversalInString(attr.Expr, true) + diags = append(diags, shimDiags...) + + switch hcl.ExprAsKeyword(expr) { case "create": pv.When = ProvisionerWhenCreate case "destroy": pv.When = ProvisionerWhenDestroy default: - if exprIsNativeQuotedString(attr.Expr) { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid \"when\" keyword", - Detail: "The \"when\" argument keyword must not be given in quotes.", - Subject: attr.Expr.Range().Ptr(), - }) - } else { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid \"when\" keyword", - Detail: "The \"when\" argument requires one of the following keywords: create or destroy.", - Subject: attr.Expr.Range().Ptr(), - }) - } + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid \"when\" keyword", + Detail: "The \"when\" argument requires one of the following keywords: create or destroy.", + Subject: expr.Range().Ptr(), + }) } } if attr, exists := content.Attributes["on_failure"]; exists { - switch hcl.ExprAsKeyword(attr.Expr) { + expr, shimDiags := shimTraversalInString(attr.Expr, true) + diags = append(diags, shimDiags...) + + switch hcl.ExprAsKeyword(expr) { case "continue": pv.OnFailure = ProvisionerOnFailureContinue case "fail": pv.OnFailure = ProvisionerOnFailureFail default: - if exprIsNativeQuotedString(attr.Expr) { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid \"on_failure\" keyword", - Detail: "The \"on_failure\" argument keyword must not be given in quotes.", - Subject: attr.Expr.Range().Ptr(), - }) - } else { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid \"on_failure\" keyword", - Detail: "The \"on_failure\" argument requires one of the following keywords: continue or fail.", - Subject: attr.Expr.Range().Ptr(), - }) - } + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid \"on_failure\" keyword", + Detail: "The \"on_failure\" argument requires one of the following keywords: continue or fail.", + Subject: attr.Expr.Range().Ptr(), + }) } } diff --git a/configs/resource.go b/configs/resource.go index 3892c86b9..1fa16553f 100644 --- a/configs/resource.go +++ b/configs/resource.go @@ -122,6 +122,9 @@ func decodeResourceBlock(block *hcl.Block) (*ManagedResource, hcl.Diagnostics) { diags = append(diags, listDiags...) for _, expr := range exprs { + expr, shimDiags := shimTraversalInString(expr, false) + diags = append(diags, shimDiags...) + traversal, travDiags := hcl.RelTraversalForExpr(expr) diags = append(diags, travDiags...) if len(traversal) != 0 { @@ -257,7 +260,11 @@ type ProviderConfigRef struct { func decodeProviderConfigRef(attr *hcl.Attribute) (*ProviderConfigRef, hcl.Diagnostics) { var diags hcl.Diagnostics - traversal, travDiags := hcl.AbsTraversalForExpr(attr.Expr) + + expr, shimDiags := shimTraversalInString(attr.Expr, false) + diags = append(diags, shimDiags...) + + traversal, travDiags := hcl.AbsTraversalForExpr(expr) // AbsTraversalForExpr produces only generic errors, so we'll discard // the errors given and produce our own with extra context. If we didn't @@ -277,7 +284,7 @@ func decodeProviderConfigRef(attr *hcl.Attribute) (*ProviderConfigRef, hcl.Diagn Severity: hcl.DiagError, Summary: "Invalid provider configuration reference", Detail: "A provider configuration reference must not be given in quotes.", - Subject: attr.Expr.Range().Ptr(), + Subject: expr.Range().Ptr(), }) return nil, diags } @@ -286,7 +293,7 @@ func decodeProviderConfigRef(attr *hcl.Attribute) (*ProviderConfigRef, hcl.Diagn Severity: hcl.DiagError, Summary: "Invalid provider configuration reference", Detail: fmt.Sprintf("The %s argument requires a provider type name, optionally followed by a period and then a configuration alias.", attr.Name), - Subject: attr.Expr.Range().Ptr(), + Subject: expr.Range().Ptr(), }) return nil, diags } diff --git a/configs/test-fixtures/valid-files/resources-dependson-quoted.tf b/configs/test-fixtures/valid-files/resources-dependson-quoted.tf new file mode 100644 index 000000000..3bf188f19 --- /dev/null +++ b/configs/test-fixtures/valid-files/resources-dependson-quoted.tf @@ -0,0 +1,8 @@ +resource "aws_security_group" "firewall" { +} + +resource "aws_instance" "web" { + depends_on = [ + "aws_security_group.firewall", + ] +} diff --git a/configs/test-fixtures/valid-files/resources-ignorechanges-quoted.tf b/configs/test-fixtures/valid-files/resources-ignorechanges-quoted.tf new file mode 100644 index 000000000..cba5be59d --- /dev/null +++ b/configs/test-fixtures/valid-files/resources-ignorechanges-quoted.tf @@ -0,0 +1,7 @@ +resource "aws_instance" "web" { + lifecycle { + ignore_changes = [ + "ami", + ] + } +} diff --git a/configs/test-fixtures/valid-files/resources-provisioner-onfailure-quoted.tf b/configs/test-fixtures/valid-files/resources-provisioner-onfailure-quoted.tf new file mode 100644 index 000000000..dcec1eb08 --- /dev/null +++ b/configs/test-fixtures/valid-files/resources-provisioner-onfailure-quoted.tf @@ -0,0 +1,6 @@ +resource "aws_security_group" "firewall" { + provisioner "local-exec" { + command = "echo hello" + on_failure = "continue" + } +} diff --git a/configs/test-fixtures/valid-files/resources-provisioner-when-quoted.tf b/configs/test-fixtures/valid-files/resources-provisioner-when-quoted.tf new file mode 100644 index 000000000..6a66b085f --- /dev/null +++ b/configs/test-fixtures/valid-files/resources-provisioner-when-quoted.tf @@ -0,0 +1,6 @@ +resource "aws_security_group" "firewall" { + provisioner "local-exec" { + command = "echo hello" + when = "destroy" + } +} diff --git a/configs/test-fixtures/invalid-files/variable-type-quoted.tf b/configs/test-fixtures/valid-files/variable-type-quoted.tf similarity index 100% rename from configs/test-fixtures/invalid-files/variable-type-quoted.tf rename to configs/test-fixtures/valid-files/variable-type-quoted.tf From c5f5340b151a544bd9d335778b259e5e2f60c99f Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 15 Feb 2018 11:14:29 -0800 Subject: [PATCH 24/25] configs: Additional guidance in doc.go There's quite a lot in this package, so hopefully this additional paragraph will help readers get oriented. --- configs/doc.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configs/doc.go b/configs/doc.go index 9688b36cd..f01eb79f4 100644 --- a/configs/doc.go +++ b/configs/doc.go @@ -11,4 +11,9 @@ // of types from the HCL API, including raw HCL diagnostic messages. Such // diagnostics can be converted into Terraform-flavored diagnostics, if needed, // using functions in the sibling package tfdiags. +// +// The Parser type is the main entry-point into this package. The LoadConfigDir +// method can be used to load a single module directory, and then a full +// configuration (including any descendent modules) can be produced using +// the top-level BuildConfig method. package configs From 4fa8c16ead14be4496e5550b31415dcb9d160316 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 15 Feb 2018 11:25:06 -0800 Subject: [PATCH 25/25] configs: support ignore_changes wildcards The initial pass of implementation here missed the special case where ignore_changes can, in the old parser, be set to ["*"] to ignore changes to all attributes. Since that syntax is awkward and non-obvious, our new decoder will instead expect ignore_changes = all, using HCL2's capability to interpret an expression as a literal keyword. For compatibility with old configurations we will still accept the ["*"] form but emit a deprecation warning to encourage moving to the new form. --- configs/compat_shim.go | 18 +++++ configs/parser_config_test.go | 10 +++ configs/resource.go | 66 ++++++++++++++++--- .../resources-ignorechanges-all-legacymix.tf | 5 ++ .../resources-ignorechanges-all-legacy.tf | 5 ++ ...resources-ignorechanges-all-legacy.tf.json | 11 ++++ .../resources-ignorechanges-all.tf | 5 ++ .../resources-ignorechanges-all.tf.json | 11 ++++ 8 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 configs/test-fixtures/invalid-files/resources-ignorechanges-all-legacymix.tf create mode 100644 configs/test-fixtures/valid-files/resources-ignorechanges-all-legacy.tf create mode 100644 configs/test-fixtures/valid-files/resources-ignorechanges-all-legacy.tf.json create mode 100644 configs/test-fixtures/valid-files/resources-ignorechanges-all.tf create mode 100644 configs/test-fixtures/valid-files/resources-ignorechanges-all.tf.json diff --git a/configs/compat_shim.go b/configs/compat_shim.go index 69d8b0d0f..2f803b3fa 100644 --- a/configs/compat_shim.go +++ b/configs/compat_shim.go @@ -3,6 +3,7 @@ package configs import ( "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/hcl2/hcl/hclsyntax" + "github.com/zclconf/go-cty/cty" ) // ------------------------------------------------------------------------- @@ -79,3 +80,20 @@ func shimTraversalInString(expr hcl.Expression, wantKeyword bool) (hcl.Expressio SrcRange: srcRange, }, diags } + +// shimIsIgnoreChangesStar returns true if the given expression seems to be +// a string literal whose value is "*". This is used to support a legacy +// form of ignore_changes = all . +// +// This function does not itself emit any diagnostics, so it's the caller's +// responsibility to emit a warning diagnostic when this function returns true. +func shimIsIgnoreChangesStar(expr hcl.Expression) bool { + val, valDiags := expr.Value(nil) + if valDiags.HasErrors() { + return false + } + if val.Type() != cty.String || val.IsNull() || !val.IsKnown() { + return false + } + return val.AsString() == "*" +} diff --git a/configs/parser_config_test.go b/configs/parser_config_test.go index ae7c8856d..ff90d5eba 100644 --- a/configs/parser_config_test.go +++ b/configs/parser_config_test.go @@ -129,6 +129,16 @@ func TestParserLoadConfigFileFailureMessages(t *testing.T) { hcl.DiagWarning, "Quoted references are deprecated", }, + { + "valid-files/resources-ignorechanges-all-legacy.tf", + hcl.DiagWarning, + "Deprecated ignore_changes wildcard", + }, + { + "valid-files/resources-ignorechanges-all-legacy.tf.json", + hcl.DiagWarning, + "Deprecated ignore_changes wildcard", + }, { "valid-files/resources-provisioner-when-quoted.tf", hcl.DiagWarning, diff --git a/configs/resource.go b/configs/resource.go index 1fa16553f..e6c9ca2af 100644 --- a/configs/resource.go +++ b/configs/resource.go @@ -26,6 +26,7 @@ type ManagedResource struct { CreateBeforeDestroy bool PreventDestroy bool IgnoreChanges []hcl.Traversal + IgnoreAllChanges bool CreateBeforeDestroySet bool PreventDestroySet bool @@ -118,19 +119,66 @@ func decodeResourceBlock(block *hcl.Block) (*ManagedResource, hcl.Diagnostics) { } if attr, exists := lcContent.Attributes["ignore_changes"]; exists { - exprs, listDiags := hcl.ExprList(attr.Expr) - diags = append(diags, listDiags...) - for _, expr := range exprs { - expr, shimDiags := shimTraversalInString(expr, false) - diags = append(diags, shimDiags...) + // ignore_changes can either be a list of relative traversals + // or it can be just the keyword "all" to ignore changes to this + // resource entirely. + // ignore_changes = [ami, instance_type] + // ignore_changes = all + // We also allow two legacy forms for compatibility with earlier + // versions: + // ignore_changes = ["ami", "instance_type"] + // ignore_changes = ["*"] - traversal, travDiags := hcl.RelTraversalForExpr(expr) - diags = append(diags, travDiags...) - if len(traversal) != 0 { - r.IgnoreChanges = append(r.IgnoreChanges, traversal) + kw := hcl.ExprAsKeyword(attr.Expr) + + switch { + case kw == "all": + r.IgnoreAllChanges = true + default: + exprs, listDiags := hcl.ExprList(attr.Expr) + diags = append(diags, listDiags...) + + var ignoreAllRange hcl.Range + + for _, expr := range exprs { + + // our expr might be the literal string "*", which + // we accept as a deprecated way of saying "all". + if shimIsIgnoreChangesStar(expr) { + r.IgnoreAllChanges = true + ignoreAllRange = expr.Range() + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "Deprecated ignore_changes wildcard", + Detail: "The [\"*\"] form of ignore_changes wildcard is reprecated. Use \"ignore_changes = all\" to ignore changes to all attributes.", + Subject: attr.Expr.Range().Ptr(), + }) + continue + } + + expr, shimDiags := shimTraversalInString(expr, false) + diags = append(diags, shimDiags...) + + traversal, travDiags := hcl.RelTraversalForExpr(expr) + diags = append(diags, travDiags...) + if len(traversal) != 0 { + r.IgnoreChanges = append(r.IgnoreChanges, traversal) + } } + + if r.IgnoreAllChanges && len(r.IgnoreChanges) != 0 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid ignore_changes ruleset", + Detail: "Cannot mix wildcard string \"*\" with non-wildcard references.", + Subject: &ignoreAllRange, + Context: attr.Expr.Range().Ptr(), + }) + } + } + } case "connection": diff --git a/configs/test-fixtures/invalid-files/resources-ignorechanges-all-legacymix.tf b/configs/test-fixtures/invalid-files/resources-ignorechanges-all-legacymix.tf new file mode 100644 index 000000000..9557379aa --- /dev/null +++ b/configs/test-fixtures/invalid-files/resources-ignorechanges-all-legacymix.tf @@ -0,0 +1,5 @@ +resource "aws_instance" "web" { + lifecycle { + ignore_changes = ["*", "foo"] + } +} diff --git a/configs/test-fixtures/valid-files/resources-ignorechanges-all-legacy.tf b/configs/test-fixtures/valid-files/resources-ignorechanges-all-legacy.tf new file mode 100644 index 000000000..6b5e61a9c --- /dev/null +++ b/configs/test-fixtures/valid-files/resources-ignorechanges-all-legacy.tf @@ -0,0 +1,5 @@ +resource "aws_instance" "web" { + lifecycle { + ignore_changes = ["*"] + } +} diff --git a/configs/test-fixtures/valid-files/resources-ignorechanges-all-legacy.tf.json b/configs/test-fixtures/valid-files/resources-ignorechanges-all-legacy.tf.json new file mode 100644 index 000000000..5502dcd50 --- /dev/null +++ b/configs/test-fixtures/valid-files/resources-ignorechanges-all-legacy.tf.json @@ -0,0 +1,11 @@ +{ + "resource": { + "aws_instance": { + "web": { + "lifecycle": { + "ignore_changes": ["*"] + } + } + } + } +} diff --git a/configs/test-fixtures/valid-files/resources-ignorechanges-all.tf b/configs/test-fixtures/valid-files/resources-ignorechanges-all.tf new file mode 100644 index 000000000..32cd23288 --- /dev/null +++ b/configs/test-fixtures/valid-files/resources-ignorechanges-all.tf @@ -0,0 +1,5 @@ +resource "aws_instance" "web" { + lifecycle { + ignore_changes = all + } +} diff --git a/configs/test-fixtures/valid-files/resources-ignorechanges-all.tf.json b/configs/test-fixtures/valid-files/resources-ignorechanges-all.tf.json new file mode 100644 index 000000000..c22020826 --- /dev/null +++ b/configs/test-fixtures/valid-files/resources-ignorechanges-all.tf.json @@ -0,0 +1,11 @@ +{ + "resource": { + "aws_instance": { + "web": { + "lifecycle": { + "ignore_changes": "all" + } + } + } + } +}