Merge pull request #23105 from hashicorp/f-tfignore

.terraformignore support

backend/remote/backend_plan.go

@@ -164,10 +164,12 @@ func (b *Remote) plan(stopCtx, cancelCtx context.Context, op *backend.Operation,
 The remote workspace is configured to work with configuration at
 %s relative to the target repository.
 
-Therefore Terraform will upload the full contents of the following directory
-to capture the filesystem context the remote workspace expects:
+Terraform will upload the contents of the following directory,
+excluding files or directories as defined by a .terraformignore file
+at %s/.terraformignore (if it is present),
+in order to capture the filesystem context the remote workspace expects:
     %s
-`), w.WorkingDirectory, configDir) + "\n")
+`), w.WorkingDirectory, configDir, configDir) + "\n")
 			}
 		}
 

go.mod

@@ -64,7 +64,7 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.5.2
 	github.com/hashicorp/go-rootcerts v1.0.0
 	github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 // indirect
-	github.com/hashicorp/go-tfe v0.3.23
+	github.com/hashicorp/go-tfe v0.3.25
 	github.com/hashicorp/go-uuid v1.0.1
 	github.com/hashicorp/go-version v1.2.0
 	github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f

go.sum

@@ -197,12 +197,12 @@ github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82k
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
 github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I=
-github.com/hashicorp/go-slug v0.3.0 h1:L0c+AvH/J64iMNF4VqRaRku2DMTEuHioPVS7kMjWIU8=
-github.com/hashicorp/go-slug v0.3.0/go.mod h1:I5tq5Lv0E2xcNXNkmx7BSfzi1PsJ2cNjs3cC3LwyhK8=
+github.com/hashicorp/go-slug v0.4.0 h1:YSz3afoEZZJVVB46NITf0+opd2cHpaYJ1XSojOyP0x8=
+github.com/hashicorp/go-slug v0.4.0/go.mod h1:I5tq5Lv0E2xcNXNkmx7BSfzi1PsJ2cNjs3cC3LwyhK8=
 github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 h1:7YOlAIO2YWnJZkQp7B5eFykaIY7C9JndqAFQyVV5BhM=
 github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
-github.com/hashicorp/go-tfe v0.3.23 h1:kd9hlFQvGubNF/CpF7T5AP/xU8uLUq8ANbI5xRDVSms=
-github.com/hashicorp/go-tfe v0.3.23/go.mod h1:SuPHR+OcxvzBZNye7nGPfwZTEyd3rWPfLVbCgyZPezM=
+github.com/hashicorp/go-tfe v0.3.25 h1:4rPk/9rSYuRoujKk5FsxSvtC/AjJCQphLS/57yr6wUM=
+github.com/hashicorp/go-tfe v0.3.25/go.mod h1:IJQ30WzRajD/W0Z8SY4lhuoOX8h5saTe95t80z8hRsk=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=

vendor/github.com/hashicorp/go-slug/README.md

@@ -31,7 +31,7 @@ package main
 
 import (
 	"bytes"
-	"ioutil"
+	"io/ioutil"
 	"log"
 	"os"
 
@@ -40,11 +40,11 @@ import (
 
 func main() {
 	// First create a buffer for storing the slug.
-	slug := bytes.NewBuffer(nil)
+	buf := bytes.NewBuffer(nil)
 
 	// Then call the Pack function with a directory path containing the
 	// configuration files and an io.Writer to write the slug to.
-	if _, err := Pack("test-fixtures/archive-dir", slug); err != nil {
+	if _, err := slug.Pack("testdata/archive-dir", buf, false); err != nil {
 		log.Fatal(err)
 	}
 
@@ -58,7 +58,7 @@ func main() {
 	// Unpacking a slug is done by calling the Unpack function with an
 	// io.Reader to read the slug from and a directory path of an existing
 	// directory to store the unpacked configuration files.
-	if err := Unpack(slug, dst); err != nil {
+	if err := slug.Unpack(buf, dst); err != nil {
 		log.Fatal(err)
 	}
 }

vendor/github.com/hashicorp/go-slug/slug.go

@@ -33,11 +33,15 @@ func Pack(src string, w io.Writer, dereference bool) (*Meta, error) {
 	// Tar the file contents.
 	tarW := tar.NewWriter(gzipW)
 
+	// Load the ignore rule configuration, which will use
+	// defaults if no .terraformignore is configured
+	ignoreRules := parseIgnoreFile(src)
+
 	// Track the metadata details as we go.
 	meta := &Meta{}
 
 	// Walk the tree of files.
-	err := filepath.Walk(src, packWalkFn(src, src, src, tarW, meta, dereference))
+	err := filepath.Walk(src, packWalkFn(src, src, src, tarW, meta, dereference, ignoreRules))
 	if err != nil {
 		return nil, err
 	}
@@ -55,17 +59,12 @@ func Pack(src string, w io.Writer, dereference bool) (*Meta, error) {
 	return meta, nil
 }
 
-func packWalkFn(root, src, dst string, tarW *tar.Writer, meta *Meta, dereference bool) filepath.WalkFunc {
+func packWalkFn(root, src, dst string, tarW *tar.Writer, meta *Meta, dereference bool, ignoreRules []rule) filepath.WalkFunc {
 	return func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
-		// Skip the .git directory.
-		if info.IsDir() && info.Name() == ".git" {
-			return filepath.SkipDir
-		}
-
 		// Get the relative path from the current src directory.
 		subpath, err := filepath.Rel(src, path)
 		if err != nil {
@@ -75,20 +74,16 @@ func packWalkFn(root, src, dst string, tarW *tar.Writer, meta *Meta, dereference
 			return nil
 		}
 
-		// Ignore the .terraform directory itself.
-		if info.IsDir() && info.Name() == ".terraform" {
+		if m := matchIgnoreRule(subpath, ignoreRules); m {
 			return nil
 		}
 
-		// Ignore any files in the .terraform directory.
-		if !info.IsDir() && filepath.Dir(subpath) == ".terraform" {
-			return nil
-		}
-
-		// Skip .terraform subdirectories, except for the modules subdirectory.
-		if strings.HasPrefix(subpath, ".terraform"+string(filepath.Separator)) &&
-			!strings.HasPrefix(subpath, filepath.Clean(".terraform/modules")) {
-			return filepath.SkipDir
+		// Catch directories so we don't end up with empty directories,
+		// the files are ignored correctly
+		if info.IsDir() {
+			if m := matchIgnoreRule(subpath+string(os.PathSeparator), ignoreRules); m {
+				return nil
+			}
 		}
 
 		// Get the relative path from the initial root directory.
@@ -159,7 +154,7 @@ func packWalkFn(root, src, dst string, tarW *tar.Writer, meta *Meta, dereference
 			// If the target is a directory we can recurse into the target
 			// directory by calling the packWalkFn with updated arguments.
 			if info.IsDir() {
-				return filepath.Walk(target, packWalkFn(root, target, path, tarW, meta, dereference))
+				return filepath.Walk(target, packWalkFn(root, target, path, tarW, meta, dereference, ignoreRules))
 			}
 
 			// Dereference this symlink by updating the header with the target file

vendor/github.com/hashicorp/go-slug/terraformignore.go

@@ -0,0 +1,225 @@
+package slug
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"text/scanner"
+)
+
+func parseIgnoreFile(rootPath string) []rule {
+	// Look for .terraformignore at our root path/src
+	file, err := os.Open(filepath.Join(rootPath, ".terraformignore"))
+	defer file.Close()
+
+	// If there's any kind of file error, punt and use the default ignore patterns
+	if err != nil {
+		// Only show the error debug if an error *other* than IsNotExist
+		if !os.IsNotExist(err) {
+			fmt.Fprintf(os.Stderr, "Error reading .terraformignore, default exclusions will apply: %v \n", err)
+		}
+		return defaultExclusions
+	}
+	return readRules(file)
+}
+
+func readRules(input io.Reader) []rule {
+	rules := defaultExclusions
+	scanner := bufio.NewScanner(input)
+	scanner.Split(bufio.ScanLines)
+
+	for scanner.Scan() {
+		pattern := scanner.Text()
+		// Ignore blank lines
+		if len(pattern) == 0 {
+			continue
+		}
+		// Trim spaces
+		pattern = strings.TrimSpace(pattern)
+		// Ignore comments
+		if pattern[0] == '#' {
+			continue
+		}
+		// New rule structure
+		rule := rule{}
+		// Exclusions
+		if pattern[0] == '!' {
+			rule.excluded = true
+			pattern = pattern[1:]
+		}
+		// If it is a directory, add ** so we catch descendants
+		if pattern[len(pattern)-1] == os.PathSeparator {
+			pattern = pattern + "**"
+		}
+		// If it starts with /, it is absolute
+		if pattern[0] == os.PathSeparator {
+			pattern = pattern[1:]
+		} else {
+			// Otherwise prepend **/
+			pattern = "**" + string(os.PathSeparator) + pattern
+		}
+		rule.val = pattern
+		rule.dirs = strings.Split(pattern, string(os.PathSeparator))
+		rules = append(rules, rule)
+	}
+
+	if err := scanner.Err(); err != nil {
+		fmt.Fprintf(os.Stderr, "Error reading .terraformignore, default exclusions will apply: %v \n", err)
+		return defaultExclusions
+	}
+	return rules
+}
+
+func matchIgnoreRule(path string, rules []rule) bool {
+	matched := false
+	path = filepath.FromSlash(path)
+	for _, rule := range rules {
+		match, _ := rule.match(path)
+
+		if match {
+			matched = !rule.excluded
+		}
+	}
+
+	if matched {
+		debug(true, path, "Skipping excluded path:", path)
+	}
+
+	return matched
+}
+
+type rule struct {
+	val      string         // the value of the rule itself
+	excluded bool           // ! is present, an exclusion rule
+	dirs     []string       // directories of the rule
+	regex    *regexp.Regexp // regular expression to match for the rule
+}
+
+func (r *rule) match(path string) (bool, error) {
+	if r.regex == nil {
+		if err := r.compile(); err != nil {
+			return false, filepath.ErrBadPattern
+		}
+	}
+
+	b := r.regex.MatchString(path)
+	debug(false, path, path, r.regex, b)
+	return b, nil
+}
+
+func (r *rule) compile() error {
+	regStr := "^"
+	pattern := r.val
+	// Go through the pattern and convert it to a regexp.
+	// Use a scanner to support utf-8 chars.
+	var scan scanner.Scanner
+	scan.Init(strings.NewReader(pattern))
+
+	sl := string(os.PathSeparator)
+	escSL := sl
+	if sl == `\` {
+		escSL += `\`
+	}
+
+	for scan.Peek() != scanner.EOF {
+		ch := scan.Next()
+		if ch == '*' {
+			if scan.Peek() == '*' {
+				// is some flavor of "**"
+				scan.Next()
+
+				// Treat **/ as ** so eat the "/"
+				if string(scan.Peek()) == sl {
+					scan.Next()
+				}
+
+				if scan.Peek() == scanner.EOF {
+					// is "**EOF" - to align with .gitignore just accept all
+					regStr += ".*"
+				} else {
+					// is "**"
+					// Note that this allows for any # of /'s (even 0) because
+					// the .* will eat everything, even /'s
+					regStr += "(.*" + escSL + ")?"
+				}
+			} else {
+				// is "*" so map it to anything but "/"
+				regStr += "[^" + escSL + "]*"
+			}
+		} else if ch == '?' {
+			// "?" is any char except "/"
+			regStr += "[^" + escSL + "]"
+		} else if ch == '.' || ch == '$' {
+			// Escape some regexp special chars that have no meaning
+			// in golang's filepath.Match
+			regStr += `\` + string(ch)
+		} else if ch == '\\' {
+			// escape next char. Note that a trailing \ in the pattern
+			// will be left alone (but need to escape it)
+			if sl == `\` {
+				// On windows map "\" to "\\", meaning an escaped backslash,
+				// and then just continue because filepath.Match on
+				// Windows doesn't allow escaping at all
+				regStr += escSL
+				continue
+			}
+			if scan.Peek() != scanner.EOF {
+				regStr += `\` + string(scan.Next())
+			} else {
+				regStr += `\`
+			}
+		} else {
+			regStr += string(ch)
+		}
+	}
+
+	regStr += "$"
+	re, err := regexp.Compile(regStr)
+	if err != nil {
+		return err
+	}
+
+	r.regex = re
+	return nil
+}
+
+/*
+	Default rules as they would appear in .terraformignore:
+	.git/
+	.terraform/
+	!.terraform/modules/
+*/
+
+var defaultExclusions = []rule{
+	{
+		val:      "**/.git/**",
+		excluded: false,
+	},
+	{
+		val:      "**/.terraform/**",
+		excluded: false,
+	},
+	{
+		val:      "**/.terraform/modules/**",
+		excluded: true,
+	},
+}
+
+func debug(printAll bool, path string, message ...interface{}) {
+	logLevel := os.Getenv("TF_IGNORE") == "trace"
+	debugPath := os.Getenv("TF_IGNORE_DEBUG")
+	isPath := debugPath != ""
+	if isPath {
+		isPath = strings.Contains(path, debugPath)
+	}
+
+	if logLevel {
+		if printAll || isPath {
+			fmt.Println(message...)
+		}
+	}
+}

vendor/github.com/hashicorp/go-tfe/go.mod

@@ -5,7 +5,7 @@ require (
 	github.com/google/go-querystring v1.0.0
 	github.com/hashicorp/go-cleanhttp v0.5.0
 	github.com/hashicorp/go-retryablehttp v0.5.2
-	github.com/hashicorp/go-slug v0.3.0
+	github.com/hashicorp/go-slug v0.4.0
 	github.com/hashicorp/go-uuid v1.0.1
 	github.com/stretchr/testify v1.3.0
 	github.com/svanharmelen/jsonapi v0.0.0-20180618144545-0c0828c3f16d

vendor/github.com/hashicorp/go-tfe/go.sum

@@ -8,8 +8,8 @@ github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6K
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-retryablehttp v0.5.2 h1:AoISa4P4IsW0/m4T6St8Yw38gTl5GtBAgfkhYh1xAz4=
 github.com/hashicorp/go-retryablehttp v0.5.2/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
-github.com/hashicorp/go-slug v0.3.0 h1:L0c+AvH/J64iMNF4VqRaRku2DMTEuHioPVS7kMjWIU8=
-github.com/hashicorp/go-slug v0.3.0/go.mod h1:I5tq5Lv0E2xcNXNkmx7BSfzi1PsJ2cNjs3cC3LwyhK8=
+github.com/hashicorp/go-slug v0.4.0 h1:YSz3afoEZZJVVB46NITf0+opd2cHpaYJ1XSojOyP0x8=
+github.com/hashicorp/go-slug v0.4.0/go.mod h1:I5tq5Lv0E2xcNXNkmx7BSfzi1PsJ2cNjs3cC3LwyhK8=
 github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

vendor/github.com/hashicorp/go-tfe/workspace.go

@@ -179,6 +179,9 @@ type WorkspaceCreateOptions struct {
 	// organization.
 	Name *string `jsonapi:"attr,name"`
 
+	// Whether the workspace will use remote or local execution mode.
+	Operations *bool `jsonapi:"attr,operations,omitempty"`
+
 	// Whether to queue all runs. Unless this is set to true, runs triggered by
 	// a webhook will not be queued until at least one run is manually queued.
 	QueueAllRuns *bool `jsonapi:"attr,queue-all-runs,omitempty"`
@@ -316,6 +319,9 @@ type WorkspaceUpdateOptions struct {
 	// disabled, any push will trigger a run.
 	FileTriggersEnabled *bool `jsonapi:"attr,file-triggers-enabled,omitempty"`
 
+	// Whether the workspace will use remote or local execution mode.
+	Operations *bool `jsonapi:"attr,operations,omitempty"`
+
 	// Whether to queue all runs. Unless this is set to true, runs triggered by
 	// a webhook will not be queued until at least one run is manually queued.
 	QueueAllRuns *bool `jsonapi:"attr,queue-all-runs,omitempty"`

vendor/modules.txt

@@ -314,9 +314,9 @@ github.com/hashicorp/go-retryablehttp
 github.com/hashicorp/go-rootcerts
 # github.com/hashicorp/go-safetemp v1.0.0
 github.com/hashicorp/go-safetemp
-# github.com/hashicorp/go-slug v0.3.0
+# github.com/hashicorp/go-slug v0.4.0
 github.com/hashicorp/go-slug
-# github.com/hashicorp/go-tfe v0.3.23
+# github.com/hashicorp/go-tfe v0.3.25
 github.com/hashicorp/go-tfe
 # github.com/hashicorp/go-uuid v1.0.1
 github.com/hashicorp/go-uuid

website/docs/backends/types/remote.html.md

@@ -166,3 +166,25 @@ The following configuration options are supported:
     workspace names are used in Terraform Cloud, and the short names
     (minus the prefix) are used on the command line. If omitted, only the
     default workspace can be used. This option conflicts with `name`.
+
+## Excluding Files from Upload with .terraformignore
+
+-> **Version note:** `.terraformignore` support was added in Terraform 0.12.11.
+
+When executing a remote `plan` or `apply` in a [CLI-driven run](/docs/cloud/run/cli.html),
+an archive of your configuration directory is uploaded to Terraform Cloud. You can define
+paths to ignore from upload via a `.terraformignore` file at the root of your configuration directory. If this file is not present, the archive will exclude the following by default:
+
+* .git/ directories
+* .terraform/ directories (exclusive of .terraform/modules)
+
+The `.terraformignore` file can include rules as one would include in a
+[.gitignore file](https://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository#Ignoring-Files)
+
+
+* Comments (starting with `#`) or blank lines are ignored
+* End a pattern with aforward slash / to specify a directory
+* Negate a pattern by starting it with an exclamation point `!`
+
+Note that unlike `.gitignore`, only the `.terraformignore` at the root of the configuration
+directory is considered.