lang/funcs: Update fileset() function to include path as separate first argument, automatically trim the path argument from results, and ensure results are always canonical with forward slash path separators

Reference: https://github.com/hashicorp/terraform/pull/22523#pullrequestreview-279694703

These changes center around better function usability and consistency with other functions. The function has not yet been released, so these breaking changes can be applied safely.
This commit is contained in:
Brian Flad 2019-08-28 11:54:04 -04:00
parent aa6dca4912
commit af7f6ef441
No known key found for this signature in database
GPG Key ID: EC6252B42B012823
6 changed files with 150 additions and 39 deletions

View File

@ -6,6 +6,7 @@ import (
"io/ioutil" "io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"strings"
"unicode/utf8" "unicode/utf8"
"github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/hcl2/hcl"
@ -212,6 +213,10 @@ func MakeFileExistsFunc(baseDir string) function.Function {
func MakeFileSetFunc(baseDir string) function.Function { func MakeFileSetFunc(baseDir string) function.Function {
return function.New(&function.Spec{ return function.New(&function.Spec{
Params: []function.Parameter{ Params: []function.Parameter{
{
Name: "path",
Type: cty.String,
},
{ {
Name: "pattern", Name: "pattern",
Type: cty.String, Type: cty.String,
@ -219,18 +224,22 @@ func MakeFileSetFunc(baseDir string) function.Function {
}, },
Type: function.StaticReturnType(cty.Set(cty.String)), Type: function.StaticReturnType(cty.Set(cty.String)),
Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
pattern := args[0].AsString() path := args[0].AsString()
pattern, err := homedir.Expand(pattern) pattern := args[1].AsString()
path, err := homedir.Expand(path)
if err != nil { if err != nil {
return cty.UnknownVal(cty.Set(cty.String)), fmt.Errorf("failed to expand ~: %s", err) return cty.UnknownVal(cty.Set(cty.String)), fmt.Errorf("failed to expand ~: %s", err)
} }
if !filepath.IsAbs(pattern) { if !filepath.IsAbs(path) {
pattern = filepath.Join(baseDir, pattern) path = filepath.Join(baseDir, path)
} }
// Ensure that the path is canonical for the host OS // Join the path to the glob pattern, while ensuring the full
pattern = filepath.Clean(pattern) // pattern is canonical for the host OS. The joined path is
// automatically cleaned during this operation.
pattern = filepath.Join(path, pattern)
matches, err := filepath.Glob(pattern) matches, err := filepath.Glob(pattern)
if err != nil { if err != nil {
@ -249,6 +258,13 @@ func MakeFileSetFunc(baseDir string) function.Function {
continue continue
} }
// Remove the path and file separator from matches.
match = strings.TrimPrefix(match, path+string(filepath.Separator))
// Return matches with the Terraform canonical pattern
// of forward slashes for cross-system compatibility.
match = filepath.ToSlash(match)
matchVals = append(matchVals, cty.StringVal(match)) matchVals = append(matchVals, cty.StringVal(match))
} }
@ -375,9 +391,9 @@ func FileExists(baseDir string, path cty.Value) (cty.Value, error) {
// The underlying function implementation works relative to a particular base // The underlying function implementation works relative to a particular base
// directory, so this wrapper takes a base directory string and uses it to // directory, so this wrapper takes a base directory string and uses it to
// construct the underlying function before calling it. // construct the underlying function before calling it.
func FileSet(baseDir string, pattern cty.Value) (cty.Value, error) { func FileSet(baseDir string, path, pattern cty.Value) (cty.Value, error) {
fn := MakeFileSetFunc(baseDir) fn := MakeFileSetFunc(baseDir)
return fn.Call([]cty.Value{pattern}) return fn.Call([]cty.Value{path, pattern})
} }
// FileBase64 reads the contents of the file at the given path. // FileBase64 reads the contents of the file at the given path.

View File

@ -226,36 +226,43 @@ func TestFileExists(t *testing.T) {
func TestFileSet(t *testing.T) { func TestFileSet(t *testing.T) {
tests := []struct { tests := []struct {
Path cty.Value
Pattern cty.Value Pattern cty.Value
Want cty.Value Want cty.Value
Err bool Err bool
}{ }{
{ {
cty.StringVal("testdata/missing"), cty.StringVal("."),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("testdata/missing*"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("*/missing"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("testdata"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("testdata*"), cty.StringVal("testdata*"),
cty.SetValEmpty(cty.String), cty.SetValEmpty(cty.String),
false, false,
}, },
{ {
cty.StringVal("."),
cty.StringVal("testdata"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("."),
cty.StringVal("testdata/missing"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("."),
cty.StringVal("testdata/missing*"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("."),
cty.StringVal("*/missing"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("."),
cty.StringVal("testdata/*.txt"), cty.StringVal("testdata/*.txt"),
cty.SetVal([]cty.Value{ cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.txt"), cty.StringVal("testdata/hello.txt"),
@ -263,6 +270,7 @@ func TestFileSet(t *testing.T) {
false, false,
}, },
{ {
cty.StringVal("."),
cty.StringVal("testdata/hello.txt"), cty.StringVal("testdata/hello.txt"),
cty.SetVal([]cty.Value{ cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.txt"), cty.StringVal("testdata/hello.txt"),
@ -270,6 +278,7 @@ func TestFileSet(t *testing.T) {
false, false,
}, },
{ {
cty.StringVal("."),
cty.StringVal("testdata/hello.???"), cty.StringVal("testdata/hello.???"),
cty.SetVal([]cty.Value{ cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.txt"), cty.StringVal("testdata/hello.txt"),
@ -277,6 +286,7 @@ func TestFileSet(t *testing.T) {
false, false,
}, },
{ {
cty.StringVal("."),
cty.StringVal("testdata/hello*"), cty.StringVal("testdata/hello*"),
cty.SetVal([]cty.Value{ cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.tmpl"), cty.StringVal("testdata/hello.tmpl"),
@ -285,6 +295,7 @@ func TestFileSet(t *testing.T) {
false, false,
}, },
{ {
cty.StringVal("."),
cty.StringVal("*/hello.txt"), cty.StringVal("*/hello.txt"),
cty.SetVal([]cty.Value{ cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.txt"), cty.StringVal("testdata/hello.txt"),
@ -292,6 +303,7 @@ func TestFileSet(t *testing.T) {
false, false,
}, },
{ {
cty.StringVal("."),
cty.StringVal("*/*.txt"), cty.StringVal("*/*.txt"),
cty.SetVal([]cty.Value{ cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.txt"), cty.StringVal("testdata/hello.txt"),
@ -299,6 +311,7 @@ func TestFileSet(t *testing.T) {
false, false,
}, },
{ {
cty.StringVal("."),
cty.StringVal("*/hello*"), cty.StringVal("*/hello*"),
cty.SetVal([]cty.Value{ cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.tmpl"), cty.StringVal("testdata/hello.tmpl"),
@ -307,20 +320,67 @@ func TestFileSet(t *testing.T) {
false, false,
}, },
{ {
cty.StringVal("."),
cty.StringVal("["), cty.StringVal("["),
cty.SetValEmpty(cty.String), cty.SetValEmpty(cty.String),
true, true,
}, },
{ {
cty.StringVal("."),
cty.StringVal("\\"), cty.StringVal("\\"),
cty.SetValEmpty(cty.String), cty.SetValEmpty(cty.String),
true, true,
}, },
{
cty.StringVal("testdata"),
cty.StringVal("missing"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("missing*"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("*.txt"),
cty.SetVal([]cty.Value{
cty.StringVal("hello.txt"),
}),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("hello.txt"),
cty.SetVal([]cty.Value{
cty.StringVal("hello.txt"),
}),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("hello.???"),
cty.SetVal([]cty.Value{
cty.StringVal("hello.txt"),
}),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("hello*"),
cty.SetVal([]cty.Value{
cty.StringVal("hello.tmpl"),
cty.StringVal("hello.txt"),
}),
false,
},
} }
for _, test := range tests { for _, test := range tests {
t.Run(fmt.Sprintf("FileSet(\".\", %#v)", test.Pattern), func(t *testing.T) { t.Run(fmt.Sprintf("FileSet(\".\", %#v, %#v)", test.Path, test.Pattern), func(t *testing.T) {
got, err := FileSet(".", test.Pattern) got, err := FileSet(".", test.Path, test.Pattern)
if test.Err { if test.Err {
if err == nil { if err == nil {

View File

@ -281,10 +281,31 @@ func TestFunctions(t *testing.T) {
"fileset": { "fileset": {
{ {
`fileset("hello.*")`, `fileset(".", "*/hello.*")`,
cty.SetVal([]cty.Value{ cty.SetVal([]cty.Value{
cty.StringVal("testdata/functions-test/hello.tmpl"), cty.StringVal("subdirectory/hello.tmpl"),
cty.StringVal("testdata/functions-test/hello.txt"), cty.StringVal("subdirectory/hello.txt"),
}),
},
{
`fileset(".", "subdirectory/hello.*")`,
cty.SetVal([]cty.Value{
cty.StringVal("subdirectory/hello.tmpl"),
cty.StringVal("subdirectory/hello.txt"),
}),
},
{
`fileset(".", "hello.*")`,
cty.SetVal([]cty.Value{
cty.StringVal("hello.tmpl"),
cty.StringVal("hello.txt"),
}),
},
{
`fileset("subdirectory", "hello.*")`,
cty.SetVal([]cty.Value{
cty.StringVal("hello.tmpl"),
cty.StringVal("hello.txt"),
}), }),
}, },
}, },

View File

@ -0,0 +1 @@
Hello, ${name}!

View File

@ -0,0 +1 @@
hello!

View File

@ -12,10 +12,13 @@ description: |-
earlier, see earlier, see
[0.11 Configuration Language: Interpolation Syntax](../../configuration-0-11/interpolation.html). [0.11 Configuration Language: Interpolation Syntax](../../configuration-0-11/interpolation.html).
`fileset` enumerates a set of regular file names given a pattern. `fileset` enumerates a set of regular file names given a path and pattern.
The path is automatically removed from the resulting set of file names and any
result still containing path separators always returns forward slash (`/`) as
the path separator for cross-system compatibility.
```hcl ```hcl
fileset(pattern) fileset(path, pattern)
``` ```
Supported pattern matches: Supported pattern matches:
@ -32,16 +35,25 @@ before Terraform takes any actions.
## Examples ## Examples
``` ```
> fileset("${path.module}/*.txt") > fileset(path.module, "files/*.txt")
[ [
"path/to/module/hello.txt", "files/hello.txt",
"path/to/module/world.txt", "files/world.txt",
]
> fileset("${path.module}/files", "*.txt")
[
"hello.txt",
"world.txt",
] ]
``` ```
A common use of `fileset` is to create one resource instance per matched file, using
[the `for_each` meta-argument](/docs/configuration/resources.html#for_each-multiple-resource-instances-defined-by-a-map-or-set-of-strings):
```hcl ```hcl
resource "example_thing" "example" { resource "example_thing" "example" {
for_each = fileset("${path.module}/files/*") for_each = fileset(path.module, "files/*")
# other configuration using each.value # other configuration using each.value
} }