configs/configload: installation from registry and go-getter

Unlike the old installer in config/module, this uses new-style
installation directories that include the static module path so that paths
we show in diagnostics will be more meaningful to the user.

As before, we retrieve the entire "package" associated with the given
source string, rather than any given subdirectory directly, because the
retrieved module may contain ../ references into parent directories which
must be resolvable after extraction.
This commit is contained in:
Martin Atkins 2018-02-13 16:36:36 -08:00
parent 3d551e25e0
commit 59939cf320
15 changed files with 915 additions and 43 deletions

View File

@ -0,0 +1,114 @@
package configload
import (
"io"
"os"
"path/filepath"
"strings"
)
// copyDir copies the src directory contents into dst. Both directories
// should already exist.
func copyDir(dst, src string) error {
src, err := filepath.EvalSymlinks(src)
if err != nil {
return err
}
walkFn := func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if path == src {
return nil
}
if strings.HasPrefix(filepath.Base(path), ".") {
// Skip any dot files
if info.IsDir() {
return filepath.SkipDir
} else {
return nil
}
}
// The "path" has the src prefixed to it. We need to join our
// destination with the path without the src on it.
dstPath := filepath.Join(dst, path[len(src):])
// we don't want to try and copy the same file over itself.
if eq, err := sameFile(path, dstPath); eq {
return nil
} else if err != nil {
return err
}
// If we have a directory, make that subdirectory, then continue
// the walk.
if info.IsDir() {
if path == filepath.Join(src, dst) {
// dst is in src; don't walk it.
return nil
}
if err := os.MkdirAll(dstPath, 0755); err != nil {
return err
}
return nil
}
// If we have a file, copy the contents.
srcF, err := os.Open(path)
if err != nil {
return err
}
defer srcF.Close()
dstF, err := os.Create(dstPath)
if err != nil {
return err
}
defer dstF.Close()
if _, err := io.Copy(dstF, srcF); err != nil {
return err
}
// Chmod it
return os.Chmod(dstPath, info.Mode())
}
return filepath.Walk(src, walkFn)
}
// sameFile tried to determine if to paths are the same file.
// If the paths don't match, we lookup the inode on supported systems.
func sameFile(a, b string) (bool, error) {
if a == b {
return true, nil
}
aIno, err := inode(a)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
bIno, err := inode(b)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
if aIno > 0 && aIno == bIno {
return true, nil
}
return false, nil
}

View File

@ -0,0 +1,122 @@
package configload
import (
"log"
"path/filepath"
cleanhttp "github.com/hashicorp/go-cleanhttp"
getter "github.com/hashicorp/go-getter"
)
// We configure our own go-getter detector and getter sets here, because
// the set of sources we support is part of Terraform's documentation and
// so we don't want any new sources introduced in go-getter to sneak in here
// and work even though they aren't documented. This also insulates us from
// any meddling that might be done by other go-getter callers linked into our
// executable.
var goGetterDetectors = []getter.Detector{
new(getter.GitHubDetector),
new(getter.BitBucketDetector),
new(getter.S3Detector),
new(getter.FileDetector),
}
var goGetterNoDetectors = []getter.Detector{}
var goGetterDecompressors = map[string]getter.Decompressor{
"bz2": new(getter.Bzip2Decompressor),
"gz": new(getter.GzipDecompressor),
"xz": new(getter.XzDecompressor),
"zip": new(getter.ZipDecompressor),
"tar.bz2": new(getter.TarBzip2Decompressor),
"tar.tbz2": new(getter.TarBzip2Decompressor),
"tar.gz": new(getter.TarGzipDecompressor),
"tgz": new(getter.TarGzipDecompressor),
"tar.xz": new(getter.TarXzDecompressor),
"txz": new(getter.TarXzDecompressor),
}
var goGetterGetters = map[string]getter.Getter{
"file": new(getter.FileGetter),
"git": new(getter.GitGetter),
"hg": new(getter.HgGetter),
"s3": new(getter.S3Getter),
"http": getterHTTPGetter,
"https": getterHTTPGetter,
}
var getterHTTPClient = cleanhttp.DefaultClient()
var getterHTTPGetter = &getter.HttpGetter{
Client: getterHTTPClient,
Netrc: true,
}
// getWithGoGetter retrieves the package referenced in the given address
// into the installation path and then returns the full path to any subdir
// indicated in the address.
//
// The errors returned by this function are those surfaced by the underlying
// go-getter library, which have very inconsistent quality as
// end-user-actionable error messages. At this time we do not have any
// reasonable way to improve these error messages at this layer because
// the underlying errors are not separatelyr recognizable.
func getWithGoGetter(instPath, addr string) (string, error) {
packageAddr, subDir := splitAddrSubdir(addr)
log.Printf("[DEBUG] will download %q to %s", packageAddr, instPath)
realAddr, err := getter.Detect(packageAddr, instPath, getter.Detectors)
if err != nil {
return "", err
}
var realSubDir string
realAddr, realSubDir = splitAddrSubdir(realAddr)
if realSubDir != "" {
subDir = filepath.Join(realSubDir, subDir)
}
if realAddr != packageAddr {
log.Printf("[TRACE] go-getter detectors rewrote %q to %q", packageAddr, realAddr)
}
client := getter.Client{
Src: realAddr,
Dst: instPath,
Pwd: instPath,
Mode: getter.ClientModeDir,
Detectors: goGetterNoDetectors, // we already did detection above
Decompressors: goGetterDecompressors,
Getters: goGetterGetters,
}
err = client.Get()
if err != nil {
return "", err
}
// Our subDir string can contain wildcards until this point, so that
// e.g. a subDir of * can expand to one top-level directory in a .tar.gz
// archive. Now that we've expanded the archive successfully we must
// resolve that into a concrete path.
var finalDir string
if subDir != "" {
finalDir, err = getter.SubdirGlob(instPath, subDir)
log.Printf("[TRACE] expanded %q to %q", subDir, finalDir)
if err != nil {
return "", err
}
} else {
finalDir = instPath
}
// If we got this far then we have apparently succeeded in downloading
// the requested object!
return filepath.Clean(finalDir), nil
}

View File

@ -0,0 +1,21 @@
// +build linux darwin openbsd netbsd solaris dragonfly
package configload
import (
"fmt"
"os"
"syscall"
)
// lookup the inode of a file on posix systems
func inode(path string) (uint64, error) {
stat, err := os.Stat(path)
if err != nil {
return 0, err
}
if st, ok := stat.Sys().(*syscall.Stat_t); ok {
return st.Ino, nil
}
return 0, fmt.Errorf("could not determine file inode")
}

View File

@ -0,0 +1,21 @@
// +build freebsd
package configload
import (
"fmt"
"os"
"syscall"
)
// lookup the inode of a file on posix systems
func inode(path string) (uint64, error) {
stat, err := os.Stat(path)
if err != nil {
return 0, err
}
if st, ok := stat.Sys().(*syscall.Stat_t); ok {
return uint64(st.Ino), nil
}
return 0, fmt.Errorf("could not determine file inode")
}

View File

@ -0,0 +1,8 @@
// +build windows
package configload
// no syscall.Stat_t on windows, return 0 for inodes
func inode(path string) (uint64, error) {
return 0, nil
}

View File

@ -10,6 +10,7 @@ import (
version "github.com/hashicorp/go-version" version "github.com/hashicorp/go-version"
"github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/hcl2/hcl"
"github.com/hashicorp/terraform/configs" "github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/registry"
"github.com/hashicorp/terraform/registry/regsrc" "github.com/hashicorp/terraform/registry/regsrc"
) )
@ -154,15 +155,16 @@ func (l *Loader) InstallModules(rootDir string, upgrade bool, hooks InstallHooks
} }
log.Printf("[TRACE] %s is a registry module at %s", key, addr) log.Printf("[TRACE] %s is a registry module at %s", key, addr)
// TODO: Implement mod, v, mDiags := l.installRegistryModule(req, key, instPath, addr, hooks)
panic("registry source installation not yet implemented") diags = append(diags, mDiags...)
return mod, v, diags
default: default:
log.Printf("[TRACE] %s address %q will be interpreted with go-getter", key, req.SourceAddr) log.Printf("[TRACE] %s address %q will be handled by go-getter", key, req.SourceAddr)
// TODO: Implement
panic("fallback source installation not yet implemented")
mod, mDiags := l.installGoGetterModule(req, key, instPath, hooks)
diags = append(diags, mDiags...)
return mod, nil, diags
} }
}, },
@ -226,12 +228,267 @@ func (l *Loader) installLocalModule(req *configs.ModuleRequest, key string, hook
Dir: newDir, Dir: newDir,
SourceAddr: req.SourceAddr, SourceAddr: req.SourceAddr,
} }
log.Printf("[TRACE] Module installer: %s installed at %s", key, newDir) log.Printf("[DEBUG] Module installer: %s installed at %s", key, newDir)
hooks.Install(key, nil, newDir) hooks.Install(key, nil, newDir)
return mod, diags return mod, diags
} }
func (l *Loader) installRegistryModule(req *configs.ModuleRequest, key string, instPath string, addr *regsrc.Module, hooks InstallHooks) (*configs.Module, *version.Version, hcl.Diagnostics) {
var diags hcl.Diagnostics
hostname, err := addr.SvcHost()
if err != nil {
// If it looks like the user was trying to use punycode then we'll generate
// a specialized error for that case. We require the unicode form of
// hostname so that hostnames are always human-readable in configuration
// and punycode can't be used to hide a malicious module hostname.
if strings.HasPrefix(addr.RawHost.Raw, "xn--") {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid module registry hostname",
Detail: "The hostname portion of this source address is not an acceptable hostname. Internationalized domain names must be given in unicode form rather than ASCII (\"punycode\") form.",
Subject: &req.SourceAddrRange,
})
} else {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid module registry hostname",
Detail: "The hostname portion of this source address is not a valid hostname.",
Subject: &req.SourceAddrRange,
})
}
return nil, nil, diags
}
reg := l.modules.Registry
log.Printf("[DEBUG] %s listing available versions of %s at %s", key, addr, hostname)
resp, err := reg.Versions(addr)
if err != nil {
if registry.IsModuleNotFound(err) {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Module not found",
Detail: fmt.Sprintf("The specified module could not be found in the module registry at %s.", hostname),
Subject: &req.SourceAddrRange,
})
} else {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Error accessing remote module registry",
Detail: fmt.Sprintf("Failed to retrieve available versions for this module from %s: %s.", hostname, err),
Subject: &req.SourceAddrRange,
})
}
return nil, nil, diags
}
// The response might contain information about dependencies to allow us
// to potentially optimize future requests, but we don't currently do that
// and so for now we'll just take the first item which is guaranteed to
// be the address we requested.
if len(resp.Modules) < 1 {
// Should never happen, but since this is a remote service that may
// be implemented by third-parties we will handle it gracefully.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid response from remote module registry",
Detail: fmt.Sprintf("The registry at %s returned an invalid response when Terraform requested available versions for this module.", hostname),
Subject: &req.SourceAddrRange,
})
return nil, nil, diags
}
modMeta := resp.Modules[0]
var latestMatch *version.Version
var latestVersion *version.Version
for _, mv := range modMeta.Versions {
v, err := version.NewVersion(mv.Version)
if err != nil {
// Should never happen if the registry server is compliant with
// the protocol, but we'll warn if not to assist someone who
// might be developing a module registry server.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagWarning,
Summary: "Invalid response from remote module registry",
Detail: fmt.Sprintf("The registry at %s returned an invalid version string %q for this module, which Terraform ignored.", hostname, mv.Version),
Subject: &req.SourceAddrRange,
})
continue
}
// If we've found a pre-release version then we'll ignore it unless
// it was exactly requested.
if v.Prerelease() != "" && req.VersionConstraint.Required.String() != v.String() {
log.Printf("[TRACE] %s ignoring %s because it is a pre-release and was not requested exactly", key, v)
continue
}
if latestVersion == nil || v.GreaterThan(latestVersion) {
latestVersion = v
}
if req.VersionConstraint.Required.Check(v) {
if latestMatch == nil || v.GreaterThan(latestMatch) {
latestMatch = v
}
}
}
if latestVersion == nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Module has no versions",
Detail: fmt.Sprintf("The specified module does not have any available versions."),
Subject: &req.SourceAddrRange,
})
return nil, nil, diags
}
if latestMatch == nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unresolvable module version constraint",
Detail: fmt.Sprintf("There is no available version of %q that matches the given version constraint. The newest available version is %s.", addr, latestVersion),
Subject: &req.VersionConstraint.DeclRange,
})
return nil, nil, diags
}
// Report up to the caller that we're about to start downloading.
packageAddr, _ := splitAddrSubdir(req.SourceAddr)
hooks.Download(key, packageAddr, latestMatch)
// If we manage to get down here then we've found a suitable version to
// install, so we need to ask the registry where we should download it from.
// The response to this is a go-getter-style address string.
dlAddr, err := reg.Location(addr, latestMatch.String())
if err != nil {
log.Printf("[ERROR] %s from %s %s: %s", key, addr, latestMatch, err)
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid response from remote module registry",
Detail: fmt.Sprintf("The remote registry at %s failed to return a download URL for %s %s.", hostname, addr, latestMatch),
Subject: &req.VersionConstraint.DeclRange,
})
return nil, nil, diags
}
log.Printf("[TRACE] %s %s %s is available at %q", key, addr, latestMatch, dlAddr)
modDir, err := getWithGoGetter(instPath, dlAddr)
if err != nil {
// Errors returned by go-getter have very inconsistent quality as
// end-user error messages, but for now we're accepting that because
// we have no way to recognize any specific errors to improve them
// and masking the error entirely would hide valuable diagnostic
// information from the user.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to download module",
Detail: fmt.Sprintf("Error attempting to download module source code from %q: %s", dlAddr, err),
Subject: &req.CallRange,
})
return nil, nil, diags
}
log.Printf("[TRACE] %s %q was downloaded to %s", key, dlAddr, modDir)
if addr.RawSubmodule != "" {
// Append the user's requested subdirectory to any subdirectory that
// was implied by any of the nested layers we expanded within go-getter.
modDir = filepath.Join(modDir, addr.RawSubmodule)
}
log.Printf("[TRACE] %s should now be at %s", key, modDir)
// Finally we are ready to try actually loading the module.
mod, mDiags := l.parser.LoadConfigDir(modDir)
if mod == nil {
// nil indicates missing or unreadable directory, so we'll
// discard the returned diags and return a more specific
// error message here. For registry modules this actually
// indicates a bug in the code above, since it's not the
// user's responsibility to create the directory in this case.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unreadable module directory",
Detail: fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir),
Subject: &req.CallRange,
})
} else {
diags = append(diags, mDiags...)
}
// Note the local location in our manifest.
l.modules.manifest[key] = moduleRecord{
Key: key,
Version: latestMatch,
Dir: modDir,
SourceAddr: req.SourceAddr,
}
log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir)
hooks.Install(key, latestMatch, modDir)
return mod, latestMatch, diags
}
func (l *Loader) installGoGetterModule(req *configs.ModuleRequest, key string, instPath string, hooks InstallHooks) (*configs.Module, hcl.Diagnostics) {
var diags hcl.Diagnostics
// Report up to the caller that we're about to start downloading.
packageAddr, _ := splitAddrSubdir(req.SourceAddr)
hooks.Download(key, packageAddr, nil)
modDir, err := getWithGoGetter(instPath, req.SourceAddr)
if err != nil {
// Errors returned by go-getter have very inconsistent quality as
// end-user error messages, but for now we're accepting that because
// we have no way to recognize any specific errors to improve them
// and masking the error entirely would hide valuable diagnostic
// information from the user.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to download module",
Detail: fmt.Sprintf("Error attempting to download module source code from %q: %s", packageAddr, err),
Subject: &req.SourceAddrRange,
})
return nil, diags
}
log.Printf("[TRACE] %s %q was downloaded to %s", key, req.SourceAddr, modDir)
mod, mDiags := l.parser.LoadConfigDir(modDir)
if mod == nil {
// nil indicates missing or unreadable directory, so we'll
// discard the returned diags and return a more specific
// error message here. For registry modules this actually
// indicates a bug in the code above, since it's not the
// user's responsibility to create the directory in this case.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unreadable module directory",
Detail: fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir),
Subject: &req.CallRange,
})
} else {
diags = append(diags, mDiags...)
}
// Note the local location in our manifest.
l.modules.manifest[key] = moduleRecord{
Key: key,
Dir: modDir,
SourceAddr: req.SourceAddr,
}
log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir)
hooks.Install(key, nil, modDir)
return mod, diags
}
func (l *Loader) packageInstallPath(modulePath []string) string { func (l *Loader) packageInstallPath(modulePath []string) string {
return filepath.Join(l.modules.Dir, strings.Join(modulePath, ".")) return filepath.Join(l.modules.Dir, strings.Join(modulePath, "."))
} }

View File

@ -1,6 +1,7 @@
package configload package configload
import ( import (
"os"
"path/filepath" "path/filepath"
"testing" "testing"
@ -9,11 +10,12 @@ import (
func TestLoaderInstallModules_local(t *testing.T) { func TestLoaderInstallModules_local(t *testing.T) {
fixtureDir := filepath.Clean("test-fixtures/local-modules") fixtureDir := filepath.Clean("test-fixtures/local-modules")
loader := newTestLoader(filepath.Join(fixtureDir, ".terraform/modules")) loader, done := tempChdirLoader(t, fixtureDir)
defer done()
hooks := &testInstallHooks{} hooks := &testInstallHooks{}
diags := loader.InstallModules(fixtureDir, false, hooks) diags := loader.InstallModules(".", false, hooks)
assertNoDiagnostics(t, diags) assertNoDiagnostics(t, diags)
wantCalls := []testInstallHookCall{ wantCalls := []testInstallHookCall{
@ -21,17 +23,210 @@ func TestLoaderInstallModules_local(t *testing.T) {
Name: "Install", Name: "Install",
ModuleAddr: "child_a", ModuleAddr: "child_a",
PackageAddr: "", PackageAddr: "",
LocalPath: "test-fixtures/local-modules/child_a", LocalPath: "child_a",
}, },
{ {
Name: "Install", Name: "Install",
ModuleAddr: "child_a.child_b", ModuleAddr: "child_a.child_b",
PackageAddr: "", PackageAddr: "",
LocalPath: "test-fixtures/local-modules/child_a/child_b", LocalPath: "child_a/child_b",
}, },
} }
assertResultDeepEqual(t, hooks.Calls, wantCalls) if assertResultDeepEqual(t, hooks.Calls, wantCalls) {
return
}
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
_, loadDiags := loader.LoadConfig(".")
assertNoDiagnostics(t, loadDiags)
}
func TestLoaderInstallModules_registry(t *testing.T) {
if os.Getenv("TF_ACC") == "" {
t.Skip("this test accesses registry.terraform.io and github.com; set TF_ACC=1 to run it")
}
fixtureDir := filepath.Clean("test-fixtures/registry-modules")
loader, done := tempChdirLoader(t, fixtureDir)
defer done()
hooks := &testInstallHooks{}
diags := loader.InstallModules(".", false, hooks)
assertNoDiagnostics(t, diags)
v := version.Must(version.NewVersion("0.0.1"))
wantCalls := []testInstallHookCall{
// the configuration builder visits each level of calls in lexicographical
// order by name, so the following list is kept in the same order.
// acctest_child_a accesses //modules/child_a directly
{
Name: "Download",
ModuleAddr: "acctest_child_a",
PackageAddr: "hashicorp/module-installer-acctest/aws", // intentionally excludes the subdir because we're downloading the whole package here
Version: v,
},
{
Name: "Install",
ModuleAddr: "acctest_child_a",
Version: v,
LocalPath: ".terraform/modules/acctest_child_a/hashicorp-terraform-aws-module-installer-acctest-853d038/modules/child_a",
},
// acctest_child_a.child_b
// (no download because it's a relative path inside acctest_child_a)
{
Name: "Install",
ModuleAddr: "acctest_child_a.child_b",
LocalPath: ".terraform/modules/acctest_child_a/hashicorp-terraform-aws-module-installer-acctest-853d038/modules/child_b",
},
// acctest_child_b accesses //modules/child_b directly
{
Name: "Download",
ModuleAddr: "acctest_child_b",
PackageAddr: "hashicorp/module-installer-acctest/aws", // intentionally excludes the subdir because we're downloading the whole package here
Version: v,
},
{
Name: "Install",
ModuleAddr: "acctest_child_b",
Version: v,
LocalPath: ".terraform/modules/acctest_child_b/hashicorp-terraform-aws-module-installer-acctest-853d038/modules/child_b",
},
// acctest_root
{
Name: "Download",
ModuleAddr: "acctest_root",
PackageAddr: "hashicorp/module-installer-acctest/aws",
Version: v,
},
{
Name: "Install",
ModuleAddr: "acctest_root",
Version: v,
LocalPath: ".terraform/modules/acctest_root/hashicorp-terraform-aws-module-installer-acctest-853d038",
},
// acctest_root.child_a
// (no download because it's a relative path inside acctest_root)
{
Name: "Install",
ModuleAddr: "acctest_root.child_a",
LocalPath: ".terraform/modules/acctest_root/hashicorp-terraform-aws-module-installer-acctest-853d038/modules/child_a",
},
// acctest_root.child_a.child_b
// (no download because it's a relative path inside acctest_root, via acctest_root.child_a)
{
Name: "Install",
ModuleAddr: "acctest_root.child_a.child_b",
LocalPath: ".terraform/modules/acctest_root/hashicorp-terraform-aws-module-installer-acctest-853d038/modules/child_b",
},
}
if assertResultDeepEqual(t, hooks.Calls, wantCalls) {
return
}
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
_, loadDiags := loader.LoadConfig(".")
assertNoDiagnostics(t, loadDiags)
}
func TestLoaderInstallModules_goGetter(t *testing.T) {
if os.Getenv("TF_ACC") == "" {
t.Skip("this test accesses github.com; set TF_ACC=1 to run it")
}
fixtureDir := filepath.Clean("test-fixtures/go-getter-modules")
loader, done := tempChdirLoader(t, fixtureDir)
defer done()
hooks := &testInstallHooks{}
diags := loader.InstallModules(".", false, hooks)
assertNoDiagnostics(t, diags)
wantCalls := []testInstallHookCall{
// the configuration builder visits each level of calls in lexicographical
// order by name, so the following list is kept in the same order.
// acctest_child_a accesses //modules/child_a directly
{
Name: "Download",
ModuleAddr: "acctest_child_a",
PackageAddr: "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1", // intentionally excludes the subdir because we're downloading the whole repo here
},
{
Name: "Install",
ModuleAddr: "acctest_child_a",
LocalPath: ".terraform/modules/acctest_child_a/modules/child_a",
},
// acctest_child_a.child_b
// (no download because it's a relative path inside acctest_child_a)
{
Name: "Install",
ModuleAddr: "acctest_child_a.child_b",
LocalPath: ".terraform/modules/acctest_child_a/modules/child_b",
},
// acctest_child_b accesses //modules/child_b directly
{
Name: "Download",
ModuleAddr: "acctest_child_b",
PackageAddr: "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1", // intentionally excludes the subdir because we're downloading the whole package here
},
{
Name: "Install",
ModuleAddr: "acctest_child_b",
LocalPath: ".terraform/modules/acctest_child_b/modules/child_b",
},
// acctest_root
{
Name: "Download",
ModuleAddr: "acctest_root",
PackageAddr: "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1",
},
{
Name: "Install",
ModuleAddr: "acctest_root",
LocalPath: ".terraform/modules/acctest_root",
},
// acctest_root.child_a
// (no download because it's a relative path inside acctest_root)
{
Name: "Install",
ModuleAddr: "acctest_root.child_a",
LocalPath: ".terraform/modules/acctest_root/modules/child_a",
},
// acctest_root.child_a.child_b
// (no download because it's a relative path inside acctest_root, via acctest_root.child_a)
{
Name: "Install",
ModuleAddr: "acctest_root.child_a.child_b",
LocalPath: ".terraform/modules/acctest_root/modules/child_b",
},
}
if assertResultDeepEqual(t, hooks.Calls, wantCalls) {
return
}
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
_, loadDiags := loader.LoadConfig(".")
assertNoDiagnostics(t, loadDiags)
} }
type testInstallHooks struct { type testInstallHooks struct {

View File

@ -1,43 +1,90 @@
package configload package configload
import ( import (
"reflect" "fmt"
"io/ioutil"
"os"
"path/filepath"
"testing" "testing"
"github.com/spf13/afero" "github.com/go-test/deep"
"github.com/davecgh/go-spew/spew"
"github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/hcl2/hcl"
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/registry"
"github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty"
) )
// newTestLoader is like NewLoader but it uses a copy-on-write overlay filesystem // tempChdir copies the contents of the given directory to a temporary
// over the real filesystem so that any files that are created cannot persist // directory and changes the test process's current working directory to
// between test runs. // point to that directory. Also returned is a function that should be
// called at the end of the test (e.g. via "defer") to restore the previous
// working directory.
// //
// It will also panic if there are any errors creating the loader, since // Tests using this helper cannot safely be run in parallel with other tests.
// these should never happen in a testing scenario. func tempChdir(t *testing.T, sourceDir string) (string, func()) {
func newTestLoader(dir string) *Loader { t.Helper()
realFS := afero.NewOsFs()
overlayFS := afero.NewMemMapFs() tmpDir, err := ioutil.TempDir("", "terraform-configload")
fs := afero.NewCopyOnWriteFs(realFS, overlayFS)
parser := configs.NewParser(fs)
reg := registry.NewClient(nil, nil, nil)
ret := &Loader{
parser: parser,
modules: moduleMgr{
FS: afero.Afero{fs},
Dir: dir,
Registry: reg,
},
}
err := ret.modules.readModuleManifestSnapshot()
if err != nil { if err != nil {
panic(err) t.Fatalf("failed to create temporary directory: %s", err)
return "", nil
} }
return ret
if err := copyDir(tmpDir, sourceDir); err != nil {
t.Fatalf("failed to copy fixture to temporary directory: %s", err)
return "", nil
}
oldDir, err := os.Getwd()
if err != nil {
t.Fatalf("failed to determine current working directory: %s", err)
return "", nil
}
err = os.Chdir(tmpDir)
if err != nil {
t.Fatalf("failed to switch to temp dir %s: %s", tmpDir, err)
return "", nil
}
t.Logf("tempChdir switched to %s after copying from %s", tmpDir, sourceDir)
return tmpDir, func() {
err := os.Chdir(oldDir)
if err != nil {
panic(fmt.Errorf("failed to restore previous working directory %s: %s", oldDir, err))
}
if os.Getenv("TF_CONFIGLOAD_TEST_KEEP_TMP") == "" {
os.RemoveAll(tmpDir)
}
}
}
// tempChdirLoader is a wrapper around tempChdir that also returns a Loader
// whose modules directory is at the conventional location within the
// created temporary directory.
func tempChdirLoader(t *testing.T, sourceDir string) (*Loader, func()) {
t.Helper()
_, done := tempChdir(t, sourceDir)
modulesDir := filepath.Clean(".terraform/modules")
err := os.MkdirAll(modulesDir, os.ModePerm)
if err != nil {
done() // undo the chdir in tempChdir so we can safely run other tests
t.Fatalf("failed to create modules directory: %s", err)
return nil, nil
}
loader, err := NewLoader(&Config{
ModulesDir: modulesDir,
})
if err != nil {
done() // undo the chdir in tempChdir so we can safely run other tests
t.Fatalf("failed to create loader: %s", err)
return nil, nil
}
return loader, done
} }
func assertNoDiagnostics(t *testing.T, diags hcl.Diagnostics) bool { func assertNoDiagnostics(t *testing.T, diags hcl.Diagnostics) bool {
@ -75,8 +122,10 @@ func assertDiagnosticSummary(t *testing.T, diags hcl.Diagnostics, want string) b
func assertResultDeepEqual(t *testing.T, got, want interface{}) bool { func assertResultDeepEqual(t *testing.T, got, want interface{}) bool {
t.Helper() t.Helper()
if !reflect.DeepEqual(got, want) { if diff := deep.Equal(got, want); diff != nil {
t.Errorf("wrong result\ngot: %swant: %s", spew.Sdump(got), spew.Sdump(want)) for _, problem := range diff {
t.Errorf("%s", problem)
}
return true return true
} }
return false return false

View File

@ -3,6 +3,8 @@ package configload
import ( import (
"strings" "strings"
"github.com/hashicorp/go-getter"
"github.com/hashicorp/terraform/registry/regsrc" "github.com/hashicorp/terraform/registry/regsrc"
) )
@ -26,3 +28,18 @@ func isRegistrySourceAddr(addr string) bool {
_, err := regsrc.ParseModuleSource(addr) _, err := regsrc.ParseModuleSource(addr)
return err == nil return err == nil
} }
// splitAddrSubdir splits the given address (which is assumed to be a
// registry address or go-getter-style address) into a package portion
// and a sub-directory portion.
//
// The package portion defines what should be downloaded and then the
// sub-directory portion, if present, specifies a sub-directory within
// the downloaded object (an archive, VCS repository, etc) that contains
// the module's configuration files.
//
// The subDir portion will be returned as empty if no subdir separator
// ("//") is present in the address.
func splitAddrSubdir(addr string) (packageAddr, subDir string) {
return getter.SourceDirSubdir(addr)
}

View File

@ -0,0 +1 @@
.terraform/*

View File

@ -0,0 +1,16 @@
# This fixture depends on a github repo at:
# https://github.com/hashicorp/terraform-aws-module-installer-acctest
# ...and expects its v0.0.1 tag to be pointing at the following commit:
# d676ab2559d4e0621d59e3c3c4cbb33958ac4608
module "acctest_root" {
source = "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1"
}
module "acctest_child_a" {
source = "github.com/hashicorp/terraform-aws-module-installer-acctest//modules/child_a?ref=v0.0.1"
}
module "acctest_child_b" {
source = "github.com/hashicorp/terraform-aws-module-installer-acctest//modules/child_b?ref=v0.0.1"
}

View File

@ -0,0 +1 @@
.terraform/*

View File

@ -0,0 +1,27 @@
# This fixture indirectly depends on a github repo at:
# https://github.com/hashicorp/terraform-aws-module-installer-acctest
# ...and expects its v0.0.1 tag to be pointing at the following commit:
# d676ab2559d4e0621d59e3c3c4cbb33958ac4608
#
# This repository is accessed indirectly via:
# https://registry.terraform.io/modules/hashicorp/module-installer-acctest/aws/0.0.1
#
# Since the tag's id is included in a downloaded archive, it is expected to
# have the following id:
# 853d03855b3290a3ca491d4c3a7684572dd42237
# (this particular assumption is encoded in the tests that use this fixture)
module "acctest_root" {
source = "hashicorp/module-installer-acctest/aws"
version = "0.0.1"
}
module "acctest_child_a" {
source = "hashicorp/module-installer-acctest/aws//modules/child_a"
version = "0.0.1"
}
module "acctest_child_b" {
source = "hashicorp/module-installer-acctest/aws//modules/child_b"
version = "0.0.1"
}

View File

@ -115,7 +115,7 @@ func (c *Client) Versions(module *regsrc.Module) (*response.ModuleVersions, erro
case http.StatusOK: case http.StatusOK:
// OK // OK
case http.StatusNotFound: case http.StatusNotFound:
return nil, fmt.Errorf("module %q not found", module.String()) return nil, &errModuleNotFound{addr: module}
default: default:
return nil, fmt.Errorf("error looking up module versions: %s", resp.Status) return nil, fmt.Errorf("error looking up module versions: %s", resp.Status)
} }

23
registry/errors.go Normal file
View File

@ -0,0 +1,23 @@
package registry
import (
"fmt"
"github.com/hashicorp/terraform/registry/regsrc"
)
type errModuleNotFound struct {
addr *regsrc.Module
}
func (e *errModuleNotFound) Error() string {
return fmt.Sprintf("module %s not found", e.addr)
}
// IsModuleNotFound returns true only if the given error is a "module not found"
// error. This allows callers to recognize this particular error condition
// as distinct from operational errors such as poor network connectivity.
func IsModuleNotFound(err error) bool {
_, ok := err.(*errModuleNotFound)
return ok
}