resilien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Move module install functionality over to internal/initwd

This commit is contained in:
Martin Atkins 2019-01-08 18:39:14 -08:00
parent 047239e68c
commit 0c0a437bcb
40 changed files with 599 additions and 896 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/local/backend_apply_test.go
View File

@ -13,8 +13,8 @@ import (
"github.com/zclconf/go-cty/cty"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/configs/configload"
"github.com/hashicorp/terraform/configs/configschema"
"github.com/hashicorp/terraform/internal/initwd"
"github.com/hashicorp/terraform/providers"
"github.com/hashicorp/terraform/states"
"github.com/hashicorp/terraform/states/statemgr"
@ -252,7 +252,7 @@ func (s failingState) WriteState(state *states.State) error {
func testOperationApply(t *testing.T, configDir string) (*backend.Operation, func()) {
t.Helper()
_, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir)
_, configLoader, configCleanup := initwd.MustLoadConfigForTests(t, configDir)
return &backend.Operation{
Type: backend.OperationTypeApply,

4
backend/local/backend_plan_test.go
View File

@ -10,8 +10,8 @@ import (
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/configs/configload"
"github.com/hashicorp/terraform/configs/configschema"
"github.com/hashicorp/terraform/internal/initwd"
"github.com/hashicorp/terraform/plans"
"github.com/hashicorp/terraform/plans/planfile"
"github.com/hashicorp/terraform/states"
@ -514,7 +514,7 @@ func TestLocal_planScaleOutNoDupeCount(t *testing.T) {
func testOperationPlan(t *testing.T, configDir string) (*backend.Operation, func()) {
t.Helper()
_, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir)
_, configLoader, configCleanup := initwd.MustLoadConfigForTests(t, configDir)
return &backend.Operation{
Type: backend.OperationTypePlan,

4
backend/local/backend_refresh_test.go
View File

@ -8,8 +8,8 @@ import (
"github.com/hashicorp/terraform/providers"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/configs/configload"
"github.com/hashicorp/terraform/configs/configschema"
"github.com/hashicorp/terraform/internal/initwd"
"github.com/hashicorp/terraform/terraform"
"github.com/zclconf/go-cty/cty"
)
@ -203,7 +203,7 @@ test_instance.foo:
func testOperationRefresh(t *testing.T, configDir string) (*backend.Operation, func()) {
t.Helper()
_, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir)
_, configLoader, configCleanup := initwd.MustLoadConfigForTests(t, configDir)
return &backend.Operation{
Type: backend.OperationTypeRefresh,

4
backend/remote/backend_apply_test.go
View File

@ -12,7 +12,7 @@ import (
tfe "github.com/hashicorp/go-tfe"
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/configs/configload"
"github.com/hashicorp/terraform/internal/initwd"
"github.com/hashicorp/terraform/plans/planfile"
"github.com/hashicorp/terraform/terraform"
"github.com/mitchellh/cli"
@ -21,7 +21,7 @@ import (
func testOperationApply(t *testing.T, configDir string) (*backend.Operation, func()) {
t.Helper()
_, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir)
_, configLoader, configCleanup := initwd.MustLoadConfigForTests(t, configDir)
return &backend.Operation{
ConfigDir: configDir,

4
backend/remote/backend_plan_test.go
View File

@ -12,7 +12,7 @@ import (
tfe "github.com/hashicorp/go-tfe"
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/configs/configload"
"github.com/hashicorp/terraform/internal/initwd"
"github.com/hashicorp/terraform/plans/planfile"
"github.com/hashicorp/terraform/terraform"
"github.com/mitchellh/cli"
@ -21,7 +21,7 @@ import (
func testOperationPlan(t *testing.T, configDir string) (*backend.Operation, func()) {
t.Helper()
_, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir)
_, configLoader, configCleanup := initwd.MustLoadConfigForTests(t, configDir)
return &backend.Operation{
ConfigDir: configDir,

10
command/command_test.go
View File

@ -7,6 +7,8 @@ import (
"encoding/json"
"flag"
"fmt"
"github.com/hashicorp/terraform/internal/initwd"
"github.com/hashicorp/terraform/registry"
"io"
"io/ioutil"
"log"
@ -141,7 +143,6 @@ func testModuleWithSnapshot(t *testing.T, name string) (*configs.Config, *config
t.Helper()
dir := filepath.Join(fixtureDir, name)
// FIXME: We're not dealing with the cleanup function here because
// this testModule function is used all over and so we don't want to
// change its interface at this late stage.
@ -150,9 +151,10 @@ func testModuleWithSnapshot(t *testing.T, name string) (*configs.Config, *config
// Test modules usually do not refer to remote sources, and for local
// sources only this ultimately just records all of the module paths
// in a JSON file so that we can load them below.
diags := loader.InstallModules(dir, true, configload.InstallHooksImpl{})
if diags.HasErrors() {
t.Fatal(diags.Error())
inst := initwd.NewModuleInstaller(loader.ModulesDir(), registry.NewClient(nil, nil))
instDiags := inst.InstallModules(dir, true, initwd.ModuleInstallHooksImpl{})
if instDiags.HasErrors() {
t.Fatal(instDiags.Err())
}
config, snap, diags := loader.LoadConfigWithSnapshot(dir)

6
command/hook_module_install.go
View File

@ -4,17 +4,17 @@ import (
"fmt"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/terraform/configs/configload"
"github.com/hashicorp/terraform/internal/initwd"
"github.com/mitchellh/cli"
)
type uiModuleInstallHooks struct {
configload.InstallHooksImpl
initwd.ModuleInstallHooksImpl
Ui cli.Ui
ShowLocalPaths bool
}
var _ configload.InstallHooks = uiModuleInstallHooks{}
var _ initwd.ModuleInstallHooks = uiModuleInstallHooks{}
func (h uiModuleInstallHooks) Download(modulePath, packageAddr string, v *version.Version) {
if v != nil {

43
command/meta_config.go
View File

@ -6,12 +6,13 @@ import (
"path/filepath"
"sort"
"github.com/hashicorp/hcl2/hcl/hclsyntax"
"github.com/hashicorp/hcl2/hcl"
"github.com/hashicorp/hcl2/hcl/hclsyntax"
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/configs/configload"
"github.com/hashicorp/terraform/configs/configschema"
"github.com/hashicorp/terraform/internal/initwd"
"github.com/hashicorp/terraform/registry"
"github.com/hashicorp/terraform/terraform"
"github.com/hashicorp/terraform/tfdiags"
"github.com/zclconf/go-cty/cty"
@ -165,7 +166,7 @@ func (m *Meta) loadHCLFile(filename string) (hcl.Body, tfdiags.Diagnostics) {
// can then be relayed to the end-user. The moduleUiInstallHooks type in
// this package has a reasonable implementation for displaying notifications
// via a provided cli.Ui.
func (m *Meta) installModules(rootDir string, upgrade bool, hooks configload.InstallHooks) tfdiags.Diagnostics {
func (m *Meta) installModules(rootDir string, upgrade bool, hooks initwd.ModuleInstallHooks) tfdiags.Diagnostics {
var diags tfdiags.Diagnostics
rootDir = m.normalizePath(rootDir)
@ -175,14 +176,9 @@ func (m *Meta) installModules(rootDir string, upgrade bool, hooks configload.Ins
return diags
}
loader, err := m.initConfigLoader()
if err != nil {
diags = diags.Append(err)
return diags
}
hclDiags := loader.InstallModules(rootDir, upgrade, hooks)
diags = diags.Append(hclDiags)
inst := m.moduleInstaller()
moreDiags := inst.InstallModules(rootDir, upgrade, hooks)
diags = diags.Append(moreDiags)
return diags
}
@ -195,18 +191,11 @@ func (m *Meta) installModules(rootDir string, upgrade bool, hooks configload.Ins
// can then be relayed to the end-user. The moduleUiInstallHooks type in
// this package has a reasonable implementation for displaying notifications
// via a provided cli.Ui.
func (m *Meta) initDirFromModule(targetDir string, addr string, hooks configload.InstallHooks) tfdiags.Diagnostics {
func (m *Meta) initDirFromModule(targetDir string, addr string, hooks initwd.ModuleInstallHooks) tfdiags.Diagnostics {
var diags tfdiags.Diagnostics
targetDir = m.normalizePath(targetDir)
loader, err := m.initConfigLoader()
if err != nil {
diags = diags.Append(err)
return diags
}
hclDiags := loader.InitDirFromModule(targetDir, addr, hooks)
diags = diags.Append(hclDiags)
moreDiags := initwd.DirFromModule(targetDir, m.modulesDir(), addr, m.registryClient(), hooks)
diags = diags.Append(moreDiags)
return diags
}
@ -327,6 +316,18 @@ func (m *Meta) initConfigLoader() (*configload.Loader, error) {
return m.configLoader, nil
}
// moduleInstaller instantiates and returns a module installer for use by
// "terraform init" (directly or indirectly).
func (m *Meta) moduleInstaller() *initwd.ModuleInstaller {
reg := m.registryClient()
return initwd.NewModuleInstaller(m.modulesDir(), reg)
}
// registryClient instantiates and returns a new Terraform Registry client.
func (m *Meta) registryClient() *registry.Client {
return registry.NewClient(m.Services, nil)
}
// configValueFromCLI parses a configuration value that was provided in a
// context in the CLI where only strings can be provided, such as on the
// command line or in an environment variable, and returns the resulting

6
configs/configload/loader.go
View File

@ -71,6 +71,12 @@ func NewLoader(config *Config) (*Loader, error) {
return ret, nil
}
// ModulesDir returns the path to the directory where the loader will look for
// the local cache of remote module packages.
func (l *Loader) ModulesDir() string {
return l.modules.Dir
}
// Parser returns the underlying parser for this loader.
//
// This is useful for loading other sorts of files than the module directories

523
configs/configload/loader_install.go
View File

@ -1,523 +0,0 @@
package configload
import (
"fmt"
"log"
"os"
"path/filepath"
"strings"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/hcl2/hcl"
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/registry"
"github.com/hashicorp/terraform/registry/regsrc"
)
// InstallModules analyses the root module in the given directory and installs
// all of its direct and transitive dependencies into the loader's modules
// directory, which must already exist.
//
// Since InstallModules makes possibly-time-consuming calls to remote services,
// a hook interface is supported to allow the caller to be notified when
// each module is installed and, for remote modules, when downloading begins.
// LoadConfig guarantees that two hook calls will not happen concurrently but
// it does not guarantee any particular ordering of hook calls. This mechanism
// is for UI feedback only and does not give the caller any control over the
// process.
//
// If modules are already installed in the target directory, they will be
// skipped unless their source address or version have changed or unless
// the upgrade flag is set.
//
// InstallModules never deletes any directory, except in the case where it
// needs to replace a directory that is already present with a newly-extracted
// package.
//
// If the returned diagnostics contains errors then the module installation
// may have wholly or partially completed. Modules must be loaded in order
// to find their dependencies, so this function does many of the same checks
// as LoadConfig as a side-effect.
//
// This function will panic if called on a loader that cannot install modules.
// Use CanInstallModules to determine if a loader can install modules, or
// refer to the documentation for that method for situations where module
// installation capability is guaranteed.
func (l *Loader) InstallModules(rootDir string, upgrade bool, hooks InstallHooks) hcl.Diagnostics {
if !l.CanInstallModules() {
panic(fmt.Errorf("InstallModules called on loader that cannot install modules"))
}
rootMod, diags := l.parser.LoadConfigDir(rootDir)
if rootMod == nil {
return diags
}
getter := reusingGetter{}
instDiags := l.installDescendentModules(rootMod, rootDir, upgrade, hooks, getter)
diags = append(diags, instDiags...)
return diags
}
func (l *Loader) installDescendentModules(rootMod *configs.Module, rootDir string, upgrade bool, hooks InstallHooks, getter reusingGetter) hcl.Diagnostics {
var diags hcl.Diagnostics
if hooks == nil {
// Use our no-op implementation as a placeholder
hooks = InstallHooksImpl{}
}
// Create a manifest record for the root module. This will be used if
// there are any relative-pathed modules in the root.
l.modules.manifest[""] = moduleRecord{
Key: "",
Dir: rootDir,
}
_, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc(
func(req *configs.ModuleRequest) (*configs.Module, *version.Version, hcl.Diagnostics) {
key := manifestKey(req.Path)
instPath := l.packageInstallPath(req.Path)
log.Printf("[DEBUG] Module installer: begin %s", key)
// First we'll check if we need to upgrade/replace an existing
// installed module, and delete it out of the way if so.
replace := upgrade
if !replace {
record, recorded := l.modules.manifest[key]
switch {
case !recorded:
log.Printf("[TRACE] %s is not yet installed", key)
replace = true
case record.SourceAddr != req.SourceAddr:
log.Printf("[TRACE] %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr)
replace = true
case record.Version != nil && !req.VersionConstraint.Required.Check(record.Version):
log.Printf("[TRACE] %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraint.Required)
replace = true
}
}
// If we _are_ planning to replace this module, then we'll remove
// it now so our installation code below won't conflict with any
// existing remnants.
if replace {
if _, recorded := l.modules.manifest[key]; recorded {
log.Printf("[TRACE] discarding previous record of %s prior to reinstall", key)
}
delete(l.modules.manifest, key)
// Deleting a module invalidates all of its descendent modules too.
keyPrefix := key + "."
for subKey := range l.modules.manifest {
if strings.HasPrefix(subKey, keyPrefix) {
if _, recorded := l.modules.manifest[subKey]; recorded {
log.Printf("[TRACE] also discarding downstream %s", subKey)
}
delete(l.modules.manifest, subKey)
}
}
}
record, recorded := l.modules.manifest[key]
if !recorded {
// Clean up any stale cache directory that might be present.
// If this is a local (relative) source then the dir will
// not exist, but we'll ignore that.
log.Printf("[TRACE] cleaning directory %s prior to install of %s", instPath, key)
err := l.modules.FS.RemoveAll(instPath)
if err != nil && !os.IsNotExist(err) {
log.Printf("[TRACE] failed to remove %s: %s", key, err)
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to remove local module cache",
Detail: fmt.Sprintf(
"Terraform tried to remove %s in order to reinstall this module, but encountered an error: %s",
instPath, err,
),
Subject: &req.CallRange,
})
return nil, nil, diags
}
} else {
// If this module is already recorded and its root directory
// exists then we will just load what's already there and
// keep our existing record.
info, err := l.modules.FS.Stat(record.Dir)
if err == nil && info.IsDir() {
mod, mDiags := l.parser.LoadConfigDir(record.Dir)
diags = append(diags, mDiags...)
log.Printf("[TRACE] Module installer: %s %s already installed in %s", key, record.Version, record.Dir)
return mod, record.Version, diags
}
}
// If we get down here then it's finally time to actually install
// the module. There are some variants to this process depending
// on what type of module source address we have.
switch {
case isLocalSourceAddr(req.SourceAddr):
log.Printf("[TRACE] %s has local path %q", key, req.SourceAddr)
mod, mDiags := l.installLocalModule(req, key, hooks)
diags = append(diags, mDiags...)
return mod, nil, diags
case isRegistrySourceAddr(req.SourceAddr):
addr, err := regsrc.ParseModuleSource(req.SourceAddr)
if err != nil {
// Should never happen because isRegistrySourceAddr already validated
panic(err)
}
log.Printf("[TRACE] %s is a registry module at %s", key, addr)
mod, v, mDiags := l.installRegistryModule(req, key, instPath, addr, hooks, getter)
diags = append(diags, mDiags...)
return mod, v, diags
default:
log.Printf("[TRACE] %s address %q will be handled by go-getter", key, req.SourceAddr)
mod, mDiags := l.installGoGetterModule(req, key, instPath, hooks, getter)
diags = append(diags, mDiags...)
return mod, nil, diags
}
},
))
diags = append(diags, cDiags...)
err := l.modules.writeModuleManifestSnapshot()
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to update module manifest",
Detail: fmt.Sprintf("Unable to write the module manifest file: %s", err),
})
}
return diags
}
// CanInstallModules returns true if InstallModules can be used with this
// loader.
//
// Loaders created with NewLoader can always install modules. Loaders created
// from plan files (where the configuration is embedded in the plan file itself)
// cannot install modules, because the plan file is read-only.
func (l *Loader) CanInstallModules() bool {
return l.modules.CanInstall
}
func (l *Loader) installLocalModule(req *configs.ModuleRequest, key string, hooks InstallHooks) (*configs.Module, hcl.Diagnostics) {
var diags hcl.Diagnostics
parentKey := manifestKey(req.Parent.Path)
parentRecord, recorded := l.modules.manifest[parentKey]
if !recorded {
// This is indicative of a bug rather than a user-actionable error
panic(fmt.Errorf("missing manifest record for parent module %s", parentKey))
}
if len(req.VersionConstraint.Required) != 0 {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid version constraint",
Detail: "A version constraint cannot be applied to a module at a relative local path.",
Subject: &req.VersionConstraint.DeclRange,
})
}
// For local sources we don't actually need to modify the
// filesystem at all because the parent already wrote
// the files we need, and so we just load up what's already here.
newDir := filepath.Join(parentRecord.Dir, req.SourceAddr)
log.Printf("[TRACE] %s uses directory from parent: %s", key, newDir)
mod, mDiags := l.parser.LoadConfigDir(newDir)
if mod == nil {
// nil indicates missing or unreadable directory, so we'll
// discard the returned diags and return a more specific
// error message here.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unreadable module directory",
Detail: fmt.Sprintf("The directory %s could not be read.", newDir),
Subject: &req.SourceAddrRange,
})
} else {
diags = append(diags, mDiags...)
}
// Note the local location in our manifest.
l.modules.manifest[key] = moduleRecord{
Key: key,
Dir: newDir,
SourceAddr: req.SourceAddr,
}
log.Printf("[DEBUG] Module installer: %s installed at %s", key, newDir)
hooks.Install(key, nil, newDir)
return mod, diags
}
func (l *Loader) installRegistryModule(req *configs.ModuleRequest, key string, instPath string, addr *regsrc.Module, hooks InstallHooks, getter reusingGetter) (*configs.Module, *version.Version, hcl.Diagnostics) {
var diags hcl.Diagnostics
hostname, err := addr.SvcHost()
if err != nil {
// If it looks like the user was trying to use punycode then we'll generate
// a specialized error for that case. We require the unicode form of
// hostname so that hostnames are always human-readable in configuration
// and punycode can't be used to hide a malicious module hostname.
if strings.HasPrefix(addr.RawHost.Raw, "xn--") {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid module registry hostname",
Detail: "The hostname portion of this source address is not an acceptable hostname. Internationalized domain names must be given in unicode form rather than ASCII (\"punycode\") form.",
Subject: &req.SourceAddrRange,
})
} else {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid module registry hostname",
Detail: "The hostname portion of this source address is not a valid hostname.",
Subject: &req.SourceAddrRange,
})
}
return nil, nil, diags
}
reg := l.modules.Registry
log.Printf("[DEBUG] %s listing available versions of %s at %s", key, addr, hostname)
resp, err := reg.ModuleVersions(addr)
if err != nil {
if registry.IsModuleNotFound(err) {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Module not found",
Detail: fmt.Sprintf("The specified module could not be found in the module registry at %s.", hostname),
Subject: &req.SourceAddrRange,
})
} else {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Error accessing remote module registry",
Detail: fmt.Sprintf("Failed to retrieve available versions for this module from %s: %s.", hostname, err),
Subject: &req.SourceAddrRange,
})
}
return nil, nil, diags
}
// The response might contain information about dependencies to allow us
// to potentially optimize future requests, but we don't currently do that
// and so for now we'll just take the first item which is guaranteed to
// be the address we requested.
if len(resp.Modules) < 1 {
// Should never happen, but since this is a remote service that may
// be implemented by third-parties we will handle it gracefully.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid response from remote module registry",
Detail: fmt.Sprintf("The registry at %s returned an invalid response when Terraform requested available versions for this module.", hostname),
Subject: &req.SourceAddrRange,
})
return nil, nil, diags
}
modMeta := resp.Modules[0]
var latestMatch *version.Version
var latestVersion *version.Version
for _, mv := range modMeta.Versions {
v, err := version.NewVersion(mv.Version)
if err != nil {
// Should never happen if the registry server is compliant with
// the protocol, but we'll warn if not to assist someone who
// might be developing a module registry server.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagWarning,
Summary: "Invalid response from remote module registry",
Detail: fmt.Sprintf("The registry at %s returned an invalid version string %q for this module, which Terraform ignored.", hostname, mv.Version),
Subject: &req.SourceAddrRange,
})
continue
}
// If we've found a pre-release version then we'll ignore it unless
// it was exactly requested.
if v.Prerelease() != "" && req.VersionConstraint.Required.String() != v.String() {
log.Printf("[TRACE] %s ignoring %s because it is a pre-release and was not requested exactly", key, v)
continue
}
if latestVersion == nil || v.GreaterThan(latestVersion) {
latestVersion = v
}
if req.VersionConstraint.Required.Check(v) {
if latestMatch == nil || v.GreaterThan(latestMatch) {
latestMatch = v
}
}
}
if latestVersion == nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Module has no versions",
Detail: fmt.Sprintf("The specified module does not have any available versions."),
Subject: &req.SourceAddrRange,
})
return nil, nil, diags
}
if latestMatch == nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unresolvable module version constraint",
Detail: fmt.Sprintf("There is no available version of %q that matches the given version constraint. The newest available version is %s.", addr, latestVersion),
Subject: &req.VersionConstraint.DeclRange,
})
return nil, nil, diags
}
// Report up to the caller that we're about to start downloading.
packageAddr, _ := splitAddrSubdir(req.SourceAddr)
hooks.Download(key, packageAddr, latestMatch)
// If we manage to get down here then we've found a suitable version to
// install, so we need to ask the registry where we should download it from.
// The response to this is a go-getter-style address string.
dlAddr, err := reg.ModuleLocation(addr, latestMatch.String())
if err != nil {
log.Printf("[ERROR] %s from %s %s: %s", key, addr, latestMatch, err)
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid response from remote module registry",
Detail: fmt.Sprintf("The remote registry at %s failed to return a download URL for %s %s.", hostname, addr, latestMatch),
Subject: &req.VersionConstraint.DeclRange,
})
return nil, nil, diags
}
log.Printf("[TRACE] %s %s %s is available at %q", key, addr, latestMatch, dlAddr)
modDir, err := getter.getWithGoGetter(instPath, dlAddr)
if err != nil {
// Errors returned by go-getter have very inconsistent quality as
// end-user error messages, but for now we're accepting that because
// we have no way to recognize any specific errors to improve them
// and masking the error entirely would hide valuable diagnostic
// information from the user.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to download module",
Detail: fmt.Sprintf("Error attempting to download module source code from %q: %s", dlAddr, err),
Subject: &req.CallRange,
})
return nil, nil, diags
}
log.Printf("[TRACE] %s %q was downloaded to %s", key, dlAddr, modDir)
if addr.RawSubmodule != "" {
// Append the user's requested subdirectory to any subdirectory that
// was implied by any of the nested layers we expanded within go-getter.
modDir = filepath.Join(modDir, addr.RawSubmodule)
}
log.Printf("[TRACE] %s should now be at %s", key, modDir)
// Finally we are ready to try actually loading the module.
mod, mDiags := l.parser.LoadConfigDir(modDir)
if mod == nil {
// nil indicates missing or unreadable directory, so we'll
// discard the returned diags and return a more specific
// error message here. For registry modules this actually
// indicates a bug in the code above, since it's not the
// user's responsibility to create the directory in this case.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unreadable module directory",
Detail: fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir),
Subject: &req.CallRange,
})
} else {
diags = append(diags, mDiags...)
}
// Note the local location in our manifest.
l.modules.manifest[key] = moduleRecord{
Key: key,
Version: latestMatch,
Dir: modDir,
SourceAddr: req.SourceAddr,
}
log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir)
hooks.Install(key, latestMatch, modDir)
return mod, latestMatch, diags
}
func (l *Loader) installGoGetterModule(req *configs.ModuleRequest, key string, instPath string, hooks InstallHooks, getter reusingGetter) (*configs.Module, hcl.Diagnostics) {
var diags hcl.Diagnostics
// Report up to the caller that we're about to start downloading.
packageAddr, _ := splitAddrSubdir(req.SourceAddr)
hooks.Download(key, packageAddr, nil)
modDir, err := getter.getWithGoGetter(instPath, req.SourceAddr)
if err != nil {
// Errors returned by go-getter have very inconsistent quality as
// end-user error messages, but for now we're accepting that because
// we have no way to recognize any specific errors to improve them
// and masking the error entirely would hide valuable diagnostic
// information from the user.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to download module",
Detail: fmt.Sprintf("Error attempting to download module source code from %q: %s", packageAddr, err),
Subject: &req.SourceAddrRange,
})
return nil, diags
}
log.Printf("[TRACE] %s %q was downloaded to %s", key, req.SourceAddr, modDir)
mod, mDiags := l.parser.LoadConfigDir(modDir)
if mod == nil {
// nil indicates missing or unreadable directory, so we'll
// discard the returned diags and return a more specific
// error message here. For registry modules this actually
// indicates a bug in the code above, since it's not the
// user's responsibility to create the directory in this case.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unreadable module directory",
Detail: fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir),
Subject: &req.CallRange,
})
} else {
diags = append(diags, mDiags...)
}
// Note the local location in our manifest.
l.modules.manifest[key] = moduleRecord{
Key: key,
Dir: modDir,
SourceAddr: req.SourceAddr,
}
log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir)
hooks.Install(key, nil, modDir)
return mod, diags
}
func (l *Loader) packageInstallPath(modulePath []string) string {
return filepath.Join(l.modules.Dir, strings.Join(modulePath, "."))
}

34
configs/configload/loader_install_hooks.go
View File

@ -1,34 +0,0 @@
package configload
import version "github.com/hashicorp/go-version"
// InstallHooks is an interface used to provide notifications about the
// installation process being orchestrated by InstallModules.
//
// This interface may have new methods added in future, so implementers should
// embed InstallHooksImpl to get no-op implementations of any unimplemented
// methods.
type InstallHooks interface {
// Download is called for modules that are retrieved from a remote source
// before that download begins, to allow a caller to give feedback
// on progress through a possibly-long sequence of downloads.
Download(moduleAddr, packageAddr string, version *version.Version)
// Install is called for each module that is installed, even if it did
// not need to be downloaded from a remote source.
Install(moduleAddr string, version *version.Version, localPath string)
}
// InstallHooksImpl is a do-nothing implementation of InstallHooks that
// can be embedded in another implementation struct to allow only partial
// implementation of the interface.
type InstallHooksImpl struct {
}
func (h InstallHooksImpl) Download(moduleAddr, packageAddr string, version *version.Version) {
}
func (h InstallHooksImpl) Install(moduleAddr string, version *version.Version, localPath string) {
}
var _ InstallHooks = InstallHooksImpl{}

14
configs/configload/loader_load.go
View File

@ -24,6 +24,18 @@ func (l *Loader) LoadConfig(rootDir string) (*configs.Config, hcl.Diagnostics) {
return nil, diags
}
// Refresh the manifest snapshot in case anything new has been installed
// since we last refreshed it.
err := l.modules.readModuleManifestSnapshot()
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to read module manifest",
Detail: fmt.Sprintf("Terraform failed to read its manifest of locally-cached modules: %s.", err),
})
return nil, diags
}
cfg, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc(l.moduleWalkerLoad))
diags = append(diags, cDiags...)
@ -39,7 +51,7 @@ func (l *Loader) moduleWalkerLoad(req *configs.ModuleRequest) (*configs.Module,
// do verify that the manifest and the configuration are in agreement
// so that we can prompt the user to run "terraform init" if not.
key := manifestKey(req.Path)
key := l.modules.manifest.ModuleKey(req.Path)
record, exists := l.modules.manifest[key]
if !exists {

15
configs/configload/loader_snapshot.go
View File

@ -10,10 +10,9 @@ import (
version "github.com/hashicorp/go-version"
"github.com/hashicorp/hcl2/hcl"
"github.com/spf13/afero"
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/internal/modsdir"
"github.com/spf13/afero"
)
// LoadConfigWithSnapshot is a variant of LoadConfig that also simultaneously
@ -83,7 +82,7 @@ type Snapshot struct {
func NewEmptySnapshot() *Snapshot {
return &Snapshot{
Modules: map[string]*SnapshotModule{
manifestKey(addrs.RootModule): &SnapshotModule{
"": &SnapshotModule{
Files: map[string][]byte{},
},
},
@ -111,11 +110,11 @@ type SnapshotModule struct {
// moduleManifest constructs a module manifest based on the contents of
// the receiving snapshot.
func (s *Snapshot) moduleManifest() moduleManifest {
ret := make(moduleManifest)
func (s *Snapshot) moduleManifest() modsdir.Manifest {
ret := make(modsdir.Manifest)
for k, modSnap := range s.Modules {
ret[k] = moduleRecord{
ret[k] = modsdir.Record{
Key: k,
Dir: modSnap.Dir,
SourceAddr: modSnap.SourceAddr,
@ -137,7 +136,7 @@ func (l *Loader) makeModuleWalkerSnapshot(snap *Snapshot) configs.ModuleWalker {
return mod, v, diags
}
key := manifestKey(req.Path)
key := l.modules.manifest.ModuleKey(req.Path)
record, exists := l.modules.manifest[key]
if !exists {

58
configs/configload/testing.go
View File

@ -4,9 +4,6 @@ import (
"io/ioutil"
"os"
"testing"
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/tfdiags"
)
// NewLoaderForTests is a variant of NewLoader that is intended to be more
@ -44,58 +41,3 @@ func NewLoaderForTests(t *testing.T) (*Loader, func()) {
return loader, cleanup
}
// LoadConfigForTests is a convenience wrapper around NewLoaderForTests,
// Loader.InstallModules and Loader.LoadConfig that allows a test configuration
// to be loaded in a single step.
//
// If module installation fails, t.Fatal (or similar) is called to halt
// execution of the test, under the assumption that installation failures are
// not expected. If installation failures _are_ expected then use
// NewLoaderForTests and work with the loader object directly. If module
// installation succeeds but generates warnings, these warnings are discarded.
//
// If installation succeeds but errors are detected during loading then a
// possibly-incomplete config is returned along with error diagnostics. The
// test run is not aborted in this case, so that the caller can make assertions
// against the returned diagnostics.
//
// As with NewLoaderForTests, a cleanup function is returned which must be
// called before the test completes in order to remove the temporary
// modules directory.
func LoadConfigForTests(t *testing.T, rootDir string) (*configs.Config, *Loader, func(), tfdiags.Diagnostics) {
t.Helper()
var diags tfdiags.Diagnostics
loader, cleanup := NewLoaderForTests(t)
hclDiags := loader.InstallModules(rootDir, true, InstallHooksImpl{})
if diags.HasErrors() {
cleanup()
diags = diags.Append(hclDiags)
t.Fatal(diags.Err())
return nil, nil, cleanup, diags
}
config, hclDiags := loader.LoadConfig(rootDir)
diags = diags.Append(hclDiags)
return config, loader, cleanup, diags
}
// MustLoadConfigForTests is a variant of LoadConfigForTests which calls
// t.Fatal (or similar) if there are any errors during loading, and thus
// does not return diagnostics at all.
//
// This is useful for concisely writing tests that don't expect errors at
// all. For tests that expect errors and need to assert against them, use
// LoadConfigForTests instead.
func MustLoadConfigForTests(t *testing.T, rootDir string) (*configs.Config, *Loader, func()) {
t.Helper()
config, loader, cleanup, diags := LoadConfigForTests(t, rootDir)
if diags.HasErrors() {
cleanup()
t.Fatal(diags.Err())
}
return config, loader, cleanup
}

12
helper/resource/testing.go
View File

@ -26,6 +26,7 @@ import (
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/configs/configload"
"github.com/hashicorp/terraform/helper/logging"
"github.com/hashicorp/terraform/internal/initwd"
"github.com/hashicorp/terraform/providers"
"github.com/hashicorp/terraform/states"
"github.com/hashicorp/terraform/terraform"
@ -832,6 +833,12 @@ func testConfig(opts terraform.ContextOpts, step TestStep) (*configs.Config, err
return nil, fmt.Errorf("Error creating child modules directory: %s", err)
}
inst := initwd.NewModuleInstaller(modulesDir, nil)
installDiags := inst.InstallModules(cfgPath, true, initwd.ModuleInstallHooksImpl{})
if installDiags.HasErrors() {
return nil, installDiags.Err()
}
loader, err := configload.NewLoader(&configload.Config{
ModulesDir: modulesDir,
})
@ -839,11 +846,6 @@ func testConfig(opts terraform.ContextOpts, step TestStep) (*configs.Config, err
return nil, fmt.Errorf("failed to create config loader: %s", err)
}
installDiags := loader.InstallModules(cfgPath, true, configload.InstallHooksImpl{})
if installDiags.HasErrors() {
return nil, installDiags
}
config, configDiags := loader.LoadConfig(cfgPath)
if configDiags.HasErrors() {
return nil, configDiags

21
internal/earlyconfig/config_build.go
View File

@ -3,6 +3,7 @@ package earlyconfig
import (
"fmt"
"sort"
"strings"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/terraform-config-inspect/tfconfig"
@ -41,14 +42,18 @@ func buildChildModules(parent *Config, walker ModuleWalker) (map[string]*Config,
copy(path, parent.Path)
path[len(path)-1] = call.Name
vc, err := version.NewConstraint(call.Version)
if err != nil {
diags = diags.Append(wrapDiagnostic(tfconfig.Diagnostic{
Severity: tfconfig.DiagError,
Summary: "Invalid version constraint",
Detail: fmt.Sprintf("Module %q (declared at %s line %d) has invalid version constraint: %s.", callName, call.Pos.Filename, call.Pos.Line, err),
}))
continue
var vc version.Constraints
if strings.TrimSpace(call.Version) != "" {
var err error
vc, err = version.NewConstraint(call.Version)
if err != nil {
diags = diags.Append(wrapDiagnostic(tfconfig.Diagnostic{
Severity: tfconfig.DiagError,
Summary: "Invalid version constraint",
Detail: fmt.Sprintf("Module %q (declared at %s line %d) has invalid version constraint %q: %s.", callName, call.Pos.Filename, call.Pos.Line, call.Version, err),
}))
continue
}
}
req := ModuleRequest{

2
internal/init/copy_dir.go → internal/initwd/copy_dir.go
View File

@ -1,4 +1,4 @@
package init
package initwd
import (
"io"

4
internal/init/doc.go → internal/initwd/doc.go
View File

@ -1,7 +1,7 @@
// Package init contains various helper functions used by the "terraform init"
// Package initwd contains various helper functions used by the "terraform init"
// command.
//
// These functions may also be used from testing code to simulate the behaviors
// of "terraform init" against test fixtures, but should not be used elsewhere
// in the main code.
package init
package initwd

265
configs/configload/loader_init_from_module.go → internal/initwd/from_module.go
View File

@ -1,7 +1,9 @@
package configload
package initwd
import (
"fmt"
"github.com/hashicorp/terraform/internal/earlyconfig"
"io/ioutil"
"log"
"os"
"path/filepath"
@ -9,14 +11,16 @@ import (
"strings"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/hcl2/hcl"
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform-config-inspect/tfconfig"
"github.com/hashicorp/terraform/internal/modsdir"
"github.com/hashicorp/terraform/registry"
"github.com/hashicorp/terraform/tfdiags"
)
const initFromModuleRootCallName = "root"
const initFromModuleRootKeyPrefix = initFromModuleRootCallName + "."
// InitDirFromModule populates the given directory (which must exist and be
// DirFromModule populates the given directory (which must exist and be
// empty) with the contents of the module at the given source address.
//
// It does this by installing the given module and all of its descendent
@ -34,13 +38,8 @@ const initFromModuleRootKeyPrefix = initFromModuleRootCallName + "."
// references using ../ from that module to be unresolvable. Error diagnostics
// are produced in that case, to prompt the user to rewrite the source strings
// to be absolute references to the original remote module.
//
// This can be installed only on a loder that can install modules, and will
// panic otherwise. Use CanInstallModules to determine if this method can be
// used, or refer to the documentation of that method for situations where
// install ability is guaranteed.
func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHooks) hcl.Diagnostics {
var diags hcl.Diagnostics
func DirFromModule(rootDir, modulesDir, sourceAddr string, reg *registry.Client, hooks ModuleInstallHooks) tfdiags.Diagnostics {
var diags tfdiags.Diagnostics
// The way this function works is pretty ugly, but we accept it because
// -from-module is a less important case than normal module installation
@ -49,20 +48,20 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
// The target directory must exist but be empty.
{
entries, err := l.modules.FS.ReadDir(rootDir)
entries, err := ioutil.ReadDir(rootDir)
if err != nil {
if os.IsNotExist(err) {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Target directory does not exist",
Detail: fmt.Sprintf("Cannot initialize non-existent directory %s.", rootDir),
})
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Target directory does not exist",
fmt.Sprintf("Cannot initialize non-existent directory %s.", rootDir),
))
} else {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to read target directory",
Detail: fmt.Sprintf("Error reading %s to ensure it is empty: %s.", rootDir, err),
})
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to read target directory",
fmt.Sprintf("Error reading %s to ensure it is empty: %s.", rootDir, err),
))
}
return diags
}
@ -74,57 +73,36 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
haveEntries = true
}
if haveEntries {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Can't populate non-empty directory",
Detail: fmt.Sprintf("The target directory %s is not empty, so it cannot be initialized with the -from-module=... option.", rootDir),
})
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Can't populate non-empty directory",
fmt.Sprintf("The target directory %s is not empty, so it cannot be initialized with the -from-module=... option.", rootDir),
))
return diags
}
}
// We use a hidden sub-loader to manage our inner installation directory,
// but it shares dependencies with the receiver to allow it to access the
// same remote resources and ensure it populates the same source code
// cache in case .
subLoader := &Loader{
parser: l.parser,
modules: l.modules, // this is a shallow copy, so we can safely mutate below
}
// Our sub-loader will have its own independent manifest and install
// directory, so we can install with it and know we won't interfere
// with the receiver.
subLoader.modules.manifest = make(moduleManifest)
subLoader.modules.Dir = filepath.Join(rootDir, ".terraform/init-from-module")
log.Printf("[DEBUG] using a child module loader in %s to initialize working directory from %q", subLoader.modules.Dir, sourceAddr)
subLoader.modules.FS.RemoveAll(subLoader.modules.Dir) // if this fails then we'll fail on MkdirAll below too
err := subLoader.modules.FS.MkdirAll(subLoader.modules.Dir, os.ModePerm)
instDir := filepath.Join(rootDir, ".terraform/init-from-module")
inst := NewModuleInstaller(instDir, reg)
log.Printf("[DEBUG] installing modules in %s to initialize working directory from %q", instDir, sourceAddr)
os.RemoveAll(instDir) // if this fails then we'll fail on MkdirAll below too
err := os.MkdirAll(instDir, os.ModePerm)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to create temporary directory",
Detail: fmt.Sprintf("Failed to create temporary directory %s: %s.", subLoader.modules.Dir, err),
})
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to create temporary directory",
fmt.Sprintf("Failed to create temporary directory %s: %s.", instDir, err),
))
return diags
}
instManifest := make(modsdir.Manifest)
retManifest := make(modsdir.Manifest)
fakeFilename := fmt.Sprintf("-from-module=%q", sourceAddr)
fakeRange := hcl.Range{
fakePos := tfconfig.SourcePos{
Filename: fakeFilename,
Start: hcl.Pos{
Line: 1,
Column: 1,
Byte: 0,
},
End: hcl.Pos{
Line: 1,
Column: len(fakeFilename) + 1, // not accurate if the address contains unicode, but irrelevant since we have no source cache for this anyway
Byte: len(fakeFilename),
},
Line: 1,
}
// -from-module allows relative paths but it's different than a normal
@ -146,16 +124,12 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
// Now we need to create an artificial root module that will seed our
// installation process.
fakeRootModule := &configs.Module{
ModuleCalls: map[string]*configs.ModuleCall{
initFromModuleRootCallName: &configs.ModuleCall{
Name: initFromModuleRootCallName,
SourceAddr: sourceAddr,
SourceAddrRange: fakeRange,
SourceSet: true,
DeclRange: fakeRange,
fakeRootModule := &tfconfig.Module{
ModuleCalls: map[string]*tfconfig.ModuleCall{
initFromModuleRootCallName: {
Name: initFromModuleRootCallName,
Source: sourceAddr,
Pos: fakePos,
},
},
}
@ -167,7 +141,7 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
Wrapped: hooks,
}
getter := reusingGetter{}
instDiags := subLoader.installDescendentModules(fakeRootModule, rootDir, true, wrapHooks, getter)
instDiags := inst.installDescendentModules(fakeRootModule, rootDir, instManifest, true, wrapHooks, getter)
diags = append(diags, instDiags...)
if instDiags.HasErrors() {
return diags
@ -175,26 +149,24 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
// If all of that succeeded then we'll now migrate what was installed
// into the final directory structure.
modulesDir := l.modules.Dir
err = subLoader.modules.FS.MkdirAll(modulesDir, os.ModePerm)
err = os.MkdirAll(modulesDir, os.ModePerm)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to create local modules directory",
Detail: fmt.Sprintf("Failed to create modules directory %s: %s.", modulesDir, err),
})
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to create local modules directory",
fmt.Sprintf("Failed to create modules directory %s: %s.", modulesDir, err),
))
return diags
}
manifest := subLoader.modules.manifest
recordKeys := make([]string, 0, len(manifest))
for k := range manifest {
recordKeys := make([]string, 0, len(instManifest))
for k := range instManifest {
recordKeys = append(recordKeys, k)
}
sort.Strings(recordKeys)
for _, recordKey := range recordKeys {
record := manifest[recordKey]
record := instManifest[recordKey]
if record.Key == initFromModuleRootCallName {
// We've found the module the user requested, which we must
@ -202,11 +174,11 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
log.Printf("[TRACE] copying new root module from %s to %s", record.Dir, rootDir)
err := copyDir(rootDir, record.Dir)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to copy root module",
Detail: fmt.Sprintf("Error copying root module %q from %s to %s: %s.", sourceAddr, record.Dir, rootDir, err),
})
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to copy root module",
fmt.Sprintf("Error copying root module %q from %s to %s: %s.", sourceAddr, record.Dir, rootDir, err),
))
continue
}
@ -215,36 +187,37 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
// and must thus be rewritten to be absolute addresses again.
// For now we can't do this rewriting automatically, but we'll
// generate an error to help the user do it manually.
mod, _ := l.parser.LoadConfigDir(rootDir) // ignore diagnostics since we're just doing value-add here anyway
for _, mc := range mod.ModuleCalls {
if pathTraversesUp(sourceAddr) {
packageAddr, givenSubdir := splitAddrSubdir(sourceAddr)
newSubdir := filepath.Join(givenSubdir, mc.SourceAddr)
if pathTraversesUp(newSubdir) {
// This should never happen in any reasonable
// configuration since this suggests a path that
// traverses up out of the package root. We'll just
// ignore this, since we'll fail soon enough anyway
// trying to resolve this path when this module is
// loaded.
mod, _ := earlyconfig.LoadModule(rootDir) // ignore diagnostics since we're just doing value-add here anyway
if mod != nil {
for _, mc := range mod.ModuleCalls {
if pathTraversesUp(mc.Source) {
packageAddr, givenSubdir := splitAddrSubdir(sourceAddr)
newSubdir := filepath.Join(givenSubdir, mc.Source)
if pathTraversesUp(newSubdir) {
// This should never happen in any reasonable
// configuration since this suggests a path that
// traverses up out of the package root. We'll just
// ignore this, since we'll fail soon enough anyway
// trying to resolve this path when this module is
// loaded.
continue
}
var newAddr = packageAddr
if newSubdir != "" {
newAddr = fmt.Sprintf("%s//%s", newAddr, filepath.ToSlash(newSubdir))
}
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Root module references parent directory",
fmt.Sprintf("The requested module %q refers to a module via its parent directory. To use this as a new root module this source string must be rewritten as a remote source address, such as %q.", sourceAddr, newAddr),
))
continue
}
var newAddr = packageAddr
if newSubdir != "" {
newAddr = fmt.Sprintf("%s//%s", newAddr, filepath.ToSlash(newSubdir))
}
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Root module references parent directory",
Detail: fmt.Sprintf("The requested module %q refers to a module via its parent directory. To use this as a new root module this source string must be rewritten as a remote source address, such as %q.", sourceAddr, newAddr),
Subject: &mc.SourceAddrRange,
})
continue
}
}
l.modules.manifest[""] = moduleRecord{
retManifest[""] = modsdir.Record{
Key: "",
Dir: rootDir,
}
@ -259,8 +232,8 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
}
newKey := record.Key[len(initFromModuleRootKeyPrefix):]
instPath := filepath.Join(l.modules.Dir, newKey)
tempPath := filepath.Join(subLoader.modules.Dir, record.Key)
instPath := filepath.Join(modulesDir, newKey)
tempPath := filepath.Join(instDir, record.Key)
// tempPath won't be present for a module that was installed from
// a relative path, so in that case we just record the installation
@ -268,11 +241,11 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
// of its parent.
if _, err := os.Stat(tempPath); err != nil {
if !os.IsNotExist(err) {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to stat temporary module install directory",
Detail: fmt.Sprintf("Error from stat %s for module %s: %s.", instPath, newKey, err),
})
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to stat temporary module install directory",
fmt.Sprintf("Error from stat %s for module %s: %s.", instPath, newKey, err),
))
continue
}
@ -283,8 +256,8 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
parentKey = "" // parent is the root module
}
parentOld := manifest[initFromModuleRootKeyPrefix+parentKey]
parentNew := l.modules.manifest[parentKey]
parentOld := instManifest[initFromModuleRootKeyPrefix+parentKey]
parentNew := retManifest[parentKey]
// We need to figure out which portion of our directory is the
// parent package path and which portion is the subdirectory
@ -301,18 +274,18 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
newRecord := record // shallow copy
newRecord.Dir = newDir
newRecord.Key = newKey
l.modules.manifest[newKey] = newRecord
retManifest[newKey] = newRecord
hooks.Install(newRecord.Key, newRecord.Version, newRecord.Dir)
continue
}
err = subLoader.modules.FS.MkdirAll(instPath, os.ModePerm)
err = os.MkdirAll(instPath, os.ModePerm)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to create module install directory",
Detail: fmt.Sprintf("Error creating directory %s for module %s: %s.", instPath, newKey, err),
})
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to create module install directory",
fmt.Sprintf("Error creating directory %s for module %s: %s.", instPath, newKey, err),
))
continue
}
@ -321,11 +294,11 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
log.Printf("[TRACE] copying new module %s from %s to %s", newKey, record.Dir, instPath)
err := copyDir(instPath, tempPath)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to copy descendent module",
Detail: fmt.Sprintf("Error copying module %q from %s to %s: %s.", newKey, tempPath, rootDir, err),
})
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to copy descendent module",
fmt.Sprintf("Error copying module %q from %s to %s: %s.", newKey, tempPath, rootDir, err),
))
continue
}
@ -339,23 +312,23 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
newRecord := record // shallow copy
newRecord.Dir = filepath.Join(instPath, subDir)
newRecord.Key = newKey
l.modules.manifest[newKey] = newRecord
retManifest[newKey] = newRecord
hooks.Install(newRecord.Key, newRecord.Version, newRecord.Dir)
}
err = l.modules.writeModuleManifestSnapshot()
retManifest.WriteSnapshotToDir(modulesDir)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to write module manifest",
Detail: fmt.Sprintf("Error writing module manifest: %s.", err),
})
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to write module manifest",
fmt.Sprintf("Error writing module manifest: %s.", err),
))
}
if !diags.HasErrors() {
// Try to clean up our temporary directory, but don't worry if we don't
// succeed since it shouldn't hurt anything.
subLoader.modules.FS.RemoveAll(subLoader.modules.Dir)
os.RemoveAll(instDir)
}
return diags
@ -373,8 +346,8 @@ func pathTraversesUp(path string) bool {
// does its own installation steps after the initial installation pass
// has completed.
type installHooksInitDir struct {
Wrapped InstallHooks
InstallHooksImpl
Wrapped ModuleInstallHooks
ModuleInstallHooksImpl
}
func (h installHooksInitDir) Download(moduleAddr, packageAddr string, version *version.Version) {

22
configs/configload/loader_init_from_module_test.go → internal/initwd/from_module_test.go
View File

@ -1,4 +1,4 @@
package configload
package initwd
import (
"os"
@ -8,20 +8,25 @@ import (
version "github.com/hashicorp/go-version"
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/configs/configload"
"github.com/hashicorp/terraform/registry"
"github.com/hashicorp/terraform/tfdiags"
)
func TestLoaderInitDirFromModule_registry(t *testing.T) {
func TestDirFromModule_registry(t *testing.T) {
if os.Getenv("TF_ACC") == "" {
t.Skip("this test accesses registry.terraform.io and github.com; set TF_ACC=1 to run it")
}
fixtureDir := filepath.Clean("test-fixtures/empty")
loader, done := tempChdirLoader(t, fixtureDir)
dir, done := tempChdir(t, fixtureDir)
modsDir := filepath.Join(dir, ".terraform/modules")
defer done()
hooks := &testInstallHooks{}
diags := loader.InitDirFromModule(".", "hashicorp/module-installer-acctest/aws//examples/main", hooks)
reg := registry.NewClient(nil, nil)
diags := DirFromModule(dir, modsDir, "hashicorp/module-installer-acctest/aws//examples/main", reg, hooks)
assertNoDiagnostics(t, diags)
v := version.Must(version.NewVersion("0.0.1"))
@ -65,10 +70,17 @@ func TestLoaderInitDirFromModule_registry(t *testing.T) {
return
}
loader, err := configload.NewLoader(&configload.Config{
ModulesDir: modsDir,
})
if err != nil {
t.Fatal(err)
}
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfig(".")
if assertNoDiagnostics(t, loadDiags) {
if assertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags)) {
return
}

4
internal/init/getter.go → internal/initwd/getter.go
View File

@ -1,4 +1,4 @@
package init
package initwd
import (
"fmt"
@ -23,6 +23,7 @@ var goGetterDetectors = []getter.Detector{
new(getter.GitHubDetector),
new(getter.BitBucketDetector),
new(getter.S3Detector),
new(getter.FileDetector),
}
var goGetterNoDetectors = []getter.Detector{}
@ -44,6 +45,7 @@ var goGetterDecompressors = map[string]getter.Decompressor{
}
var goGetterGetters = map[string]getter.Getter{
"file": new(getter.FileGetter),
"git": new(getter.GitGetter),
"hg": new(getter.HgGetter),
"s3": new(getter.S3Getter),

2
internal/init/inode.go → internal/initwd/inode.go
View File

@ -1,6 +1,6 @@
// +build linux darwin openbsd netbsd solaris dragonfly
package init
package initwd
import (
"fmt"

2
internal/init/inode_freebsd.go → internal/initwd/inode_freebsd.go
View File

@ -1,6 +1,6 @@
// +build freebsd
package init
package initwd
import (
"fmt"

2
internal/init/inode_windows.go → internal/initwd/inode_windows.go
View File

@ -1,6 +1,6 @@
// +build windows
package init
package initwd
// no syscall.Stat_t on windows, return 0 for inodes
func inode(path string) (uint64, error) {

50
internal/init/module_install.go → internal/initwd/module_install.go
View File

@ -1,4 +1,4 @@
package init
package initwd
import (
"fmt"
@ -58,7 +58,9 @@ func NewModuleInstaller(modsDir string, reg *registry.Client) *ModuleInstaller {
// Use CanInstallModules to determine if a loader can install modules, or
// refer to the documentation for that method for situations where module
// installation capability is guaranteed.
func (i *ModuleInstaller) InstallModules(rootDir string, upgrade bool, hooks InstallHooks) tfdiags.Diagnostics {
func (i *ModuleInstaller) InstallModules(rootDir string, upgrade bool, hooks ModuleInstallHooks) tfdiags.Diagnostics {
log.Printf("[TRACE] ModuleInstaller: installing child modules for %s into %s", rootDir, i.modsDir)
rootMod, diags := earlyconfig.LoadModule(rootDir)
if rootMod == nil {
return diags
@ -81,12 +83,12 @@ func (i *ModuleInstaller) InstallModules(rootDir string, upgrade bool, hooks Ins
return diags
}
func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, rootDir string, manifest modsdir.Manifest, upgrade bool, hooks InstallHooks, getter reusingGetter) tfdiags.Diagnostics {
func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, rootDir string, manifest modsdir.Manifest, upgrade bool, hooks ModuleInstallHooks, getter reusingGetter) tfdiags.Diagnostics {
var diags tfdiags.Diagnostics
if hooks == nil {
// Use our no-op implementation as a placeholder
hooks = InstallHooksImpl{}
hooks = ModuleInstallHooksImpl{}
}
// Create a manifest record for the root module. This will be used if
@ -111,13 +113,13 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
record, recorded := manifest[key]
switch {
case !recorded:
log.Printf("[TRACE] %s is not yet installed", key)
log.Printf("[TRACE] ModuleInstaller: %s is not yet installed", key)
replace = true
case record.SourceAddr != req.SourceAddr:
log.Printf("[TRACE] %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr)
log.Printf("[TRACE] ModuleInstaller: %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr)
replace = true
case record.Version != nil && !req.VersionConstraints.Check(record.Version):
log.Printf("[TRACE] %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraints)
log.Printf("[TRACE] ModuleInstaller: %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraints)
replace = true
}
}
@ -127,7 +129,7 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
// existing remnants.
if replace {
if _, recorded := manifest[key]; recorded {
log.Printf("[TRACE] discarding previous record of %s prior to reinstall", key)
log.Printf("[TRACE] ModuleInstaller: discarding previous record of %s prior to reinstall", key)
}
delete(manifest, key)
// Deleting a module invalidates all of its descendent modules too.
@ -135,7 +137,7 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
for subKey := range manifest {
if strings.HasPrefix(subKey, keyPrefix) {
if _, recorded := manifest[subKey]; recorded {
log.Printf("[TRACE] also discarding downstream %s", subKey)
log.Printf("[TRACE] ModuleInstaller: also discarding downstream %s", subKey)
}
delete(manifest, subKey)
}
@ -147,10 +149,10 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
// Clean up any stale cache directory that might be present.
// If this is a local (relative) source then the dir will
// not exist, but we'll ignore that.
log.Printf("[TRACE] cleaning directory %s prior to install of %s", instPath, key)
log.Printf("[TRACE] ModuleInstaller: cleaning directory %s prior to install of %s", instPath, key)
err := os.RemoveAll(instPath)
if err != nil && !os.IsNotExist(err) {
log.Printf("[TRACE] failed to remove %s: %s", key, err)
log.Printf("[TRACE] ModuleInstaller: failed to remove %s: %s", key, err)
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to remove local module cache",
@ -170,7 +172,7 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
mod, mDiags := earlyconfig.LoadModule(record.Dir)
diags = diags.Append(mDiags)
log.Printf("[TRACE] Module installer: %s %s already installed in %s", key, record.Version, record.Dir)
log.Printf("[TRACE] ModuleInstaller: Module installer: %s %s already installed in %s", key, record.Version, record.Dir)
return mod, record.Version, diags
}
}
@ -181,7 +183,7 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
switch {
case isLocalSourceAddr(req.SourceAddr):
log.Printf("[TRACE] %s has local path %q", key, req.SourceAddr)
log.Printf("[TRACE] ModuleInstaller: %s has local path %q", key, req.SourceAddr)
mod, mDiags := i.installLocalModule(req, key, manifest, hooks)
diags = append(diags, mDiags...)
return mod, nil, diags
@ -192,14 +194,14 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
// Should never happen because isRegistrySourceAddr already validated
panic(err)
}
log.Printf("[TRACE] %s is a registry module at %s", key, addr)
log.Printf("[TRACE] ModuleInstaller: %s is a registry module at %s", key, addr)
mod, v, mDiags := i.installRegistryModule(req, key, instPath, addr, manifest, hooks, getter)
diags = append(diags, mDiags...)
return mod, v, diags
default:
log.Printf("[TRACE] %s address %q will be handled by go-getter", key, req.SourceAddr)
log.Printf("[TRACE] ModuleInstaller: %s address %q will be handled by go-getter", key, req.SourceAddr)
mod, mDiags := i.installGoGetterModule(req, key, instPath, manifest, hooks, getter)
diags = append(diags, mDiags...)
@ -222,7 +224,7 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
return diags
}
func (i *ModuleInstaller) installLocalModule(req *earlyconfig.ModuleRequest, key string, manifest modsdir.Manifest, hooks InstallHooks) (*tfconfig.Module, tfdiags.Diagnostics) {
func (i *ModuleInstaller) installLocalModule(req *earlyconfig.ModuleRequest, key string, manifest modsdir.Manifest, hooks ModuleInstallHooks) (*tfconfig.Module, tfdiags.Diagnostics) {
var diags tfdiags.Diagnostics
parentKey := manifest.ModuleKey(req.Parent.Path)
@ -244,7 +246,7 @@ func (i *ModuleInstaller) installLocalModule(req *earlyconfig.ModuleRequest, key
// filesystem at all because the parent already wrote
// the files we need, and so we just load up what's already here.
newDir := filepath.Join(parentRecord.Dir, req.SourceAddr)
log.Printf("[TRACE] %s uses directory from parent: %s", key, newDir)
log.Printf("[TRACE] ModuleInstaller: %s uses directory from parent: %s", key, newDir)
mod, mDiags := earlyconfig.LoadModule(newDir)
if mod == nil {
// nil indicates missing or unreadable directory, so we'll
@ -271,7 +273,7 @@ func (i *ModuleInstaller) installLocalModule(req *earlyconfig.ModuleRequest, key
return mod, diags
}
func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest, key string, instPath string, addr *regsrc.Module, manifest modsdir.Manifest, hooks InstallHooks, getter reusingGetter) (*tfconfig.Module, *version.Version, tfdiags.Diagnostics) {
func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest, key string, instPath string, addr *regsrc.Module, manifest modsdir.Manifest, hooks ModuleInstallHooks, getter reusingGetter) (*tfconfig.Module, *version.Version, tfdiags.Diagnostics) {
var diags tfdiags.Diagnostics
hostname, err := addr.SvcHost()
@ -353,7 +355,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest,
// If we've found a pre-release version then we'll ignore it unless
// it was exactly requested.
if v.Prerelease() != "" && req.VersionConstraints.String() != v.String() {
log.Printf("[TRACE] %s ignoring %s because it is a pre-release and was not requested exactly", key, v)
log.Printf("[TRACE] ModuleInstaller: %s ignoring %s because it is a pre-release and was not requested exactly", key, v)
continue
}
@ -404,7 +406,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest,
return nil, nil, diags
}
log.Printf("[TRACE] %s %s %s is available at %q", key, addr, latestMatch, dlAddr)
log.Printf("[TRACE] ModuleInstaller: %s %s %s is available at %q", key, addr, latestMatch, dlAddr)
modDir, err := getter.getWithGoGetter(instPath, dlAddr)
if err != nil {
@ -421,7 +423,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest,
return nil, nil, diags
}
log.Printf("[TRACE] %s %q was downloaded to %s", key, dlAddr, modDir)
log.Printf("[TRACE] ModuleInstaller: %s %q was downloaded to %s", key, dlAddr, modDir)
if addr.RawSubmodule != "" {
// Append the user's requested subdirectory to any subdirectory that
@ -429,7 +431,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest,
modDir = filepath.Join(modDir, addr.RawSubmodule)
}
log.Printf("[TRACE] %s should now be at %s", key, modDir)
log.Printf("[TRACE] ModuleInstaller: %s should now be at %s", key, modDir)
// Finally we are ready to try actually loading the module.
mod, mDiags := earlyconfig.LoadModule(modDir)
@ -461,7 +463,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest,
return mod, latestMatch, diags
}
func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest, key string, instPath string, manifest modsdir.Manifest, hooks InstallHooks, getter reusingGetter) (*tfconfig.Module, tfdiags.Diagnostics) {
func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest, key string, instPath string, manifest modsdir.Manifest, hooks ModuleInstallHooks, getter reusingGetter) (*tfconfig.Module, tfdiags.Diagnostics) {
var diags tfdiags.Diagnostics
// Report up to the caller that we're about to start downloading.
@ -483,7 +485,7 @@ func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest,
return nil, diags
}
log.Printf("[TRACE] %s %q was downloaded to %s", key, req.SourceAddr, modDir)
log.Printf("[TRACE] ModuleInstaller: %s %q was downloaded to %s", key, req.SourceAddr, modDir)
mod, mDiags := earlyconfig.LoadModule(modDir)
if mod == nil {

16
internal/init/module_install_hooks.go → internal/initwd/module_install_hooks.go
View File

@ -1,16 +1,16 @@
package init
package initwd
import (
version "github.com/hashicorp/go-version"
)
// InstallHooks is an interface used to provide notifications about the
// ModuleInstallHooks is an interface used to provide notifications about the
// installation process being orchestrated by InstallModules.
//
// This interface may have new methods added in future, so implementers should
// embed InstallHooksImpl to get no-op implementations of any unimplemented
// methods.
type InstallHooks interface {
type ModuleInstallHooks interface {
// Download is called for modules that are retrieved from a remote source
// before that download begins, to allow a caller to give feedback
// on progress through a possibly-long sequence of downloads.
@ -21,16 +21,16 @@ type InstallHooks interface {
Install(moduleAddr string, version *version.Version, localPath string)
}
// InstallHooksImpl is a do-nothing implementation of InstallHooks that
// ModuleInstallHooksImpl is a do-nothing implementation of InstallHooks that
// can be embedded in another implementation struct to allow only partial
// implementation of the interface.
type InstallHooksImpl struct {
type ModuleInstallHooksImpl struct {
}
func (h InstallHooksImpl) Download(moduleAddr, packageAddr string, version *version.Version) {
func (h ModuleInstallHooksImpl) Download(moduleAddr, packageAddr string, version *version.Version) {
}
func (h InstallHooksImpl) Install(moduleAddr string, version *version.Version, localPath string) {
func (h ModuleInstallHooksImpl) Install(moduleAddr string, version *version.Version, localPath string) {
}
var _ InstallHooks = InstallHooksImpl{}
var _ ModuleInstallHooks = ModuleInstallHooksImpl{}

169
configs/configload/loader_install_test.go → internal/initwd/module_install_test.go
View File

@ -1,23 +1,47 @@
package configload
package initwd
import (
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"strings"
"testing"
"github.com/go-test/deep"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/configs/configload"
"github.com/hashicorp/terraform/helper/logging"
"github.com/hashicorp/terraform/registry"
"github.com/hashicorp/terraform/tfdiags"
)
func TestLoaderInstallModules_local(t *testing.T) {
func TestMain(m *testing.M) {
flag.Parse()
if testing.Verbose() {
// if we're verbose, use the logging requested by TF_LOG
logging.SetOutput()
} else {
// otherwise silence all logs
log.SetOutput(ioutil.Discard)
}
os.Exit(m.Run())
}
func TestModuleInstaller(t *testing.T) {
fixtureDir := filepath.Clean("test-fixtures/local-modules")
loader, done := tempChdirLoader(t, fixtureDir)
dir, done := tempChdir(t, fixtureDir)
defer done()
hooks := &testInstallHooks{}
diags := loader.InstallModules(".", false, hooks)
modulesDir := filepath.Join(dir, ".terraform/modules")
inst := NewModuleInstaller(modulesDir, nil)
diags := inst.InstallModules(".", false, hooks)
assertNoDiagnostics(t, diags)
wantCalls := []testInstallHookCall{
@ -39,10 +63,17 @@ func TestLoaderInstallModules_local(t *testing.T) {
return
}
loader, err := configload.NewLoader(&configload.Config{
ModulesDir: modulesDir,
})
if err != nil {
t.Fatal(err)
}
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfig(".")
assertNoDiagnostics(t, loadDiags)
assertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags))
wantTraces := map[string]string{
"": "in root module",
@ -67,13 +98,14 @@ func TestLoaderInstallModules_registry(t *testing.T) {
t.Skip("this test accesses registry.terraform.io and github.com; set TF_ACC=1 to run it")
}
fixtureDir := filepath.Clean("test-fixtures/registry-modules")
loader, done := tempChdirLoader(t, fixtureDir)
fixtureDir := filepath.Clean("test-fixtures/local-modules")
dir, done := tempChdir(t, fixtureDir)
defer done()
hooks := &testInstallHooks{}
diags := loader.InstallModules(".", false, hooks)
modulesDir := filepath.Join(dir, ".terraform/modules")
inst := NewModuleInstaller(modulesDir, registry.NewClient(nil, nil))
diags := inst.InstallModules(dir, false, hooks)
assertNoDiagnostics(t, diags)
v := version.Must(version.NewVersion("0.0.1"))
@ -153,10 +185,17 @@ func TestLoaderInstallModules_registry(t *testing.T) {
return
}
loader, err := configload.NewLoader(&configload.Config{
ModulesDir: modulesDir,
})
if err != nil {
t.Fatal(err)
}
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfig(".")
assertNoDiagnostics(t, loadDiags)
assertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags))
wantTraces := map[string]string{
"": "in local caller for registry-modules",
@ -187,12 +226,13 @@ func TestLoaderInstallModules_goGetter(t *testing.T) {
}
fixtureDir := filepath.Clean("test-fixtures/go-getter-modules")
loader, done := tempChdirLoader(t, fixtureDir)
dir, done := tempChdir(t, fixtureDir)
defer done()
hooks := &testInstallHooks{}
diags := loader.InstallModules(".", false, hooks)
modulesDir := filepath.Join(dir, ".terraform/modules")
inst := NewModuleInstaller(modulesDir, registry.NewClient(nil, nil))
diags := inst.InstallModules(dir, false, hooks)
assertNoDiagnostics(t, diags)
wantCalls := []testInstallHookCall{
@ -264,10 +304,17 @@ func TestLoaderInstallModules_goGetter(t *testing.T) {
return
}
loader, err := configload.NewLoader(&configload.Config{
ModulesDir: modulesDir,
})
if err != nil {
t.Fatal(err)
}
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfig(".")
assertNoDiagnostics(t, loadDiags)
assertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags))
wantTraces := map[string]string{
"": "in local caller for go-getter-modules",
@ -321,3 +368,97 @@ func (h *testInstallHooks) Install(moduleAddr string, version *version.Version,
LocalPath: localPath,
})
}
// tempChdir copies the contents of the given directory to a temporary
// directory and changes the test process's current working directory to
// point to that directory. Also returned is a function that should be
// called at the end of the test (e.g. via "defer") to restore the previous
// working directory.
//
// Tests using this helper cannot safely be run in parallel with other tests.
func tempChdir(t *testing.T, sourceDir string) (string, func()) {
t.Helper()
tmpDir, err := ioutil.TempDir("", "terraform-configload")
if err != nil {
t.Fatalf("failed to create temporary directory: %s", err)
return "", nil
}
if err := copyDir(tmpDir, sourceDir); err != nil {
t.Fatalf("failed to copy fixture to temporary directory: %s", err)
return "", nil
}
oldDir, err := os.Getwd()
if err != nil {
t.Fatalf("failed to determine current working directory: %s", err)
return "", nil
}
err = os.Chdir(tmpDir)
if err != nil {
t.Fatalf("failed to switch to temp dir %s: %s", tmpDir, err)
return "", nil
}
// Most of the tests need this, so we'll make it just in case.
os.MkdirAll(filepath.Join(tmpDir, ".terraform/modules"), os.ModePerm)
t.Logf("tempChdir switched to %s after copying from %s", tmpDir, sourceDir)
return tmpDir, func() {
err := os.Chdir(oldDir)
if err != nil {
panic(fmt.Errorf("failed to restore previous working directory %s: %s", oldDir, err))
}
if os.Getenv("TF_CONFIGLOAD_TEST_KEEP_TMP") == "" {
os.RemoveAll(tmpDir)
}
}
}
func assertNoDiagnostics(t *testing.T, diags tfdiags.Diagnostics) bool {
t.Helper()
return assertDiagnosticCount(t, diags, 0)
}
func assertDiagnosticCount(t *testing.T, diags tfdiags.Diagnostics, want int) bool {
t.Helper()
if len(diags) != 0 {
t.Errorf("wrong number of diagnostics %d; want %d", len(diags), want)
for _, diag := range diags {
t.Logf("- %s", diag)
}
return true
}
return false
}
func assertDiagnosticSummary(t *testing.T, diags tfdiags.Diagnostics, want string) bool {
t.Helper()
for _, diag := range diags {
if diag.Description().Summary == want {
return false
}
}
t.Errorf("missing diagnostic summary %q", want)
for _, diag := range diags {
t.Logf("- %s", diag)
}
return true
}
func assertResultDeepEqual(t *testing.T, got, want interface{}) bool {
t.Helper()
if diff := deep.Equal(got, want); diff != nil {
for _, problem := range diff {
t.Errorf("%s", problem)
}
return true
}
return false
}

10
internal/initwd/test-fixtures/already-installed/root.tf Normal file
View File

@ -0,0 +1,10 @@
module "child_a" {
source = "example.com/foo/bar_a/baz"
version = ">= 1.0.0"
}
module "child_b" {
source = "example.com/foo/bar_b/baz"
version = ">= 1.0.0"
}

0
internal/initwd/test-fixtures/empty/.gitignore vendored Normal file
View File

1
internal/initwd/test-fixtures/go-getter-modules/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
.terraform/*

21
internal/initwd/test-fixtures/go-getter-modules/root.tf Normal file
View File

@ -0,0 +1,21 @@
# This fixture depends on a github repo at:
# https://github.com/hashicorp/terraform-aws-module-installer-acctest
# ...and expects its v0.0.1 tag to be pointing at the following commit:
# d676ab2559d4e0621d59e3c3c4cbb33958ac4608
variable "v" {
description = "in local caller for go-getter-modules"
default = ""
}
module "acctest_root" {
source = "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1"
}
module "acctest_child_a" {
source = "github.com/hashicorp/terraform-aws-module-installer-acctest//modules/child_a?ref=v0.0.1"
}
module "acctest_child_b" {
source = "github.com/hashicorp/terraform-aws-module-installer-acctest//modules/child_b?ref=v0.0.1"
}

9
internal/initwd/test-fixtures/local-modules/child_a/child_a.tf Normal file
View File

@ -0,0 +1,9 @@
variable "v" {
description = "in child_a module"
default = ""
}
module "child_b" {
source = "./child_b"
}

9
internal/initwd/test-fixtures/local-modules/child_a/child_b/child_b.tf Normal file
View File

@ -0,0 +1,9 @@
variable "v" {
description = "in child_b module"
default = ""
}
output "hello" {
value = "Hello from child_b!"
}

9
internal/initwd/test-fixtures/local-modules/root.tf Normal file
View File

@ -0,0 +1,9 @@
variable "v" {
description = "in root module"
default = ""
}
module "child_a" {
source = "./child_a"
}

1
internal/initwd/test-fixtures/registry-modules/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
.terraform/*

33
internal/initwd/test-fixtures/registry-modules/root.tf Normal file
View File

@ -0,0 +1,33 @@
# This fixture indirectly depends on a github repo at:
# https://github.com/hashicorp/terraform-aws-module-installer-acctest
# ...and expects its v0.0.1 tag to be pointing at the following commit:
# d676ab2559d4e0621d59e3c3c4cbb33958ac4608
#
# This repository is accessed indirectly via:
# https://registry.terraform.io/modules/hashicorp/module-installer-acctest/aws/0.0.1
#
# Since the tag's id is included in a downloaded archive, it is expected to
# have the following id:
# 853d03855b3290a3ca491d4c3a7684572dd42237
# (this particular assumption is encoded in the tests that use this fixture)
variable "v" {
description = "in local caller for registry-modules"
default = ""
}
module "acctest_root" {
source = "hashicorp/module-installer-acctest/aws"
version = "0.0.1"
}
module "acctest_child_a" {
source = "hashicorp/module-installer-acctest/aws//modules/child_a"
version = "0.0.1"
}
module "acctest_child_b" {
source = "hashicorp/module-installer-acctest/aws//modules/child_b"
version = "0.0.1"
}

67
internal/initwd/testing.go Normal file
View File

@ -0,0 +1,67 @@
package initwd
import (
"github.com/hashicorp/terraform/registry"
"testing"
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/configs/configload"
"github.com/hashicorp/terraform/tfdiags"
)
// LoadConfigForTests is a convenience wrapper around configload.NewLoaderForTests,
// ModuleInstaller.InstallModules and configload.Loader.LoadConfig that allows
// a test configuration to be loaded in a single step.
//
// If module installation fails, t.Fatal (or similar) is called to halt
// execution of the test, under the assumption that installation failures are
// not expected. If installation failures _are_ expected then use
// NewLoaderForTests and work with the loader object directly. If module
// installation succeeds but generates warnings, these warnings are discarded.
//
// If installation succeeds but errors are detected during loading then a
// possibly-incomplete config is returned along with error diagnostics. The
// test run is not aborted in this case, so that the caller can make assertions
// against the returned diagnostics.
//
// As with NewLoaderForTests, a cleanup function is returned which must be
// called before the test completes in order to remove the temporary
// modules directory.
func LoadConfigForTests(t *testing.T, rootDir string) (*configs.Config, *configload.Loader, func(), tfdiags.Diagnostics) {
t.Helper()
var diags tfdiags.Diagnostics
loader, cleanup := configload.NewLoaderForTests(t)
inst := NewModuleInstaller(loader.ModulesDir(), registry.NewClient(nil, nil))
moreDiags := inst.InstallModules(rootDir, true, ModuleInstallHooksImpl{})
diags = diags.Append(moreDiags)
if diags.HasErrors() {
cleanup()
t.Fatal(diags.Err())
return nil, nil, func() {}, diags
}
config, hclDiags := loader.LoadConfig(rootDir)
diags = diags.Append(hclDiags)
return config, loader, cleanup, diags
}
// MustLoadConfigForTests is a variant of LoadConfigForTests which calls
// t.Fatal (or similar) if there are any errors during loading, and thus
// does not return diagnostics at all.
//
// This is useful for concisely writing tests that don't expect errors at
// all. For tests that expect errors and need to assert against them, use
// LoadConfigForTests instead.
func MustLoadConfigForTests(t *testing.T, rootDir string) (*configs.Config, *configload.Loader, func()) {
t.Helper()
config, loader, cleanup, diags := LoadConfigForTests(t, rootDir)
if diags.HasErrors() {
cleanup()
t.Fatal(diags.Err())
}
return config, loader, cleanup
}

5
internal/modsdir/manifest.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
@ -95,6 +96,9 @@ func ReadManifestSnapshotForDir(dir string) (Manifest, error) {
fn := filepath.Join(dir, ManifestSnapshotFilename)
r, err := os.Open(fn)
if err != nil {
if os.IsNotExist(err) {
return make(Manifest), nil // missing file is okay and treated as empty
}
return nil, err
}
return ReadManifestSnapshot(r)
@ -125,6 +129,7 @@ func (m Manifest) WriteSnapshot(w io.Writer) error {
func (m Manifest) WriteSnapshotToDir(dir string) error {
fn := filepath.Join(dir, ManifestSnapshotFilename)
log.Printf("[TRACE] modsdir: writing modules manifest to %s", fn)
w, err := os.Create(fn)
if err != nil {
return err

13
repl/session_test.go
View File

@ -2,6 +2,7 @@ package repl
import (
"flag"
"github.com/hashicorp/terraform/internal/initwd"
"io/ioutil"
"log"
"os"
@ -11,7 +12,6 @@ import (
"github.com/zclconf/go-cty/cty"
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/configs/configload"
"github.com/hashicorp/terraform/configs/configschema"
"github.com/hashicorp/terraform/helper/logging"
"github.com/hashicorp/terraform/providers"
@ -202,17 +202,10 @@ func testSession(t *testing.T, test testSessionTest) {
},
}
loader, cleanup := configload.NewLoaderForTests(t)
config, _, cleanup, configDiags := initwd.LoadConfigForTests(t, "testdata/config-fixture")
defer cleanup()
configDiags := loader.InstallModules("testdata/config-fixture", false, nil)
if configDiags.HasErrors() {
t.Fatalf("unexpected problems initializing test config: %s", configDiags.Error())
}
config, configDiags := loader.LoadConfig("testdata/config-fixture")
if configDiags.HasErrors() {
t.Fatalf("unexpected problems loading config: %s", configDiags.Error())
t.Fatalf("unexpected problems loading config: %s", configDiags.Err())
}
// Build the TF context

17
terraform/terraform_test.go
View File

@ -2,6 +2,8 @@ package terraform
import (
"flag"
"github.com/hashicorp/terraform/internal/initwd"
"github.com/hashicorp/terraform/registry"
"io"
"io/ioutil"
"log"
@ -102,7 +104,6 @@ func testModuleWithSnapshot(t *testing.T, name string) (*configs.Config, *config
t.Helper()
dir := filepath.Join(fixtureDir, name)
// FIXME: We're not dealing with the cleanup function here because
// this testModule function is used all over and so we don't want to
// change its interface at this late stage.
@ -111,9 +112,10 @@ func testModuleWithSnapshot(t *testing.T, name string) (*configs.Config, *config
// Test modules usually do not refer to remote sources, and for local
// sources only this ultimately just records all of the module paths
// in a JSON file so that we can load them below.
diags := loader.InstallModules(dir, true, configload.InstallHooksImpl{})
if diags.HasErrors() {
t.Fatal(diags.Error())
inst := initwd.NewModuleInstaller(loader.ModulesDir(), registry.NewClient(nil, nil))
instDiags := inst.InstallModules(dir, true, initwd.ModuleInstallHooksImpl{})
if instDiags.HasErrors() {
t.Fatal(instDiags.Err())
}
config, snap, diags := loader.LoadConfigWithSnapshot(dir)
@ -162,9 +164,10 @@ func testModuleInline(t *testing.T, sources map[string]string) *configs.Config {
// Test modules usually do not refer to remote sources, and for local
// sources only this ultimately just records all of the module paths
// in a JSON file so that we can load them below.
diags := loader.InstallModules(cfgPath, true, configload.InstallHooksImpl{})
if diags.HasErrors() {
t.Fatal(diags.Error())
inst := initwd.NewModuleInstaller(loader.ModulesDir(), registry.NewClient(nil, nil))
instDiags := inst.InstallModules(cfgPath, true, initwd.ModuleInstallHooksImpl{})
if instDiags.HasErrors() {
t.Fatal(instDiags.Err())
}
config, diags := loader.LoadConfig(cfgPath)