resilien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

command/init: Read, respect, and update provider dependency locks 

This changes the approach used by the provider installer to remember
between runs which selections it has previously made, using the lock file
format implemented in internal/depsfile.

This means that version constraints in the configuration are considered
only for providers we've not seen before or when -upgrade mode is active.
This commit is contained in:
Martin Atkins 2020-10-02 16:41:56 -07:00
parent 4a1b081afb
commit b3f5c7f1e6
20 changed files with 800 additions and 519 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
command/e2etest/init_test.go
View File

@ -44,8 +44,8 @@ func TestInitProviders(t *testing.T) {
t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)")
}
if !strings.Contains(stdout, "* hashicorp/template: version = ") {
t.Errorf("provider pinning recommendation is missing from output:\n%s", stdout)
if !strings.Contains(stdout, "Terraform has created a lock file") {
t.Errorf("lock file notification is missing from output:\n%s", stdout)
}
}

65
command/init.go
View File

@ -5,7 +5,6 @@ import (
"fmt"
"log"
"os"
"sort"
"strings"
"github.com/hashicorp/hcl/v2"
@ -422,9 +421,9 @@ the backend configuration is present and valid.
func (c *InitCommand) getProviders(config *configs.Config, state *states.State, upgrade bool, pluginDirs []string) (output, abort bool, diags tfdiags.Diagnostics) {
// First we'll collect all the provider dependencies we can see in the
// configuration and the state.
reqs, moreDiags := config.ProviderRequirements()
diags = diags.Append(moreDiags)
if moreDiags.HasErrors() {
reqs, hclDiags := config.ProviderRequirements()
diags = diags.Append(hclDiags)
if hclDiags.HasErrors() {
return false, true, diags
}
if state != nil {
@ -444,6 +443,10 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
))
}
}
previousLocks, moreDiags := c.lockedDependencies()
diags = diags.Append(moreDiags)
if diags.HasErrors() {
return false, true, diags
}
@ -729,7 +732,7 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
ctx, done := c.InterruptibleContext()
defer done()
ctx = evts.OnContext(ctx)
selected, err := inst.EnsureProviderVersions(ctx, reqs, mode)
newLocks, err := inst.EnsureProviderVersions(ctx, previousLocks, reqs, mode)
if ctx.Err() == context.Canceled {
c.showDiagnostics(diags)
c.Ui.Error("Provider installation was canceled by an interrupt signal.")
@ -746,29 +749,41 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
return true, true, diags
}
// If any providers have "floating" versions (completely unconstrained)
// we'll suggest the user constrain with a pessimistic constraint to
// avoid implicitly adopting a later major release.
constraintSuggestions := make(map[string]string)
for addr, version := range selected {
req := reqs[addr]
if len(req) == 0 {
constraintSuggestions[addr.ForDisplay()] = "~> " + version.String()
// If the provider dependencies have changed since the last run then we'll
// say a little about that in case the reader wasn't expecting a change.
// (When we later integrate module dependencies into the lock file we'll
// probably want to refactor this so that we produce one lock-file related
// message for all changes together, but this is here for now just because
// it's the smallest change relative to what came before it, which was
// a hidden JSON file specifically for tracking providers.)
if !newLocks.Equal(previousLocks) {
if previousLocks.Empty() {
// A change from empty to non-empty is special because it suggests
// we're running "terraform init" for the first time against a
// new configuration. In that case we'll take the opportunity to
// say a little about what the dependency lock file is, for new
// users or those who are upgrading from a previous Terraform
// version that didn't have dependency lock files.
c.Ui.Output(c.Colorize().Color(`
Terraform has created a lock file [bold].terraform.lock.hcl[reset] to record the provider
selections it made above. Include this file in your version control repository
so that Terraform can guarantee to make the same selections by default when
you run "terraform init" in the future.`))
} else {
c.Ui.Output(c.Colorize().Color(`
Terraform has made some changes to the provider dependency selections recorded
in the .terraform.lock.hcl file. Review those changes and commit them to your
version control system if they represent changes you intended to make.`))
}
}
if len(constraintSuggestions) != 0 {
names := make([]string, 0, len(constraintSuggestions))
for name := range constraintSuggestions {
names = append(names, name)
}
sort.Strings(names)
c.Ui.Output(outputInitProvidersUnconstrained)
for _, name := range names {
c.Ui.Output(fmt.Sprintf("* %s: version = %q", name, constraintSuggestions[name]))
}
}
// TODO: Check whether newLocks is different from previousLocks and mention
// in the UI if so. We should emit a different message if previousLocks was
// empty, because that indicates we were creating a lock file for the first
// time and so we need to introduce the user to the idea of it.
moreDiags = c.replaceLockedDependencies(newLocks)
diags = diags.Append(moreDiags)
return true, false, diags
}

181
command/init_test.go
View File

@ -1,6 +1,7 @@
package command
import (
"bytes"
"context"
"encoding/json"
"fmt"
@ -20,6 +21,7 @@ import (
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/configs/configschema"
"github.com/hashicorp/terraform/internal/depsfile"
"github.com/hashicorp/terraform/internal/getproviders"
"github.com/hashicorp/terraform/internal/providercache"
"github.com/hashicorp/terraform/states"
@ -1126,13 +1128,13 @@ func TestInit_getProviderInvalidPackage(t *testing.T) {
}
wantErrors := []string{
"Failed to validate installed provider",
"Failed to install provider",
"could not find executable file starting with terraform-provider-package",
}
got := ui.ErrorWriter.String()
for _, wantError := range wantErrors {
if !strings.Contains(got, wantError) {
t.Fatalf("missing error:\nwant: %q\n got: %q", wantError, got)
t.Fatalf("missing error:\nwant: %q\ngot:\n%s", wantError, got)
}
}
}
@ -1267,29 +1269,38 @@ func TestInit_providerSource(t *testing.T) {
t.Errorf("wrong cache directory contents after upgrade\n%s", diff)
}
inst := m.providerInstaller()
gotSelected, err := inst.SelectedPackages()
locks, err := m.lockedDependencies()
if err != nil {
t.Fatalf("failed to get selected packages from installer: %s", err)
t.Fatalf("failed to get locked dependencies: %s", err)
}
wantSelected := map[addrs.Provider]*providercache.CachedProvider{
addrs.NewDefaultProvider("test-beta"): {
Provider: addrs.NewDefaultProvider("test-beta"),
Version: getproviders.MustParseVersion("1.2.4"),
PackageDir: expectedPackageInstallPath("test-beta", "1.2.4", false),
},
addrs.NewDefaultProvider("test"): {
Provider: addrs.NewDefaultProvider("test"),
Version: getproviders.MustParseVersion("1.2.3"),
PackageDir: expectedPackageInstallPath("test", "1.2.3", false),
},
addrs.NewDefaultProvider("source"): {
Provider: addrs.NewDefaultProvider("source"),
Version: getproviders.MustParseVersion("1.2.3"),
PackageDir: expectedPackageInstallPath("source", "1.2.3", false),
},
gotProviderLocks := locks.AllProviders()
wantProviderLocks := map[addrs.Provider]*depsfile.ProviderLock{
addrs.NewDefaultProvider("test-beta"): depsfile.NewProviderLock(
addrs.NewDefaultProvider("test-beta"),
getproviders.MustParseVersion("1.2.4"),
getproviders.MustParseVersionConstraints("= 1.2.4"),
[]getproviders.Hash{
getproviders.HashScheme1.New("see6W06w09Ea+AobFJ+mbvPTie6ASqZAAdlFZbs8BSM="),
},
),
addrs.NewDefaultProvider("test"): depsfile.NewProviderLock(
addrs.NewDefaultProvider("test"),
getproviders.MustParseVersion("1.2.3"),
getproviders.MustParseVersionConstraints("= 1.2.3"),
[]getproviders.Hash{
getproviders.HashScheme1.New("wlbEC2mChQZ2hhgUhl6SeVLPP7fMqOFUZAQhQ9GIIno="),
},
),
addrs.NewDefaultProvider("source"): depsfile.NewProviderLock(
addrs.NewDefaultProvider("source"),
getproviders.MustParseVersion("1.2.3"),
getproviders.MustParseVersionConstraints("= 1.2.3"),
[]getproviders.Hash{
getproviders.HashScheme1.New("myS3qb3px3tRBq1ZWRYJeUH+kySWpBc0Yy8rw6W7/p4="),
},
),
}
if diff := cmp.Diff(wantSelected, gotSelected); diff != "" {
if diff := cmp.Diff(gotProviderLocks, wantProviderLocks, depsfile.ProviderLockComparer); diff != "" {
t.Errorf("wrong version selections after upgrade\n%s", diff)
}
@ -1435,32 +1446,40 @@ func TestInit_getUpgradePlugins(t *testing.T) {
t.Errorf("wrong cache directory contents after upgrade\n%s", diff)
}
inst := m.providerInstaller()
gotSelected, err := inst.SelectedPackages()
locks, err := m.lockedDependencies()
if err != nil {
t.Fatalf("failed to get selected packages from installer: %s", err)
t.Fatalf("failed to get locked dependencies: %s", err)
}
wantSelected := map[addrs.Provider]*providercache.CachedProvider{
addrs.NewDefaultProvider("between"): {
Provider: addrs.NewDefaultProvider("between"),
Version: getproviders.MustParseVersion("2.3.4"),
PackageDir: expectedPackageInstallPath("between", "2.3.4", false),
},
addrs.NewDefaultProvider("exact"): {
Provider: addrs.NewDefaultProvider("exact"),
Version: getproviders.MustParseVersion("1.2.3"),
PackageDir: expectedPackageInstallPath("exact", "1.2.3", false),
},
addrs.NewDefaultProvider("greater-than"): {
Provider: addrs.NewDefaultProvider("greater-than"),
Version: getproviders.MustParseVersion("2.3.4"),
PackageDir: expectedPackageInstallPath("greater-than", "2.3.4", false),
},
gotProviderLocks := locks.AllProviders()
wantProviderLocks := map[addrs.Provider]*depsfile.ProviderLock{
addrs.NewDefaultProvider("between"): depsfile.NewProviderLock(
addrs.NewDefaultProvider("between"),
getproviders.MustParseVersion("2.3.4"),
getproviders.MustParseVersionConstraints("> 1.0.0, < 3.0.0"),
[]getproviders.Hash{
getproviders.HashScheme1.New("JVqAvZz88A+hS2wHVtTWQkHaxoA/LrUAz0H3jPBWPIA="),
},
),
addrs.NewDefaultProvider("exact"): depsfile.NewProviderLock(
addrs.NewDefaultProvider("exact"),
getproviders.MustParseVersion("1.2.3"),
getproviders.MustParseVersionConstraints("= 1.2.3"),
[]getproviders.Hash{
getproviders.HashScheme1.New("H1TxWF8LyhBb6B4iUdKhLc/S9sC/jdcrCykpkbGcfbg="),
},
),
addrs.NewDefaultProvider("greater-than"): depsfile.NewProviderLock(
addrs.NewDefaultProvider("greater-than"),
getproviders.MustParseVersion("2.3.4"),
getproviders.MustParseVersionConstraints(">= 2.3.3"),
[]getproviders.Hash{
getproviders.HashScheme1.New("SJPpXx/yoFE/W+7eCipjJ+G21xbdnTBD7lWodZ8hWkU="),
},
),
}
if diff := cmp.Diff(wantSelected, gotSelected); diff != "" {
if diff := cmp.Diff(gotProviderLocks, wantProviderLocks, depsfile.ProviderLockComparer); diff != "" {
t.Errorf("wrong version selections after upgrade\n%s", diff)
}
}
func TestInit_getProviderMissing(t *testing.T) {
@ -1537,7 +1556,7 @@ func TestInit_providerLockFile(t *testing.T) {
defer testChdir(t, td)()
providerSource, close := newMockProviderSource(t, map[string][]string{
"test": []string{"1.2.3"},
"test": {"1.2.3"},
})
defer close()
@ -1557,23 +1576,26 @@ func TestInit_providerLockFile(t *testing.T) {
t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
}
selectionsFile := ".terraform/plugins/selections.json"
buf, err := ioutil.ReadFile(selectionsFile)
lockFile := ".terraform.lock.hcl"
buf, err := ioutil.ReadFile(lockFile)
if err != nil {
t.Fatalf("failed to read provider selections file %s: %s", selectionsFile, err)
t.Fatalf("failed to read dependency lock file %s: %s", lockFile, err)
}
buf = bytes.TrimSpace(buf)
// The hash in here is for the fake package that newMockProviderSource produces
// (so it'll change if newMockProviderSource starts producing different contents)
wantLockFile := strings.TrimSpace(`
{
"registry.terraform.io/hashicorp/test": {
"hash": "h1:wlbEC2mChQZ2hhgUhl6SeVLPP7fMqOFUZAQhQ9GIIno=",
"version": "1.2.3"
}
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/test" {
version = "1.2.3"
constraints = "1.2.3"
hashes = ["h1:wlbEC2mChQZ2hhgUhl6SeVLPP7fMqOFUZAQhQ9GIIno="]
}
`)
if string(buf) != wantLockFile {
t.Errorf("wrong provider selections file contents\ngot: %s\nwant: %s", buf, wantLockFile)
if diff := cmp.Diff(wantLockFile, string(buf)); diff != "" {
t.Errorf("wrong dependency lock file contents\n%s", diff)
}
}
@ -1697,29 +1719,38 @@ func TestInit_pluginDirProviders(t *testing.T) {
t.Fatalf("bad: \n%s", ui.ErrorWriter)
}
inst := m.providerInstaller()
gotSelected, err := inst.SelectedPackages()
locks, err := m.lockedDependencies()
if err != nil {
t.Fatalf("failed to get selected packages from installer: %s", err)
t.Fatalf("failed to get locked dependencies: %s", err)
}
wantSelected := map[addrs.Provider]*providercache.CachedProvider{
addrs.NewDefaultProvider("between"): {
Provider: addrs.NewDefaultProvider("between"),
Version: getproviders.MustParseVersion("2.3.4"),
PackageDir: expectedPackageInstallPath("between", "2.3.4", false),
},
addrs.NewDefaultProvider("exact"): {
Provider: addrs.NewDefaultProvider("exact"),
Version: getproviders.MustParseVersion("1.2.3"),
PackageDir: expectedPackageInstallPath("exact", "1.2.3", false),
},
addrs.NewDefaultProvider("greater-than"): {
Provider: addrs.NewDefaultProvider("greater-than"),
Version: getproviders.MustParseVersion("2.3.4"),
PackageDir: expectedPackageInstallPath("greater-than", "2.3.4", false),
},
gotProviderLocks := locks.AllProviders()
wantProviderLocks := map[addrs.Provider]*depsfile.ProviderLock{
addrs.NewDefaultProvider("between"): depsfile.NewProviderLock(
addrs.NewDefaultProvider("between"),
getproviders.MustParseVersion("2.3.4"),
getproviders.MustParseVersionConstraints("> 1.0.0, < 3.0.0"),
[]getproviders.Hash{
getproviders.HashScheme1.New("JVqAvZz88A+hS2wHVtTWQkHaxoA/LrUAz0H3jPBWPIA="),
},
),
addrs.NewDefaultProvider("exact"): depsfile.NewProviderLock(
addrs.NewDefaultProvider("exact"),
getproviders.MustParseVersion("1.2.3"),
getproviders.MustParseVersionConstraints("= 1.2.3"),
[]getproviders.Hash{
getproviders.HashScheme1.New("H1TxWF8LyhBb6B4iUdKhLc/S9sC/jdcrCykpkbGcfbg="),
},
),
addrs.NewDefaultProvider("greater-than"): depsfile.NewProviderLock(
addrs.NewDefaultProvider("greater-than"),
getproviders.MustParseVersion("2.3.4"),
getproviders.MustParseVersionConstraints(">= 2.3.3"),
[]getproviders.Hash{
getproviders.HashScheme1.New("SJPpXx/yoFE/W+7eCipjJ+G21xbdnTBD7lWodZ8hWkU="),
},
),
}
if diff := cmp.Diff(wantSelected, gotSelected); diff != "" {
if diff := cmp.Diff(gotProviderLocks, wantProviderLocks, depsfile.ProviderLockComparer); diff != "" {
t.Errorf("wrong version selections after upgrade\n%s", diff)
}
@ -1983,7 +2014,7 @@ func installFakeProviderPackagesElsewhere(t *testing.T, cacheDir *providercache.
if err != nil {
t.Fatalf("failed to prepare fake package for %s %s: %s", name, versionStr, err)
}
_, err = cacheDir.InstallPackage(context.Background(), meta)
_, err = cacheDir.InstallPackage(context.Background(), meta, nil)
if err != nil {
t.Fatalf("failed to install fake package for %s %s: %s", name, versionStr, err)
}

7
command/meta.go
View File

@ -410,7 +410,12 @@ func (m *Meta) contextOpts() (*terraform.ContextOpts, error) {
// This situation shouldn't arise commonly in practice because
// the selections file is generated programmatically.
log.Printf("[WARN] Failed to determine selected providers: %s", err)
providerFactories = nil
// variable providerFactories may now be incomplete, which could
// lead to errors reported downstream from here. providerFactories
// tries to populate as many providers as possible even in an
// error case, so that operations not using problematic providers
// can still succeed.
}
opts.Providers = providerFactories
opts.Provisioners = m.provisionerFactories()

62
command/meta_dependencies.go Normal file
View File

@ -0,0 +1,62 @@
package command
import (
"os"
"github.com/hashicorp/terraform/internal/depsfile"
"github.com/hashicorp/terraform/tfdiags"
)
// dependenclyLockFilename is the filename of the dependency lock file.
//
// This file should live in the same directory as the .tf files for the
// root module of the configuration, alongside the .terraform directory
// as long as that directory's path isn't overridden by the TF_DATA_DIR
// environment variable.
//
// We always expect to find this file in the current working directory
// because that should also be the root module directory.
//
// Some commands have legacy command line arguments that make the root module
// directory something other than the root module directory; when using those,
// the lock file will be written in the "wrong" place (the current working
// directory instead of the root module directory) but we do that intentionally
// to match where the ".terraform" directory would also be written in that
// case. Eventually we will phase out those legacy arguments in favor of the
// global -chdir=... option, which _does_ preserve the intended invariant
// that the root module directory is always the current working directory.
const dependencyLockFilename = ".terraform.lock.hcl"
// lockedDependencies reads the dependency lock information from the lock file
// in the current working directory.
//
// If the lock file doesn't exist at the time of the call, lockedDependencies
// indicates success and returns an empty Locks object. If the file does
// exist then the result is either a representation of the contents of that
// file at the instant of the call or error diagnostics explaining some way
// in which the lock file is invalid.
//
// The result is a snapshot of the locked dependencies at the time of the call
// and does not update as a result of calling replaceLockedDependencies
// or any other modification method.
func (m *Meta) lockedDependencies() (*depsfile.Locks, tfdiags.Diagnostics) {
// We check that the file exists first, because the underlying HCL
// parser doesn't distinguish that error from other error types
// in a machine-readable way but we want to treat that as a success
// with no locks. There is in theory a race condition here in that
// the file could be created or removed in the meantime, but we're not
// promising to support two concurrent dependency installation processes.
_, err := os.Stat(dependencyLockFilename)
if os.IsNotExist(err) {
return depsfile.NewLocks(), nil
}
return depsfile.LoadLocksFromFile(dependencyLockFilename)
}
// replaceLockedDependencies creates or overwrites the lock file in the
// current working directory to contain the information recorded in the given
// locks object.
func (m *Meta) replaceLockedDependencies(new *depsfile.Locks) tfdiags.Diagnostics {
return depsfile.SaveLocksToFile(new, dependencyLockFilename)
}

93
command/meta_providers.go
View File

@ -7,6 +7,7 @@ import (
"path/filepath"
hclog "github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-multierror"
plugin "github.com/hashicorp/go-plugin"
"github.com/hashicorp/terraform/addrs"
@ -160,34 +161,86 @@ func (m *Meta) providerInstallSource() getproviders.Source {
// be honored with what is currently in the cache, such as if a selected
// package has been removed from the cache or if the contents of a selected
// package have been modified outside of the installer. If it returns an error,
// the returned map may be incomplete or invalid.
// the returned map may be incomplete or invalid, but will be as complete
// as possible given the cause of the error.
func (m *Meta) providerFactories() (map[addrs.Provider]providers.Factory, error) {
// We don't have to worry about potentially calling
// providerInstallerCustomSource here because we're only using this
// installer for its SelectedPackages method, which does not consult
// any provider sources.
inst := m.providerInstaller()
selected, err := inst.SelectedPackages()
if err != nil {
return nil, fmt.Errorf("failed to recall provider packages selected by earlier 'terraform init': %s", err)
locks, diags := m.lockedDependencies()
if diags.HasErrors() {
return nil, fmt.Errorf("failed to read dependency lock file: %s", diags.Err())
}
// We'll always run through all of our providers, even if one of them
// encounters an error, so that we can potentially report multiple errors
// where appropriate and so that callers can potentially make use of the
// partial result we return if e.g. they want to enumerate which providers
// are available, or call into one of the providers that didn't fail.
var err error
// For the providers from the lock file, we expect them to be already
// available in the provider cache because "terraform init" should already
// have put them there.
providerLocks := locks.AllProviders()
cacheDir := m.providerLocalCacheDir()
// The internal providers are _always_ available, even if the configuration
// doesn't request them, because they don't need any special installation
// and they'll just be ignored if not used.
internalFactories := m.internalProviders()
factories := make(map[addrs.Provider]providers.Factory, len(selected)+len(internalFactories)+len(m.UnmanagedProviders))
// The Terraform SDK test harness (and possibly other callers in future)
// can ask that we use its own already-started provider servers, which we
// call "unmanaged" because Terraform isn't responsible for starting
// and stopping them.
unmanagedProviders := m.UnmanagedProviders
factories := make(map[addrs.Provider]providers.Factory, len(providerLocks)+len(internalFactories)+len(unmanagedProviders))
for name, factory := range internalFactories {
factories[addrs.NewBuiltInProvider(name)] = factory
}
for provider, reattach := range m.UnmanagedProviders {
factories[provider] = unmanagedProviderFactory(provider, reattach)
}
for provider, cached := range selected {
for provider, lock := range providerLocks {
reportError := func(thisErr error) {
err = multierror.Append(err, thisErr)
// We'll populate a provider factory that just echoes our error
// again if called, which allows us to still report a helpful
// error even if it gets detected downstream somewhere from the
// caller using our partial result.
factories[provider] = providerFactoryError(thisErr)
}
version := lock.Version()
cached := cacheDir.ProviderVersion(provider, version)
if cached == nil {
reportError(fmt.Errorf(
"there is no package for %s %s cached in %s",
provider, version, cacheDir.BasePath(),
))
continue
}
// The cached package must match one of the checksums recorded in
// the lock file, if any.
if allowedHashes := lock.PreferredHashes(); len(allowedHashes) != 0 {
matched, err := cached.MatchesAnyHash(allowedHashes)
if err != nil {
reportError(fmt.Errorf(
"failed to verify checksum of %s %s package cached in in %s: %s",
provider, version, cacheDir.BasePath(), err,
))
continue
}
if !matched {
reportError(fmt.Errorf(
"the cached package for %s %s (in %s) does not match any of the checksums recorded in the dependency lock file",
provider, version, cacheDir.BasePath(),
))
continue
}
}
factories[provider] = providerFactory(cached)
}
return factories, nil
for provider, reattach := range unmanagedProviders {
factories[provider] = unmanagedProviderFactory(provider, reattach)
}
return factories, err
}
func (m *Meta) internalProviders() map[string]providers.Factory {
@ -286,3 +339,13 @@ func unmanagedProviderFactory(provider addrs.Provider, reattach *plugin.Reattach
return p, nil
}
}
// providerFactoryError is a stub providers.Factory that returns an error
// when called. It's used to allow providerFactories to still produce a
// factory for each available provider in an error case, for situations
// where the caller can do something useful with that partial result.
func providerFactoryError(err error) providers.Factory {
return func() (providers.Interface, error) {
return nil, err
}
}

46
command/version.go
View File

@ -6,6 +6,9 @@ import (
"fmt"
"sort"
"strings"
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/internal/depsfile"
)
// VersionCommand is a Command implementation prints the version.
@ -80,29 +83,28 @@ func (c *VersionCommand) Run(args []string) int {
}
}
// We'll also attempt to print out the selected plugin versions. We can
// do this only if "terraform init" was already run and thus we've committed
// to a specific set of plugins. If not, the plugins lock will be empty
// and so we'll show _no_ providers.
// We'll also attempt to print out the selected plugin versions. We do
// this based on the dependency lock file, and so the result might be
// empty or incomplete if the user hasn't successfully run "terraform init"
// since the most recent change to dependencies.
//
// Generally-speaking this is a best-effort thing that will give us a good
// result in the usual case where the user successfully ran "terraform init"
// and then hit a problem running _another_ command.
providerInstaller := c.providerInstaller()
providerSelections, err := providerInstaller.SelectedPackages()
var pluginVersions []string
if err != nil {
// we'll just ignore it and show no plugins at all, then.
providerSelections = nil
}
for providerAddr, cached := range providerSelections {
version := cached.Version.String()
if version == "0.0.0" {
pluginVersions = append(pluginVersions, fmt.Sprintf("+ provider %s (unversioned)", providerAddr))
} else {
pluginVersions = append(pluginVersions, fmt.Sprintf("+ provider %s v%s", providerAddr, version))
var providerVersions []string
var providerLocks map[addrs.Provider]*depsfile.ProviderLock
if locks, err := c.lockedDependencies(); err == nil {
providerLocks = locks.AllProviders()
for providerAddr, lock := range providerLocks {
version := lock.Version().String()
if version == "0.0.0" {
providerVersions = append(providerVersions, fmt.Sprintf("+ provider %s (unversioned)", providerAddr))
} else {
providerVersions = append(providerVersions, fmt.Sprintf("+ provider %s v%s", providerAddr, version))
}
}
}
// If we have a version check function, then let's check for
// the latest version as well.
if c.CheckFunc != nil {
@ -120,8 +122,8 @@ func (c *VersionCommand) Run(args []string) int {
if jsonOutput {
selectionsOutput := make(map[string]string)
for providerAddr, cached := range providerSelections {
version := cached.Version.String()
for providerAddr, lock := range providerLocks {
version := lock.Version().String()
selectionsOutput[providerAddr.String()] = version
}
@ -148,9 +150,9 @@ func (c *VersionCommand) Run(args []string) int {
return 0
} else {
c.Ui.Output(versionString.String())
if len(pluginVersions) != 0 {
sort.Strings(pluginVersions)
for _, str := range pluginVersions {
if len(providerVersions) != 0 {
sort.Strings(providerVersions)
for _, str := range providerVersions {
c.Ui.Output(str)
}
}

149
command/version_test.go
View File

@ -1,10 +1,15 @@
package command
import (
"io/ioutil"
"os"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/internal/depsfile"
"github.com/hashicorp/terraform/internal/getproviders"
"github.com/mitchellh/cli"
)
@ -13,49 +18,49 @@ func TestVersionCommand_implements(t *testing.T) {
}
func TestVersion(t *testing.T) {
fixtureDir := "testdata/providers-schema/basic"
td := tempDir(t)
testCopyDir(t, fixtureDir, td)
td, err := ioutil.TempDir("", "terraform-test-version")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(td)
defer testChdir(t, td)()
ui := new(cli.MockUi)
// We'll create a fixed dependency lock file in our working directory
// so we can verify that the version command shows the information
// from it.
locks := depsfile.NewLocks()
locks.SetProvider(
addrs.NewDefaultProvider("test2"),
getproviders.MustParseVersion("1.2.3"),
nil,
nil,
)
locks.SetProvider(
addrs.NewDefaultProvider("test1"),
getproviders.MustParseVersion("7.8.9-beta.2"),
nil,
nil,
)
providerSource, close := newMockProviderSource(t, map[string][]string{
"test": []string{"1.2.3"},
})
defer close()
m := Meta{
testingOverrides: metaOverridesForProvider(testProvider()),
Ui: ui,
ProviderSource: providerSource,
}
// `terraform init`
ic := &InitCommand{
Meta: m,
}
if code := ic.Run([]string{}); code != 0 {
t.Fatalf("init failed\n%s", ui.ErrorWriter)
}
// flush the init output from the mock ui
ui.OutputWriter.Reset()
// `terraform version`
ui := cli.NewMockUi()
c := &VersionCommand{
Meta: m,
Meta: Meta{
Ui: ui,
},
Version: "4.5.6",
VersionPrerelease: "foo",
}
if err := c.replaceLockedDependencies(locks); err != nil {
t.Fatal(err)
}
if code := c.Run([]string{}); code != 0 {
t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
}
actual := strings.TrimSpace(ui.OutputWriter.String())
expected := "Terraform v4.5.6-foo\n+ provider registry.terraform.io/hashicorp/test v1.2.3"
expected := "Terraform v4.5.6-foo\n+ provider registry.terraform.io/hashicorp/test1 v7.8.9-beta.2\n+ provider registry.terraform.io/hashicorp/test2 v1.2.3"
if actual != expected {
t.Fatalf("wrong output\ngot: %#v\nwant: %#v", actual, expected)
t.Fatalf("wrong output\ngot:\n%s\nwant:\n%s", actual, expected)
}
}
@ -108,38 +113,21 @@ func TestVersion_outdated(t *testing.T) {
}
func TestVersion_json(t *testing.T) {
fixtureDir := "testdata/providers-schema/basic"
td := tempDir(t)
testCopyDir(t, fixtureDir, td)
td, err := ioutil.TempDir("", "terraform-test-version")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(td)
defer testChdir(t, td)()
ui := new(cli.MockUi)
providerSource, close := newMockProviderSource(t, map[string][]string{
"test": []string{"1.2.3"},
})
defer close()
m := Meta{
testingOverrides: metaOverridesForProvider(testProvider()),
Ui: ui,
ProviderSource: providerSource,
ui := cli.NewMockUi()
meta := Meta{
Ui: ui,
}
// `terraform init`
ic := &InitCommand{
Meta: m,
}
if code := ic.Run([]string{}); code != 0 {
t.Fatalf("init failed\n%s", ui.ErrorWriter)
}
// flush the init output from the mock ui
ui.OutputWriter.Reset()
// `terraform version -json` without prerelease
c := &VersionCommand{
Meta: m,
Meta: meta,
Version: "4.5.6",
}
if code := c.Run([]string{"-json"}); code != 0 {
@ -147,28 +135,65 @@ func TestVersion_json(t *testing.T) {
}
actual := strings.TrimSpace(ui.OutputWriter.String())
expected := "{\n \"terraform_version\": \"4.5.6\",\n \"terraform_revision\": \"\",\n \"provider_selections\": {\n \"registry.terraform.io/hashicorp/test\": \"1.2.3\"\n },\n \"terraform_outdated\": false\n}"
if actual != expected {
t.Fatalf("wrong output\ngot: %#v\nwant: %#v", actual, expected)
expected := strings.TrimSpace(`
{
"terraform_version": "4.5.6",
"terraform_revision": "",
"provider_selections": {},
"terraform_outdated": false
}
`)
if diff := cmp.Diff(expected, actual); diff != "" {
t.Fatalf("wrong output\n%s", diff)
}
// flush the output from the mock ui
ui.OutputWriter.Reset()
// `terraform version -json` with prerelease
// Now we'll create a fixed dependency lock file in our working directory
// so we can verify that the version command shows the information
// from it.
locks := depsfile.NewLocks()
locks.SetProvider(
addrs.NewDefaultProvider("test2"),
getproviders.MustParseVersion("1.2.3"),
nil,
nil,
)
locks.SetProvider(
addrs.NewDefaultProvider("test1"),
getproviders.MustParseVersion("7.8.9-beta.2"),
nil,
nil,
)
// `terraform version -json` with prerelease and provider dependencies
c = &VersionCommand{
Meta: m,
Meta: meta,
Version: "4.5.6",
VersionPrerelease: "foo",
}
if err := c.replaceLockedDependencies(locks); err != nil {
t.Fatal(err)
}
if code := c.Run([]string{"-json"}); code != 0 {
t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
}
actual = strings.TrimSpace(ui.OutputWriter.String())
expected = "{\n \"terraform_version\": \"4.5.6-foo\",\n \"terraform_revision\": \"\",\n \"provider_selections\": {\n \"registry.terraform.io/hashicorp/test\": \"1.2.3\"\n },\n \"terraform_outdated\": false\n}"
if actual != expected {
t.Fatalf("wrong output\ngot: %#v\nwant: %#v", actual, expected)
expected = strings.TrimSpace(`
{
"terraform_version": "4.5.6-foo",
"terraform_revision": "",
"provider_selections": {
"registry.terraform.io/hashicorp/test1": "7.8.9-beta.2",
"registry.terraform.io/hashicorp/test2": "1.2.3"
},
"terraform_outdated": false
}
`)
if diff := cmp.Diff(expected, actual); diff != "" {
t.Fatalf("wrong output\n%s", diff)
}
}

88
internal/depsfile/locks.go
View File

@ -2,6 +2,7 @@ package depsfile
import (
"fmt"
"sort"
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/internal/getproviders"
@ -46,6 +47,18 @@ func (l *Locks) Provider(addr addrs.Provider) *ProviderLock {
return l.providers[addr]
}
// AllProviders returns a map describing all of the provider locks in the
// receiver.
func (l *Locks) AllProviders() map[addrs.Provider]*ProviderLock {
// We return a copy of our internal map so that future calls to
// SetProvider won't modify the map we're returning, or vice-versa.
ret := make(map[addrs.Provider]*ProviderLock, len(l.providers))
for k, v := range l.providers {
ret[k] = v
}
return ret
}
// SetProvider creates a new lock or replaces the existing lock for the given
// provider.
//
@ -53,6 +66,10 @@ func (l *Locks) Provider(addr addrs.Provider) *ProviderLock {
// invalidates any ProviderLock object previously returned from Provider or
// SetProvider for the given provider address.
//
// The ownership of the backing array for the slice of hashes passes to this
// function, and so the caller must not read or write that backing array after
// calling SetProvider.
//
// Only lockable providers can be passed to this method. If you pass a
// non-lockable provider address then this function will panic. Use
// function ProviderIsLockable to determine whether a particular provider
@ -62,14 +79,61 @@ func (l *Locks) SetProvider(addr addrs.Provider, version getproviders.Version, c
panic(fmt.Sprintf("Locks.SetProvider with non-lockable provider %s", addr))
}
new := &ProviderLock{
new := NewProviderLock(addr, version, constraints, hashes)
l.providers[new.addr] = new
return new
}
// NewProviderLock creates a new ProviderLock object that isn't associated
// with any Locks object.
//
// This is here primarily for testing. Most callers should use Locks.SetProvider
// to construct a new provider lock and insert it into a Locks object at the
// same time.
//
// The ownership of the backing array for the slice of hashes passes to this
// function, and so the caller must not read or write that backing array after
// calling NewProviderLock.
//
// Only lockable providers can be passed to this method. If you pass a
// non-lockable provider address then this function will panic. Use
// function ProviderIsLockable to determine whether a particular provider
// should participate in the version locking mechanism.
func NewProviderLock(addr addrs.Provider, version getproviders.Version, constraints getproviders.VersionConstraints, hashes []getproviders.Hash) *ProviderLock {
if !ProviderIsLockable(addr) {
panic(fmt.Sprintf("Locks.NewProviderLock with non-lockable provider %s", addr))
}
// Normalize the hashes into lexical order so that we can do straightforward
// equality tests between different locks for the same provider. The
// hashes are logically a set, so the given order is insignificant.
sort.Slice(hashes, func(i, j int) bool {
return string(hashes[i]) < string(hashes[j])
})
// This is a slightly-tricky in-place deduping to avoid unnecessarily
// allocating a new array in the common case where there are no duplicates:
// we iterate over "hashes" at the same time as appending to another slice
// with the same backing array, relying on the fact that deduping can only
// _skip_ elements from the input, and will never generate additional ones
// that would cause the writer to get ahead of the reader. This also
// assumes that we already sorted the items, which means that any duplicates
// will be consecutive in the sequence.
dedupeHashes := hashes[:0]
prevHash := getproviders.NilHash
for _, hash := range hashes {
if hash != prevHash {
dedupeHashes = append(dedupeHashes, hash)
prevHash = hash
}
}
return &ProviderLock{
addr: addr,
version: version,
versionConstraints: constraints,
hashes: hashes,
hashes: dedupeHashes,
}
l.providers[addr] = new
return new
}
// ProviderIsLockable returns true if the given provider is eligible for
@ -121,20 +185,14 @@ func (l *Locks) Equal(other *Locks) bool {
}
// Although "hashes" is declared as a slice, it's logically an
// unordered set and so we'll compare it as such.
// unordered set. However, we normalize the slice of hashes when
// recieving it in NewProviderLock, so we can just do a simple
// item-by-item equality test here.
if len(thisLock.hashes) != len(otherLock.hashes) {
return false
}
found := make(map[getproviders.Hash]int, len(thisLock.hashes))
for _, hash := range thisLock.hashes {
found[hash]++
}
for _, hash := range otherLock.hashes {
found[hash]++
}
for _, count := range found {
if count != 2 {
// It wasn't in both sets, then
for i := range thisLock.hashes {
if thisLock.hashes[i] != otherLock.hashes[i] {
return false
}
}

22
internal/depsfile/testing.go Normal file
View File

@ -0,0 +1,22 @@
package depsfile
import (
"github.com/google/go-cmp/cmp"
)
// ProviderLockComparer is an option for github.com/google/go-cmp/cmp that
// specifies how to compare values of type depsfile.ProviderLock.
//
// Use this, rather than crafting comparison options yourself, in case the
// comparison strategy needs to change in future due to implementation details
// of the ProviderLock type.
var ProviderLockComparer cmp.Option
func init() {
// For now, direct comparison of the unexported fields is good enough
// because we store everything in a normalized form. If that changes
// later then we might need to write a custom transformer to a hidden
// type with exported fields, so we can retain the ability for cmp to
// still report differences deeply.
ProviderLockComparer = cmp.AllowUnexported(ProviderLock{})
}

10
internal/getproviders/hash.go
View File

@ -425,6 +425,16 @@ func (m PackageMeta) MatchesHash(want Hash) (bool, error) {
return PackageMatchesHash(m.Location, want)
}
// MatchesAnyHash returns true if the package at the location associated with
// the receiver matches at least one of the given hashes, or false otherwise.
//
// If it cannot read from the given location, MatchesHash returns an error.
// Unlike the signular MatchesHash, MatchesAnyHash considers an unsupported
// hash format to be a successful non-match.
func (m PackageMeta) MatchesAnyHash(acceptable []Hash) (bool, error) {
return PackageMatchesAnyHash(m.Location, acceptable)
}
// HashV1 computes a hash of the contents of the package at the location
// associated with the receiver using hash algorithm 1.
//

12
internal/providercache/cached_provider.go
View File

@ -50,7 +50,7 @@ func (cp *CachedProvider) Hash() (getproviders.Hash, error) {
// MatchesHash returns true if the package on disk matches the given hash,
// or false otherwise. If it cannot traverse the package directory and read
// all of the files in it, or if the hash is in an unsupported format,
// CheckHash returns an error.
// MatchesHash returns an error.
//
// MatchesHash may accept hashes in a number of different formats. Over time
// the set of supported formats may grow and shrink.
@ -58,6 +58,16 @@ func (cp *CachedProvider) MatchesHash(want getproviders.Hash) (bool, error) {
return getproviders.PackageMatchesHash(cp.PackageLocation(), want)
}
// MatchesAnyHash returns true if the package on disk matches the given hash,
// or false otherwise. If it cannot traverse the package directory and read
// all of the files in it, MatchesAnyHash returns an error.
//
// Unlike the singular MatchesHash, MatchesAnyHash considers unsupported hash
// formats as successfully non-matching, rather than returning an error.
func (cp *CachedProvider) MatchesAnyHash(allowed []getproviders.Hash) (bool, error) {
return getproviders.PackageMatchesAnyHash(cp.PackageLocation(), allowed)
}
// HashV1 computes a hash of the contents of the package directory associated
// with the receiving cached provider using hash algorithm 1.
//

6
internal/providercache/dir.go
View File

@ -66,6 +66,12 @@ func NewDirWithPlatform(baseDir string, platform getproviders.Platform) *Dir {
}
}
// BasePath returns the filesystem path of the base directory of this
// cache directory.
func (d *Dir) BasePath() string {
return filepath.Clean(d.baseDir)
}
// AllAvailablePackages returns a description of all of the packages already
// present in the directory. The cache entries are grouped by the provider
// they relate to and then sorted by version precedence, with highest

38
internal/providercache/dir_modify.go
View File

@ -11,7 +11,12 @@ import (
// InstallPackage takes a metadata object describing a package available for
// installation, retrieves that package, and installs it into the receiving
// cache directory.
func (d *Dir) InstallPackage(ctx context.Context, meta getproviders.PackageMeta) (*getproviders.PackageAuthenticationResult, error) {
//
// If the allowedHashes set has non-zero length then at least one of the hashes
// in the set must match the package that "entry" refers to. If none of the
// hashes match then the returned error message assumes that the hashes came
// from a lock file.
func (d *Dir) InstallPackage(ctx context.Context, meta getproviders.PackageMeta, allowedHashes []getproviders.Hash) (*getproviders.PackageAuthenticationResult, error) {
if meta.TargetPlatform != d.targetPlatform {
return nil, fmt.Errorf("can't install %s package into cache directory expecting %s", meta.TargetPlatform, d.targetPlatform)
}
@ -26,11 +31,11 @@ func (d *Dir) InstallPackage(ctx context.Context, meta getproviders.PackageMeta)
log.Printf("[TRACE] providercache.Dir.InstallPackage: installing %s v%s from %s", meta.Provider, meta.Version, meta.Location)
switch meta.Location.(type) {
case getproviders.PackageHTTPURL:
return installFromHTTPURL(ctx, meta, newPath)
return installFromHTTPURL(ctx, meta, newPath, allowedHashes)
case getproviders.PackageLocalArchive:
return installFromLocalArchive(ctx, meta, newPath)
return installFromLocalArchive(ctx, meta, newPath, allowedHashes)
case getproviders.PackageLocalDir:
return installFromLocalDir(ctx, meta, newPath)
return installFromLocalDir(ctx, meta, newPath, allowedHashes)
default:
// Should not get here, because the above should be exhaustive for
// all implementations of getproviders.Location.
@ -48,7 +53,26 @@ func (d *Dir) InstallPackage(ctx context.Context, meta getproviders.PackageMeta)
// It's invalid to link a CachedProvider from a particular Dir into that same
// Dir, because that would otherwise potentially replace a real package
// directory with a circular link back to itself.
func (d *Dir) LinkFromOtherCache(entry *CachedProvider) error {
//
// If the allowedHashes set has non-zero length then at least one of the hashes
// in the set must match the package that "entry" refers to. If none of the
// hashes match then the returned error message assumes that the hashes came
// from a lock file.
func (d *Dir) LinkFromOtherCache(entry *CachedProvider, allowedHashes []getproviders.Hash) error {
if len(allowedHashes) > 0 {
if matches, err := entry.MatchesAnyHash(allowedHashes); err != nil {
return fmt.Errorf(
"failed to calculate checksum for cached copy of %s %s in %s: %s",
entry.Provider, entry.Version, d.baseDir, err,
)
} else if !matches {
return fmt.Errorf(
"the provider cache at %s has a copy of %s %s that doesn't match any of the checksums recorded in the dependency lock file",
d.baseDir, entry.Provider, entry.Version,
)
}
}
newPath := getproviders.UnpackedDirectoryPathForPackage(
d.baseDir, entry.Provider, entry.Version, d.targetPlatform,
)
@ -76,6 +100,8 @@ func (d *Dir) LinkFromOtherCache(entry *CachedProvider) error {
entry.Provider.Type, entry.Version, d.targetPlatform),
Location: getproviders.PackageLocalDir(currentPath),
}
_, err := installFromLocalDir(context.TODO(), meta, newPath)
// No further hash check here because we already checked the hash
// of the source directory above.
_, err := installFromLocalDir(context.TODO(), meta, newPath, nil)
return err
}

4
internal/providercache/dir_modify_test.go
View File

@ -46,7 +46,7 @@ func TestInstallPackage(t *testing.T) {
Location: getproviders.PackageLocalArchive("testdata/terraform-provider-null_2.1.0_linux_amd64.zip"),
}
result, err := tmpDir.InstallPackage(context.TODO(), meta)
result, err := tmpDir.InstallPackage(context.TODO(), meta, nil)
if err != nil {
t.Fatalf("InstallPackage failed: %s", err)
}
@ -129,7 +129,7 @@ func TestLinkFromOtherCache(t *testing.T) {
t.Fatalf("null provider has no latest version in source directory")
}
err = tmpDir.LinkFromOtherCache(cacheEntry)
err = tmpDir.LinkFromOtherCache(cacheEntry, nil)
if err != nil {
t.Fatalf("LinkFromOtherCache failed: %s", err)
}

317
internal/providercache/installer.go
View File

@ -3,7 +3,6 @@ package providercache
import (
"context"
"fmt"
"path/filepath"
"sort"
"strings"
@ -11,6 +10,7 @@ import (
"github.com/hashicorp/terraform/addrs"
copydir "github.com/hashicorp/terraform/internal/copy"
"github.com/hashicorp/terraform/internal/depsfile"
"github.com/hashicorp/terraform/internal/getproviders"
)
@ -129,24 +129,28 @@ func (i *Installer) SetUnmanagedProviderTypes(types map[addrs.Provider]struct{})
// failures then those notifications will be redundant with the ones included
// in the final returned error value so callers should show either one or the
// other, and not both.
func (i *Installer) EnsureProviderVersions(ctx context.Context, reqs getproviders.Requirements, mode InstallMode) (getproviders.Selections, error) {
func (i *Installer) EnsureProviderVersions(ctx context.Context, locks *depsfile.Locks, reqs getproviders.Requirements, mode InstallMode) (*depsfile.Locks, error) {
errs := map[addrs.Provider]error{}
evts := installerEventsForContext(ctx)
// We'll work with a copy of the given locks, so we can modify it and
// return the updated locks without affecting the caller's object.
// We'll add or replace locks in here during our work so that the final
// locks file reflects what the installer has selected.
locks = locks.DeepCopy()
if cb := evts.PendingProviders; cb != nil {
cb(reqs)
}
// Here we'll keep track of which exact version we've selected for each
// provider in the requirements.
selected := map[addrs.Provider]getproviders.Version{}
// Step 1: Which providers might we need to fetch a new version of?
// This produces the subset of requirements we need to ask the provider
// source about.
have := i.targetDir.AllAvailablePackages()
// source about. If we're in the normal (non-upgrade) mode then we'll
// just ask the source to confirm the continued existence of what
// was locked, or otherwise we'll find the newest version matching the
// configured version constraint.
mightNeed := map[addrs.Provider]getproviders.VersionSet{}
MightNeedProvider:
locked := map[addrs.Provider]bool{}
for provider, versionConstraints := range reqs {
if provider.IsBuiltIn() {
// Built in providers do not require installation but we'll still
@ -189,35 +193,35 @@ MightNeedProvider:
continue
}
acceptableVersions := versions.MeetingConstraints(versionConstraints)
if mode.forceQueryAllProviders() {
// If our mode calls for us to look for newer versions regardless
// of whether an existing version is acceptable, we "might need"
// _all_ of the requested providers.
mightNeed[provider] = acceptableVersions
continue
}
havePackages, ok := have[provider]
if !ok { // If we don't have any versions at all then we'll definitely need it
mightNeed[provider] = acceptableVersions
continue
}
// If we already have some versions installed and our mode didn't
// force us to check for new ones anyway then we'll check only if
// there isn't already at least one version in our cache that is
// in the set of acceptable versions.
for _, pkg := range havePackages {
if acceptableVersions.Has(pkg.Version) {
// We will take no further actions for this provider, because
// a version we have is already acceptable.
selected[provider] = pkg.Version
if cb := evts.ProviderAlreadyInstalled; cb != nil {
cb(provider, pkg.Version)
if !mode.forceQueryAllProviders() {
// If we're not forcing potential changes of version then an
// existing selection from the lock file takes priority over
// the currently-configured version constraints.
if lock := locks.Provider(provider); lock != nil {
if !acceptableVersions.Has(lock.Version()) {
err := fmt.Errorf(
"locked provider %s %s does not match configured version constraint %s; must use terraform init -upgrade to allow selection of new versions",
provider, lock.Version(), getproviders.VersionConstraintsString(versionConstraints),
)
errs[provider] = err
// This is a funny case where we're returning an error
// before we do any querying at all. To keep the event
// stream consistent without introducing an extra event
// type, we'll emit an artificial QueryPackagesBegin for
// this provider before we indicate that it failed using
// QueryPackagesFailure.
if cb := evts.QueryPackagesBegin; cb != nil {
cb(provider, versionConstraints)
}
if cb := evts.QueryPackagesFailure; cb != nil {
cb(provider, err)
}
continue
}
continue MightNeedProvider
acceptableVersions = versions.Only(lock.Version())
locked[provider] = true
}
}
// If we get here then we didn't find any cached version that is
// in our set of acceptable versions.
mightNeed[provider] = acceptableVersions
}
@ -267,7 +271,15 @@ NeedProvider:
}
// If we get here then the source has no packages that meet the given
// version constraint, which we model as a query error.
err = fmt.Errorf("no available releases match the given constraints %s", getproviders.VersionConstraintsString(reqs[provider]))
if locked[provider] {
// This situation should be a rare one: it suggests that a
// version was previously available but was yanked for some
// reason.
lock := locks.Provider(provider)
err = fmt.Errorf("the previously-selected version %s is no longer available", lock.Version())
} else {
err = fmt.Errorf("no available releases match the given constraints %s", getproviders.VersionConstraintsString(reqs[provider]))
}
errs[provider] = err
if cb := evts.QueryPackagesFailure; cb != nil {
cb(provider, err)
@ -286,6 +298,12 @@ NeedProvider:
return nil, err
}
lock := locks.Provider(provider)
var preferredHashes []getproviders.Hash
if lock != nil && lock.Version() == version { // hash changes are expected if the version is also changing
preferredHashes = lock.PreferredHashes()
}
if i.globalCacheDir != nil {
// Step 3a: If our global cache already has this version available then
// we'll just link it in.
@ -293,7 +311,16 @@ NeedProvider:
if cb := evts.LinkFromCacheBegin; cb != nil {
cb(provider, version, i.globalCacheDir.baseDir)
}
err := i.targetDir.LinkFromOtherCache(cached)
if _, err := cached.ExecutableFile(); err != nil {
err := fmt.Errorf("provider binary not found: %s", err)
errs[provider] = err
if cb := evts.LinkFromCacheFailure; cb != nil {
cb(provider, version, err)
}
continue
}
err := i.targetDir.LinkFromOtherCache(cached, preferredHashes)
if err != nil {
errs[provider] = err
if cb := evts.LinkFromCacheFailure; cb != nil {
@ -306,12 +333,60 @@ NeedProvider:
new := i.targetDir.ProviderVersion(provider, version)
if new == nil {
err := fmt.Errorf("after linking %s from provider cache at %s it is still not detected in the target directory; this is a bug in Terraform", provider, i.globalCacheDir.baseDir)
errs[provider] = err
if cb := evts.LinkFromCacheFailure; cb != nil {
cb(provider, version, err)
}
continue
}
selected[provider] = version
// The LinkFromOtherCache call above should've verified that
// the package matches one of the hashes previously recorded,
// if any. We'll now augment those hashes with one freshly
// calculated from the package we just linked, which allows
// the lock file to gradually transition to recording newer hash
// schemes when they become available.
var newHashes []getproviders.Hash
if lock != nil && lock.Version() == version {
// If the version we're installing is identical to the
// one we previously locked then we'll keep all of the
// hashes we saved previously and add to it. Otherwise
// we'll be starting fresh, because each version has its
// own set of packages and thus its own hashes.
newHashes = append(newHashes, preferredHashes...)
// NOTE: The behavior here is unfortunate when a particular
// provider version was already cached on the first time
// the current configuration requested it, because that
// means we don't currently get the opportunity to fetch
// and verify the checksums for the new package from
// upstream. That's currently unavoidable because upstream
// checksums are in the "ziphash" format and so we can't
// verify them against our cache directory's unpacked
// packages: we'd need to go fetch the package from the
// origin and compare against it, which would defeat the
// purpose of the global cache.
//
// If we fetch from upstream on the first encounter with
// a particular provider then we'll end up in the other
// codepath below where we're able to also include the
// checksums from the origin registry.
}
newHash, err := cached.Hash()
if err != nil {
err := fmt.Errorf("after linking %s from provider cache at %s, failed to compute a checksum for it: %s", provider, i.globalCacheDir.baseDir, err)
errs[provider] = err
if cb := evts.LinkFromCacheFailure; cb != nil {
cb(provider, version, err)
}
continue
}
// The hashes slice gets deduplicated in the lock file
// implementation, so we don't worry about potentially
// creating a duplicate here.
newHashes = append(newHashes, newHash)
lock = locks.SetProvider(provider, version, reqs[provider], newHashes)
if cb := evts.LinkFromCacheSuccess; cb != nil {
cb(provider, version, new.PackageDir)
}
@ -350,7 +425,7 @@ NeedProvider:
installTo = i.targetDir
linkTo = nil // no linking needed
}
authResult, err := installTo.InstallPackage(ctx, meta)
authResult, err := installTo.InstallPackage(ctx, meta, preferredHashes)
if err != nil {
// TODO: Consider retrying for certain kinds of error that seem
// likely to be transient. For now, we just treat all errors equally.
@ -369,12 +444,22 @@ NeedProvider:
}
continue
}
if _, err := new.ExecutableFile(); err != nil {
err := fmt.Errorf("provider binary not found: %s", err)
errs[provider] = err
if cb := evts.FetchPackageFailure; cb != nil {
cb(provider, version, err)
}
continue
}
if linkTo != nil {
// We skip emitting the "LinkFromCache..." events here because
// it's simpler for the caller to treat them as mutually exclusive.
// We can just subsume the linking step under the "FetchPackage..."
// series here (and that's why we use FetchPackageFailure below).
err := linkTo.LinkFromOtherCache(new)
// We also don't do a hash check here because we already did that
// as part of the installTo.InstallPackage call above.
err := linkTo.LinkFromOtherCache(new, nil)
if err != nil {
errs[provider] = err
if cb := evts.FetchPackageFailure; cb != nil {
@ -384,7 +469,50 @@ NeedProvider:
}
}
authResults[provider] = authResult
selected[provider] = version
// The InstallPackage call above should've verified that
// the package matches one of the hashes previously recorded,
// if any. We'll now augment those hashes with a new set populated
// with the hashes returned by the upstream source and from the
// package we've just installed, which allows the lock file to
// gradually transition to newer hash schemes when they become
// available.
//
// This is assuming that if a package matches both a hash we saw before
// _and_ a new hash then the new hash is a valid substitute for
// the previous hash.
//
// The hashes slice gets deduplicated in the lock file
// implementation, so we don't worry about potentially
// creating duplicates here.
var newHashes []getproviders.Hash
if lock != nil && lock.Version() == version {
// If the version we're installing is identical to the
// one we previously locked then we'll keep all of the
// hashes we saved previously and add to it. Otherwise
// we'll be starting fresh, because each version has its
// own set of packages and thus its own hashes.
newHashes = append(newHashes, preferredHashes...)
}
newHash, err := new.Hash()
if err != nil {
err := fmt.Errorf("after installing %s, failed to compute a checksum for it: %s", provider, err)
errs[provider] = err
if cb := evts.FetchPackageFailure; cb != nil {
cb(provider, version, err)
}
continue
}
newHashes = append(newHashes, newHash)
if authResult.SignedByAnyParty() {
// We'll trust new hashes from upstream only if they were verified
// as signed by a suitable key. Otherwise, we'd record only
// a new hash we just calculated ourselves from the bytes on disk,
// and so the hashes would cover only the current platform.
newHashes = append(newHashes, meta.AcceptableHashes()...)
}
lock = locks.SetProvider(provider, version, reqs[provider], newHashes)
if cb := evts.FetchPackageSuccess; cb != nil {
cb(provider, version, new.PackageDir, authResult)
}
@ -395,115 +523,12 @@ NeedProvider:
cb(authResults)
}
// We'll remember our selections in a lock file inside the target directory,
// so callers can recover those exact selections later by calling
// SelectedPackages on the same installer.
lockEntries := map[addrs.Provider]lockFileEntry{}
for provider, version := range selected {
cached := i.targetDir.ProviderVersion(provider, version)
if cached == nil {
err := fmt.Errorf("selected package for %s is no longer present in the target directory; this is a bug in Terraform", provider)
errs[provider] = err
if cb := evts.HashPackageFailure; cb != nil {
cb(provider, version, err)
}
continue
}
if _, err := cached.ExecutableFile(); err != nil {
err := fmt.Errorf("provider binary not found: %s", err)
errs[provider] = err
if cb := evts.HashPackageFailure; cb != nil {
cb(provider, version, err)
}
continue
}
hash, err := cached.Hash()
if err != nil {
errs[provider] = fmt.Errorf("failed to calculate checksum for installed provider %s package: %s", provider, err)
if cb := evts.HashPackageFailure; cb != nil {
cb(provider, version, err)
}
continue
}
lockEntries[provider] = lockFileEntry{
SelectedVersion: version,
PackageHash: hash.String(),
}
}
err := i.lockFile().Write(lockEntries)
if err != nil {
// This is one of few cases where this function does _not_ return an
// InstallerError, because failure to write the lock file is a more
// general problem, not specific to a certain provider.
return selected, fmt.Errorf("failed to record a manifest of selected providers: %s", err)
}
if len(errs) > 0 {
return selected, InstallerError{
return locks, InstallerError{
ProviderErrors: errs,
}
}
return selected, nil
}
func (i *Installer) lockFile() *lockFile {
return &lockFile{
filename: filepath.Join(i.targetDir.baseDir, "selections.json"),
}
}
// SelectedPackages returns the metadata about the packages chosen by the
// most recent call to EnsureProviderVersions, which are recorded in a lock
// file in the installer's target directory.
//
// If EnsureProviderVersions has never been run against the current target
// directory, the result is a successful empty response indicating that nothing
// is selected.
//
// SelectedPackages also verifies that the package contents are consistent
// with the checksums that were recorded at installation time, reporting an
// error if not.
func (i *Installer) SelectedPackages() (map[addrs.Provider]*CachedProvider, error) {
entries, err := i.lockFile().Read()
if err != nil {
// Read does not return an error for "file not found", so this should
// always be some other error.
return nil, fmt.Errorf("failed to read selections file: %s", err)
}
ret := make(map[addrs.Provider]*CachedProvider, len(entries))
errs := make(map[addrs.Provider]error)
for provider, entry := range entries {
cached := i.targetDir.ProviderVersion(provider, entry.SelectedVersion)
if cached == nil {
errs[provider] = fmt.Errorf("package for selected version %s is no longer available in the local cache directory", entry.SelectedVersion)
continue
}
hash, err := getproviders.ParseHash(entry.PackageHash)
if err != nil {
errs[provider] = fmt.Errorf("local cache for %s has invalid hash %q: %s", provider, entry.PackageHash, err)
continue
}
ok, err := cached.MatchesHash(hash)
if err != nil {
errs[provider] = fmt.Errorf("failed to verify checksum for v%s package: %s", entry.SelectedVersion, err)
continue
}
if !ok {
errs[provider] = fmt.Errorf("checksum mismatch for v%s package", entry.SelectedVersion)
continue
}
ret[provider] = cached
}
if len(errs) > 0 {
return ret, InstallerError{
ProviderErrors: errs,
}
}
return ret, nil
return locks, nil
}
// InstallMode customizes the details of how an install operation treats

63
internal/providercache/installer_test.go
View File

@ -15,6 +15,7 @@ import (
svchost "github.com/hashicorp/terraform-svchost"
"github.com/hashicorp/terraform-svchost/disco"
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/internal/depsfile"
"github.com/hashicorp/terraform/internal/getproviders"
)
@ -35,33 +36,33 @@ func TestEnsureProviderVersions_local_source(t *testing.T) {
installer := NewInstaller(dir, source)
tests := map[string]struct {
provider string
version string
installed bool
err string
provider string
version string
wantHash getproviders.Hash // getproviders.NilHash if not expected to be installed
err string
}{
"install-unpacked": {
provider: "null",
version: "2.0.0",
installed: true,
provider: "null",
version: "2.0.0",
wantHash: getproviders.HashScheme1.New("qjsREM4DqEWECD43FcPqddZ9oxCG+IaMTxvWPciS05g="),
},
"invalid-zip-file": {
provider: "null",
version: "2.1.0",
installed: false,
err: "zip: not a valid zip file",
provider: "null",
version: "2.1.0",
wantHash: getproviders.NilHash,
err: "zip: not a valid zip file",
},
"version-constraint-unmet": {
provider: "null",
version: "2.2.0",
installed: false,
err: "no available releases match the given constraints 2.2.0",
provider: "null",
version: "2.2.0",
wantHash: getproviders.NilHash,
err: "no available releases match the given constraints 2.2.0",
},
"missing-executable": {
provider: "missing/executable",
version: "2.0.0",
installed: true,
err: "provider binary not found: could not find executable file starting with terraform-provider-executable",
provider: "missing/executable",
version: "2.0.0",
wantHash: getproviders.NilHash, // installation fails for a provider with no executable
err: "provider binary not found: could not find executable file starting with terraform-provider-executable",
},
}
@ -75,14 +76,24 @@ func TestEnsureProviderVersions_local_source(t *testing.T) {
reqs := getproviders.Requirements{
provider: versionConstraint,
}
wantSelected := getproviders.Selections{provider: version}
if !test.installed {
wantSelected = getproviders.Selections{}
newLocks, err := installer.EnsureProviderVersions(ctx, depsfile.NewLocks(), reqs, InstallNewProvidersOnly)
gotProviderlocks := newLocks.AllProviders()
wantProviderLocks := map[addrs.Provider]*depsfile.ProviderLock{
provider: depsfile.NewProviderLock(
provider,
version,
getproviders.MustParseVersionConstraints("= 2.0.0"),
[]getproviders.Hash{
test.wantHash,
},
),
}
if test.wantHash == getproviders.NilHash {
wantProviderLocks = map[addrs.Provider]*depsfile.ProviderLock{}
}
selected, err := installer.EnsureProviderVersions(ctx, reqs, InstallNewProvidersOnly)
if diff := cmp.Diff(wantSelected, selected); diff != "" {
if diff := cmp.Diff(wantProviderLocks, gotProviderlocks, depsfile.ProviderLockComparer); diff != "" {
t.Errorf("wrong selected\n%s", diff)
}
@ -158,7 +169,7 @@ func TestEnsureProviderVersions_protocol_errors(t *testing.T) {
test.provider: test.inputVersion,
}
ctx := context.TODO()
_, err := installer.EnsureProviderVersions(ctx, reqs, InstallNewProvidersOnly)
_, err := installer.EnsureProviderVersions(ctx, depsfile.NewLocks(), reqs, InstallNewProvidersOnly)
switch err := err.(type) {
case nil:

114
internal/providercache/lock_file.go
View File

@ -1,114 +0,0 @@
package providercache
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/internal/getproviders"
)
// lockFile represents a file on disk that captures selected versions and
// their associated package checksums resulting from an install process, so
// that later consumers of that install process can be sure they are reading
// an identical set of providers to what the install process intended.
//
// This is an internal type used to encapsulate the reading, parsing,
// serializing, and writing of lock files. Its public interface is via methods
// on type Installer.
type lockFile struct {
filename string
}
// LockFileEntry represents an entry for a specific provider in a LockFile.
type lockFileEntry struct {
SelectedVersion getproviders.Version
PackageHash string
}
var _ json.Marshaler = (*lockFileEntry)(nil)
var _ json.Unmarshaler = (*lockFileEntry)(nil)
// Read returns the current locks captured in the lock file.
//
// If the file does not exist, the result is successful but empty to indicate
// that no providers at all are available for use.
func (lf *lockFile) Read() (map[addrs.Provider]lockFileEntry, error) {
buf, err := ioutil.ReadFile(lf.filename)
if err != nil {
if os.IsNotExist(err) {
return nil, nil // no file means no locks yet
}
return nil, err
}
var rawEntries map[string]*lockFileEntry
err = json.Unmarshal(buf, &rawEntries)
if err != nil {
return nil, fmt.Errorf("error parsing %s: %s", lf.filename, err)
}
ret := make(map[addrs.Provider]lockFileEntry, len(rawEntries))
for providerStr, entry := range rawEntries {
provider, diags := addrs.ParseProviderSourceString(providerStr)
if diags.HasErrors() {
// This file is both generated and consumed by Terraform, so we
// don't use super-detailed error messages for problems in it.
// If we get here without someone tampering with the file then
// it's presumably a bug in either our serializer or our parser.
return nil, fmt.Errorf("error parsing %s: invalid provider address %q", lf.filename, providerStr)
}
ret[provider] = *entry
}
return ret, nil
}
// Write stores a new set of entries in the lock file, disarding any
// selections previously stored there.
func (lf *lockFile) Write(new map[addrs.Provider]lockFileEntry) error {
toStore := make(map[string]*lockFileEntry, len(new))
for provider := range new {
entry := new[provider] // so that each reference below is to a different object
toStore[provider.String()] = &entry
}
buf, err := json.MarshalIndent(toStore, "", " ")
if err != nil {
return fmt.Errorf("error writing %s: %s", lf.filename, err)
}
os.MkdirAll(
filepath.Dir(lf.filename), 0775,
) // ignore error since WriteFile below will generate a better one anyway
return ioutil.WriteFile(lf.filename, buf, 0664)
}
func (lfe *lockFileEntry) UnmarshalJSON(src []byte) error {
type Raw struct {
VersionStr string `json:"version"`
Hash string `json:"hash"`
}
var raw Raw
err := json.Unmarshal(src, &raw)
if err != nil {
return err
}
version, err := getproviders.ParseVersion(raw.VersionStr)
if err != nil {
return fmt.Errorf("invalid version number: %s", err)
}
lfe.SelectedVersion = version
lfe.PackageHash = raw.Hash
return nil
}
func (lfe *lockFileEntry) MarshalJSON() ([]byte, error) {
return json.Marshal(map[string]interface{}{
"version": lfe.SelectedVersion.String(),
"hash": lfe.PackageHash,
})
}

27
internal/providercache/package_install.go
View File

@ -23,7 +23,7 @@ import (
// specific protocol and set of expectations.)
var unzip = getter.ZipDecompressor{}
func installFromHTTPURL(ctx context.Context, meta getproviders.PackageMeta, targetDir string) (*getproviders.PackageAuthenticationResult, error) {
func installFromHTTPURL(ctx context.Context, meta getproviders.PackageMeta, targetDir string, allowedHashes []getproviders.Hash) (*getproviders.PackageAuthenticationResult, error) {
url := meta.Location.String()
// When we're installing from an HTTP URL we expect the URL to refer to
@ -83,7 +83,9 @@ func installFromHTTPURL(ctx context.Context, meta getproviders.PackageMeta, targ
// We can now delegate to installFromLocalArchive for extraction. To do so,
// we construct a new package meta description using the local archive
// path as the location, and skipping authentication.
// path as the location, and skipping authentication. installFromLocalMeta
// is responsible for verifying that the archive matches the allowedHashes,
// though.
localMeta := getproviders.PackageMeta{
Provider: meta.Provider,
Version: meta.Version,
@ -93,13 +95,13 @@ func installFromHTTPURL(ctx context.Context, meta getproviders.PackageMeta, targ
Location: localLocation,
Authentication: nil,
}
if _, err := installFromLocalArchive(ctx, localMeta, targetDir); err != nil {
if _, err := installFromLocalArchive(ctx, localMeta, targetDir, allowedHashes); err != nil {
return nil, err
}
return authResult, nil
}
func installFromLocalArchive(ctx context.Context, meta getproviders.PackageMeta, targetDir string) (*getproviders.PackageAuthenticationResult, error) {
func installFromLocalArchive(ctx context.Context, meta getproviders.PackageMeta, targetDir string, allowedHashes []getproviders.Hash) (*getproviders.PackageAuthenticationResult, error) {
var authResult *getproviders.PackageAuthenticationResult
if meta.Authentication != nil {
var err error
@ -107,6 +109,21 @@ func installFromLocalArchive(ctx context.Context, meta getproviders.PackageMeta,
return nil, err
}
}
if len(allowedHashes) > 0 {
if matches, err := meta.MatchesAnyHash(allowedHashes); err != nil {
return authResult, fmt.Errorf(
"failed to calculate checksum for %s %s package at %s: %s",
meta.Provider, meta.Version, meta.Location, err,
)
} else if !matches {
return authResult, fmt.Errorf(
"the current package for %s %s doesn't match any of the checksums previously recorded in the dependency lock file",
meta.Provider, meta.Version,
)
}
}
filename := meta.Location.String()
err := unzip.Decompress(targetDir, filename, true)
@ -121,7 +138,7 @@ func installFromLocalArchive(ctx context.Context, meta getproviders.PackageMeta,
// a local directory source _and_ of linking a package from another cache
// in LinkFromOtherCache, because they both do fundamentally the same
// operation: symlink if possible, or deep-copy otherwise.
func installFromLocalDir(ctx context.Context, meta getproviders.PackageMeta, targetDir string) (*getproviders.PackageAuthenticationResult, error) {
func installFromLocalDir(ctx context.Context, meta getproviders.PackageMeta, targetDir string, allowedHashes []getproviders.Hash) (*getproviders.PackageAuthenticationResult, error) {
sourceDir := meta.Location.String()
absNew, err := filepath.Abs(targetDir)

11
tools/terraform-bundle/package.go
View File

@ -17,6 +17,7 @@ import (
"github.com/hashicorp/terraform-svchost/disco"
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/httpclient"
"github.com/hashicorp/terraform/internal/depsfile"
"github.com/hashicorp/terraform/internal/getproviders"
"github.com/hashicorp/terraform/internal/providercache"
discovery "github.com/hashicorp/terraform/plugin/discovery"
@ -378,7 +379,7 @@ func (c *PackageCommand) ensureProviderVersions(installer *providercache.Install
ProviderAlreadyInstalled: func(provider addrs.Provider, selectedVersion getproviders.Version) {
c.ui.Info(fmt.Sprintf("- Using previously-installed %s v%s", provider.ForDisplay(), selectedVersion))
},
QueryPackagesBegin: func(provider addrs.Provider, versionConstraints getproviders.VersionConstraints) {
QueryPackagesBegin: func(provider addrs.Provider, versionConstraints getproviders.VersionConstraints, locked bool) {
if len(versionConstraints) > 0 {
c.ui.Info(fmt.Sprintf("- Finding %s versions matching %q...", provider.ForDisplay(), getproviders.VersionConstraintsString(versionConstraints)))
} else {
@ -405,7 +406,13 @@ func (c *PackageCommand) ensureProviderVersions(installer *providercache.Install
return err
}
req[provider] = cstr
_, err = installer.EnsureProviderVersions(ctx, req, mode)
// We always start with no locks here, because we want to take
// the newest version matching the given version constraint, and
// never consider anything that might've been selected before.
locks := depsfile.NewLocks()
_, err = installer.EnsureProviderVersions(ctx, locks, req, mode)
if err != nil {
return err
}