From ae080481c0d9190b168f98d007912aa20fa14b34 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 27 Mar 2020 16:50:04 -0700 Subject: [PATCH] internal/providercache: Installer records its selections in a file Just as with the old installer mechanism, our goal is that explicit provider installation is the only way that new provider versions can be selected. To achieve that, we conclude each call to EnsureProviderVersions by writing a selections lock file into the target directory. A later caller can then recall the selections from that file by calling SelectedPackages, which both ensures that it selects the same set of versions and also verifies that the checksums recorded by the installer still match. This new selections.json file has a different layout than our old plugins.json lock file. Not only does it use a different hashing algorithm than before, we also record explicitly which version of each provider was selected. In the old model, we'd repeat normal discovery when reloading the lock file and then fail with a confusing error message if discovery happened to select a different version, but now we'll be able to distinguish between a package that's gone missing since installation (which could previously have then selected a different available version) from a package that has been modified. --- internal/providercache/installer.go | 91 ++++++++++++++++ internal/providercache/installer_events.go | 6 ++ internal/providercache/lock_file.go | 114 +++++++++++++++++++++ 3 files changed, 211 insertions(+) create mode 100644 internal/providercache/lock_file.go diff --git a/internal/providercache/installer.go b/internal/providercache/installer.go index 66c7395b1..82e5beccd 100644 --- a/internal/providercache/installer.go +++ b/internal/providercache/installer.go @@ -3,6 +3,7 @@ package providercache import ( "context" "fmt" + "path/filepath" "sort" "strings" @@ -267,6 +268,7 @@ NeedProvider: new := installTo.ProviderVersion(provider, version) if new == nil { err := fmt.Errorf("after installing %s it is still not detected in the target directory; this is a bug in Terraform", provider) + errs[provider] = err if cb := evts.FetchPackageFailure; cb != nil { cb(provider, version, err) } @@ -292,6 +294,41 @@ NeedProvider: } } + // We'll remember our selections in a lock file inside the target directory, + // so callers can recover those exact selections later by calling + // SelectedPackages on the same installer. + lockEntries := map[addrs.Provider]lockFileEntry{} + for provider, version := range selected { + cached := i.targetDir.ProviderVersion(provider, version) + if cached == nil { + err := fmt.Errorf("selected package for %s is no longer present in the target directory; this is a bug in Terraform", provider) + errs[provider] = err + if cb := evts.HashPackageFailure; cb != nil { + cb(provider, version, err) + } + continue + } + hash, err := cached.Hash() + if err != nil { + errs[provider] = fmt.Errorf("failed to calculate checksum for installed provider %s package: %s", provider, err) + if cb := evts.HashPackageFailure; cb != nil { + cb(provider, version, err) + } + continue + } + lockEntries[provider] = lockFileEntry{ + SelectedVersion: version, + PackageHash: hash, + } + } + err := i.lockFile().Write(lockEntries) + if err != nil { + // This is one of few cases where this function does _not_ return an + // InstallerError, because failure to write the lock file is a more + // general problem, not specific to a certain provider. + return selected, fmt.Errorf("failed to record a manifest of selected providers: %s", err) + } + if len(errs) > 0 { return selected, InstallerError{ ProviderErrors: errs, @@ -300,6 +337,60 @@ NeedProvider: return selected, nil } +func (i *Installer) lockFile() *lockFile { + return &lockFile{ + filename: filepath.Join(i.targetDir.baseDir, "selections.json"), + } +} + +// SelectedPackages returns the metadata about the packages chosen by the +// most recent call to EnsureProviderVersions, which are recorded in a lock +// file in the installer's target directory. +// +// If EnsureProviderVersions has never been run against the current target +// directory, the result is a successful empty response indicating that nothing +// is selected. +// +// SelectedPackages also verifies that the package contents are consistent +// with the checksums that were recorded at installation time, reporting an +// error if not. +func (i *Installer) SelectedPackages() (map[addrs.Provider]*CachedProvider, error) { + entries, err := i.lockFile().Read() + if err != nil { + // Read does not return an error for "file not found", so this should + // always be some other error. + return nil, fmt.Errorf("failed to read selections file: %s", err) + } + + ret := make(map[addrs.Provider]*CachedProvider, len(entries)) + errs := make(map[addrs.Provider]error) + for provider, entry := range entries { + cached := i.targetDir.ProviderVersion(provider, entry.SelectedVersion) + if cached == nil { + errs[provider] = fmt.Errorf("package for selected version %s is no longer available in the local cache directory", entry.SelectedVersion) + continue + } + + ok, err := cached.MatchesHash(entry.PackageHash) + if err != nil { + errs[provider] = fmt.Errorf("failed to verify checksum for v%s package: %s", entry.SelectedVersion, err) + continue + } + if !ok { + errs[provider] = fmt.Errorf("checksum mismatch for v%s package", entry.SelectedVersion) + continue + } + ret[provider] = cached + } + + if len(errs) > 0 { + return ret, InstallerError{ + ProviderErrors: errs, + } + } + return ret, nil +} + // InstallMode customizes the details of how an install operation treats // providers that have versions already cached in the target directory. type InstallMode rune diff --git a/internal/providercache/installer_events.go b/internal/providercache/installer_events.go index 593cbe166..05722bd88 100644 --- a/internal/providercache/installer_events.go +++ b/internal/providercache/installer_events.go @@ -93,6 +93,12 @@ type InstallerEvents struct { FetchPackageSuccess func(provider addrs.Provider, version getproviders.Version, localDir string) FetchPackageRetry func(provider addrs.Provider, version getproviders.Version, err error) FetchPackageFailure func(provider addrs.Provider, version getproviders.Version, err error) + + // HashPackageFailure is called if the installer is unable to determine + // the hash of the contents of an installed package after installation. + // In that case, the selection will not be recorded in the target cache + // directory's lock file. + HashPackageFailure func(provider addrs.Provider, version getproviders.Version, err error) } // OnContext produces a context with all of the same behaviors as the given diff --git a/internal/providercache/lock_file.go b/internal/providercache/lock_file.go new file mode 100644 index 000000000..7b572dd73 --- /dev/null +++ b/internal/providercache/lock_file.go @@ -0,0 +1,114 @@ +package providercache + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/hashicorp/terraform/addrs" + "github.com/hashicorp/terraform/internal/getproviders" +) + +// lockFile represents a file on disk that captures selected versions and +// their associated package checksums resulting from an install process, so +// that later consumers of that install process can be sure they are reading +// an identical set of providers to what the install process intended. +// +// This is an internal type used to encapsulate the reading, parsing, +// serializing, and writing of lock files. Its public interface is via methods +// on type Installer. +type lockFile struct { + filename string +} + +// LockFileEntry represents an entry for a specific provider in a LockFile. +type lockFileEntry struct { + SelectedVersion getproviders.Version + PackageHash string +} + +var _ json.Marshaler = (*lockFileEntry)(nil) +var _ json.Unmarshaler = (*lockFileEntry)(nil) + +// Read returns the current locks captured in the lock file. +// +// If the file does not exist, the result is successful but empty to indicate +// that no providers at all are available for use. +func (lf *lockFile) Read() (map[addrs.Provider]lockFileEntry, error) { + buf, err := ioutil.ReadFile(lf.filename) + if err != nil { + if os.IsNotExist(err) { + return nil, nil // no file means no locks yet + } + return nil, err + } + + var rawEntries map[string]*lockFileEntry + err = json.Unmarshal(buf, &rawEntries) + if err != nil { + return nil, fmt.Errorf("error parsing %s: %s", lf.filename, err) + } + + ret := make(map[addrs.Provider]lockFileEntry, len(rawEntries)) + for providerStr, entry := range rawEntries { + provider, diags := addrs.ParseProviderSourceString(providerStr) + if diags.HasErrors() { + // This file is both generated and consumed by Terraform, so we + // don't use super-detailed error messages for problems in it. + // If we get here without someone tampering with the file then + // it's presumably a bug in either our serializer or our parser. + return nil, fmt.Errorf("error parsing %s: invalid provider address %q", lf.filename, providerStr) + } + ret[provider] = *entry + } + + return ret, nil +} + +// Write stores a new set of entries in the lock file, disarding any +// selections previously stored there. +func (lf *lockFile) Write(new map[addrs.Provider]lockFileEntry) error { + toStore := make(map[string]*lockFileEntry, len(new)) + for provider := range new { + entry := new[provider] // so that each reference below is to a different object + toStore[provider.String()] = &entry + } + + buf, err := json.MarshalIndent(toStore, "", " ") + if err != nil { + return fmt.Errorf("error writing %s: %s", lf.filename, err) + } + + os.MkdirAll( + filepath.Dir(lf.filename), 0660, + ) // ignore error since WriteFile below will generate a better one anyway + return ioutil.WriteFile(lf.filename, buf, 0660) +} + +func (lfe *lockFileEntry) UnmarshalJSON(src []byte) error { + type Raw struct { + VersionStr string `json:"version"` + Hash string `json:"hash"` + } + var raw Raw + err := json.Unmarshal(src, &raw) + if err != nil { + return err + } + version, err := getproviders.ParseVersion(raw.VersionStr) + if err != nil { + return fmt.Errorf("invalid version number: %s", err) + } + lfe.SelectedVersion = version + lfe.PackageHash = raw.Hash + return nil +} + +func (lfe *lockFileEntry) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + "version": lfe.SelectedVersion.String(), + "hash": lfe.PackageHash, + }) +}