terraform/plugin/discovery/get.go

694 lines
24 KiB
Go
Raw Normal View History

package discovery
import (
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"github.com/hashicorp/errwrap"
getter "github.com/hashicorp/go-getter"
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform-svchost/disco"
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/httpclient"
"github.com/hashicorp/terraform/registry"
"github.com/hashicorp/terraform/registry/regsrc"
"github.com/hashicorp/terraform/registry/response"
"github.com/hashicorp/terraform/tfdiags"
tfversion "github.com/hashicorp/terraform/version"
"github.com/mitchellh/cli"
)
// Releases are located by querying the terraform registry.
const protocolVersionHeader = "x-terraform-protocol-version"
var httpClient *http.Client
var errVersionNotFound = errors.New("version not found")
func init() {
httpClient = httpclient.New()
httpGetter := &getter.HttpGetter{
Client: httpClient,
Netrc: true,
}
getter.Getters["http"] = httpGetter
getter.Getters["https"] = httpGetter
}
// An Installer maintains a local cache of plugins by downloading plugins
// from an online repository.
type Installer interface {
Get(provider addrs.Provider, req Constraints) (PluginMeta, tfdiags.Diagnostics, error)
PurgeUnused(used map[string]PluginMeta) (removed PluginMetaSet, err error)
}
// ProviderInstaller is an Installer implementation that knows how to
// download Terraform providers from the official HashiCorp releases service
// into a local directory. The files downloaded are compliant with the
// naming scheme expected by FindPlugins, so the target directory of a
// provider installer can be used as one of several plugin discovery sources.
type ProviderInstaller struct {
Dir string
// Cache is used to access and update a local cache of plugins if non-nil.
// Can be nil to disable caching.
Cache PluginCache
PluginProtocolVersion uint
// OS and Arch specify the OS and architecture that should be used when
// installing plugins. These use the same labels as the runtime.GOOS and
// runtime.GOARCH variables respectively, and indeed the values of these
// are used as defaults if either of these is the empty string.
OS string
Arch string
// Skip checksum and signature verification
SkipVerify bool
Ui cli.Ui // Ui for output
// Services is a required *disco.Disco, which may have services and
// credentials pre-loaded.
Services *disco.Disco
// registry client
registry *registry.Client
}
// Get is part of an implementation of type Installer, and attempts to download
// and install a Terraform provider matching the given constraints.
//
// This method may return one of a number of sentinel errors from this
// package to indicate issues that are likely to be resolvable via user action:
//
// ErrorNoSuchProvider: no provider with the given name exists in the repository.
// ErrorNoSuitableVersion: the provider exists but no available version matches constraints.
// ErrorNoVersionCompatible: a plugin was found within the constraints but it is
// incompatible with the current Terraform version.
//
// These errors should be recognized and handled as special cases by the caller
// to present a suitable user-oriented error message.
//
// All other errors indicate an internal problem that is likely _not_ solvable
// through user action, or at least not within Terraform's scope. Error messages
// are produced under the assumption that if presented to the user they will
// be presented alongside context about what is being installed, and thus the
// error messages do not redundantly include such information.
func (i *ProviderInstaller) Get(provider addrs.Provider, req Constraints) (PluginMeta, tfdiags.Diagnostics, error) {
var diags tfdiags.Diagnostics
// a little bit of initialization.
if i.OS == "" {
i.OS = runtime.GOOS
}
if i.Arch == "" {
i.Arch = runtime.GOARCH
}
if i.registry == nil {
i.registry = registry.NewClient(i.Services, nil)
}
// get a full listing of versions for the requested provider
allVersions, err := i.listProviderVersions(provider)
// TODO: return multiple errors
if err != nil {
log.Printf("[DEBUG] %s", err)
if registry.IsServiceUnreachable(err) {
registryHost, err := i.hostname()
if err == nil && registryHost == regsrc.PublicRegistryHost.Raw {
return PluginMeta{}, diags, ErrorPublicRegistryUnreachable
}
return PluginMeta{}, diags, ErrorServiceUnreachable
}
if registry.IsServiceNotProvided(err) {
return PluginMeta{}, diags, err
}
return PluginMeta{}, diags, ErrorNoSuchProvider
}
// Add any warnings from the response to diags
for _, warning := range allVersions.Warnings {
hostname, err := i.hostname()
if err != nil {
return PluginMeta{}, diags, err
}
diag := tfdiags.SimpleWarning(fmt.Sprintf("%s: %s", hostname, warning))
diags = diags.Append(diag)
}
if len(allVersions.Versions) == 0 {
return PluginMeta{}, diags, ErrorNoSuitableVersion
}
2018-11-28 17:26:16 +01:00
providerSource := allVersions.ID
// Filter the list of plugin versions to those which meet the version constraints
versions := allowedVersions(allVersions, req)
if len(versions) == 0 {
return PluginMeta{}, diags, ErrorNoSuitableVersion
}
// sort them newest to oldest. The newest version wins!
response.ProviderVersionCollection(versions).Sort()
// if the chosen provider version does not support the requested platform,
// filter the list of acceptable versions to those that support that platform
if err := i.checkPlatformCompatibility(versions[0]); err != nil {
versions = i.platformCompatibleVersions(versions)
if len(versions) == 0 {
return PluginMeta{}, diags, ErrorNoVersionCompatibleWithPlatform
}
}
// we now have a winning platform-compatible version
versionMeta := versions[0]
v := VersionStr(versionMeta.Version).MustParse()
// check protocol compatibility
if err := i.checkPluginProtocol(versionMeta); err != nil {
closestMatch, err := i.findClosestProtocolCompatibleVersion(allVersions.Versions)
if err != nil {
// No operation here if we can't find a version with compatible protocol
return PluginMeta{}, diags, err
}
// Prompt version suggestion to UI based on closest protocol match
var errMsg string
closestVersion := VersionStr(closestMatch.Version).MustParse()
if v.NewerThan(closestVersion) {
errMsg = providerProtocolTooNew
} else {
errMsg = providerProtocolTooOld
}
constraintStr := req.String()
if constraintStr == "" {
constraintStr = "(any version)"
}
return PluginMeta{}, diags, errwrap.Wrap(ErrorVersionIncompatible, fmt.Errorf(fmt.Sprintf(
errMsg, provider, v.String(), tfversion.String(),
closestVersion.String(), closestVersion.MinorUpgradeConstraintStr(), constraintStr)))
}
2018-11-28 17:26:16 +01:00
downloadURLs, err := i.listProviderDownloadURLs(providerSource, versionMeta.Version)
2019-07-19 21:20:50 +02:00
if err != nil {
return PluginMeta{}, diags, err
}
providerURL := downloadURLs.DownloadURL
if !i.SkipVerify {
// Terraform verifies the integrity of a provider release before downloading
// the plugin binary. The digital signature (SHA256SUMS.sig) on the
// release distribution (SHA256SUMS) is verified with the public key of the
// publisher provided in the Terraform Registry response, ensuring that
// everything is as intended by the publisher. The checksum of the provider
// plugin is expected in the SHA256SUMS file and is double checked to match
// the checksum of the original published release to the Registry. This
// enforces immutability of releases between the Registry and the plugin's
// host location. Lastly, the integrity of the binary is verified upon
// download matches the Registry and signed checksum.
sha256, err := i.getProviderChecksum(downloadURLs)
if err != nil {
return PluginMeta{}, diags, err
}
// add the checksum parameter for go-getter to verify the download for us.
if sha256 != "" {
providerURL = providerURL + "?checksum=sha256:" + sha256
}
}
printedProviderName := fmt.Sprintf("%q (%s)", provider.Type, providerSource)
i.Ui.Info(fmt.Sprintf("- Downloading plugin for provider %s %s...", printedProviderName, versionMeta.Version))
log.Printf("[DEBUG] getting provider %s version %q", printedProviderName, versionMeta.Version)
err = i.install(provider, v, providerURL)
if err != nil {
return PluginMeta{}, diags, err
}
// Find what we just installed
// (This is weird, because go-getter doesn't directly return
// information about what was extracted, and we just extracted
// the archive directly into a shared dir here.)
log.Printf("[DEBUG] looking for the %s %s plugin we just installed", provider.Type, versionMeta.Version)
metas := FindPlugins("provider", []string{i.Dir})
log.Printf("[DEBUG] all plugins found %#v", metas)
metas, _ = metas.ValidateVersions()
metas = metas.WithName(provider.Type).WithVersion(v)
log.Printf("[DEBUG] filtered plugins %#v", metas)
if metas.Count() == 0 {
// This should never happen. Suggests that the release archive
// contains an executable file whose name doesn't match the
// expected convention.
return PluginMeta{}, diags, fmt.Errorf(
"failed to find installed plugin version %s; this is a bug in Terraform and should be reported",
versionMeta.Version,
)
}
if metas.Count() > 1 {
// This should also never happen, and suggests that a
// particular version was re-released with a different
// executable filename. We consider releases as immutable, so
// this is an error.
return PluginMeta{}, diags, fmt.Errorf(
"multiple plugins installed for version %s; this is a bug in Terraform and should be reported",
versionMeta.Version,
)
}
// By now we know we have exactly one meta, and so "Newest" will
// return that one.
return metas.Newest(), diags, nil
}
func (i *ProviderInstaller) install(provider addrs.Provider, version Version, url string) error {
if i.Cache != nil {
log.Printf("[DEBUG] looking for provider %s %s in plugin cache", provider.Type, version)
cached := i.Cache.CachedPluginPath("provider", provider.Type, version)
if cached == "" {
log.Printf("[DEBUG] %s %s not yet in cache, so downloading %s", provider.Type, version, url)
err := getter.Get(i.Cache.InstallDir(), url)
if err != nil {
return err
}
// should now be in cache
cached = i.Cache.CachedPluginPath("provider", provider.Type, version)
if cached == "" {
// should never happen if the getter is behaving properly
// and the plugins are packaged properly.
return fmt.Errorf("failed to find downloaded plugin in cache %s", i.Cache.InstallDir())
}
}
// Link or copy the cached binary into our install dir so the
// normal resolution machinery can find it.
filename := filepath.Base(cached)
targetPath := filepath.Join(i.Dir, filename)
// check if the target dir exists, and create it if not
var err error
if _, StatErr := os.Stat(i.Dir); os.IsNotExist(StatErr) {
err = os.MkdirAll(i.Dir, 0700)
}
if err != nil {
return err
}
log.Printf("[DEBUG] installing %s %s to %s from local cache %s", provider.Type, version, targetPath, cached)
// Delete if we can. If there's nothing there already then no harm done.
// This is important because we can't create a link if there's
// already a file of the same name present.
// (any other error here we'll catch below when we try to write here)
os.Remove(targetPath)
// We don't attempt linking on Windows because links are not
// comprehensively supported by all tools/apps in Windows and
// so we choose to be conservative to avoid creating any
// weird issues for Windows users.
linkErr := errors.New("link not supported for Windows") // placeholder error, never actually returned
if runtime.GOOS != "windows" {
// Try hard linking first. Hard links are preferable because this
// creates a self-contained directory that doesn't depend on the
// cache after install.
linkErr = os.Link(cached, targetPath)
// If that failed, try a symlink. This _does_ depend on the cache
// after install, so the user must manage the cache more carefully
// in this case, but avoids creating redundant copies of the
// plugins on disk.
if linkErr != nil {
linkErr = os.Symlink(cached, targetPath)
}
}
// If we still have an error then we'll try a copy as a fallback.
// In this case either the OS is Windows or the target filesystem
// can't support symlinks.
if linkErr != nil {
srcFile, err := os.Open(cached)
if err != nil {
return fmt.Errorf("failed to open cached plugin %s: %s", cached, err)
}
defer srcFile.Close()
destFile, err := os.OpenFile(targetPath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, os.ModePerm)
if err != nil {
return fmt.Errorf("failed to create %s: %s", targetPath, err)
}
_, err = io.Copy(destFile, srcFile)
if err != nil {
destFile.Close()
return fmt.Errorf("failed to copy cached plugin from %s to %s: %s", cached, targetPath, err)
}
err = destFile.Close()
if err != nil {
return fmt.Errorf("error creating %s: %s", targetPath, err)
}
}
// One way or another, by the time we get here we should have either
// a link or a copy of the cached plugin within i.Dir, as expected.
} else {
log.Printf("[DEBUG] plugin cache is disabled, so downloading %s %s from %s", provider.Type, version, url)
err := getter.Get(i.Dir, url)
if err != nil {
return err
}
}
return nil
}
func (i *ProviderInstaller) PurgeUnused(used map[string]PluginMeta) (PluginMetaSet, error) {
purge := make(PluginMetaSet)
present := FindPlugins("provider", []string{i.Dir})
for meta := range present {
chosen, ok := used[meta.Name]
if !ok {
purge.Add(meta)
}
if chosen.Path != meta.Path {
purge.Add(meta)
}
}
removed := make(PluginMetaSet)
var errs error
for meta := range purge {
path := meta.Path
err := os.Remove(path)
if err != nil {
errs = multierror.Append(errs, fmt.Errorf(
"failed to remove unused provider plugin %s: %s",
path, err,
))
} else {
removed.Add(meta)
}
}
return removed, errs
}
func (i *ProviderInstaller) getProviderChecksum(resp *response.TerraformProviderPlatformLocation) (string, error) {
// Get SHA256SUMS file.
shasums, err := getFile(resp.ShasumsURL)
if err != nil {
log.Printf("[ERROR] error fetching checksums from %q: %s", resp.ShasumsURL, err)
return "", ErrorMissingChecksumVerification
}
// Get SHA256SUMS.sig file.
signature, err := getFile(resp.ShasumsSignatureURL)
if err != nil {
log.Printf("[ERROR] error fetching checksums signature from %q: %s", resp.ShasumsSignatureURL, err)
return "", ErrorSignatureVerification
}
// Verify the GPG signature returned from the Registry.
asciiArmor := resp.SigningKeys.GPGASCIIArmor()
signer, err := verifySig(shasums, signature, asciiArmor)
if err != nil {
log.Printf("[ERROR] error verifying signature: %s", err)
return "", ErrorSignatureVerification
}
// Also verify the GPG signature against the HashiCorp public key. This is
// a temporary additional check until a more robust key verification
// process is added in a future release.
_, err = verifySig(shasums, signature, HashicorpPublicKey)
if err != nil {
log.Printf("[ERROR] error verifying signature against HashiCorp public key: %s", err)
return "", ErrorSignatureVerification
}
// Display identity for GPG key which succeeded verifying the signature.
// This could also be used to display to the user with i.Ui.Info().
identities := []string{}
for k := range signer.Identities {
identities = append(identities, k)
}
identity := strings.Join(identities, ", ")
log.Printf("[DEBUG] verified GPG signature with key from %s", identity)
// Extract checksum for this os/arch platform binary and verify against Registry
checksum := checksumForFile(shasums, resp.Filename)
if checksum == "" {
log.Printf("[ERROR] missing checksum for %s from source %s", resp.Filename, resp.ShasumsURL)
return "", ErrorMissingChecksumVerification
} else if checksum != resp.Shasum {
log.Printf("[ERROR] unexpected checksum for %s from source %q", resp.Filename, resp.ShasumsURL)
return "", ErrorChecksumVerification
}
return checksum, nil
}
func (i *ProviderInstaller) hostname() (string, error) {
provider := regsrc.NewTerraformProvider("", i.OS, i.Arch)
svchost, err := provider.SvcHost()
if err != nil {
return "", err
}
return svchost.ForDisplay(), nil
}
// list all versions available for the named provider
func (i *ProviderInstaller) listProviderVersions(provider addrs.Provider) (*response.TerraformProviderVersions, error) {
req := regsrc.NewTerraformProvider(provider.Type, i.OS, i.Arch)
versions, err := i.registry.TerraformProviderVersions(req)
return versions, err
}
func (i *ProviderInstaller) listProviderDownloadURLs(name, version string) (*response.TerraformProviderPlatformLocation, error) {
urls, err := i.registry.TerraformProviderLocation(regsrc.NewTerraformProvider(name, i.OS, i.Arch), version)
if urls == nil {
return nil, fmt.Errorf("No download urls found for provider %s", name)
}
return urls, err
}
// findClosestProtocolCompatibleVersion searches for the provider version with the closest protocol match.
// Prerelease versions are filtered.
func (i *ProviderInstaller) findClosestProtocolCompatibleVersion(versions []*response.TerraformProviderVersion) (*response.TerraformProviderVersion, error) {
// Loop through all the provider versions to find the earliest and latest
// versions that match the installer protocol to then select the closest of the two
var latest, earliest *response.TerraformProviderVersion
for _, version := range versions {
// Prereleases are filtered and will not be suggested
v, err := VersionStr(version.Version).Parse()
if err != nil || v.IsPrerelease() {
continue
}
if err := i.checkPluginProtocol(version); err == nil {
if earliest == nil {
// Found the first provider version with compatible protocol
earliest = version
}
// Update the latest protocol compatible version
latest = version
}
}
if earliest == nil {
// No compatible protocol was found for any version
return nil, ErrorNoVersionCompatible
}
// Convert protocols to comparable types
protoString := strconv.Itoa(int(i.PluginProtocolVersion))
protocolVersion, err := VersionStr(protoString).Parse()
if err != nil {
return nil, fmt.Errorf("invalid plugin protocol version: %q", i.PluginProtocolVersion)
}
earliestVersionProtocol, err := VersionStr(earliest.Protocols[0]).Parse()
if err != nil {
return nil, err
}
// Compare installer protocol version with the first protocol listed of the earliest match
// [A, B] where A is assumed the earliest compatible major version of the protocol pair
if protocolVersion.NewerThan(earliestVersionProtocol) {
// Provider protocols are too old, the closest version is the earliest compatible version
return earliest, nil
}
// Provider protocols are too new, the closest version is the latest compatible version
return latest, nil
}
func (i *ProviderInstaller) checkPluginProtocol(versionMeta *response.TerraformProviderVersion) error {
// TODO: should this be a different error? We should probably differentiate between
// no compatible versions and no protocol versions listed at all
if len(versionMeta.Protocols) == 0 {
return fmt.Errorf("no plugin protocol versions listed")
}
protoString := strconv.Itoa(int(i.PluginProtocolVersion))
protocolVersion, err := VersionStr(protoString).Parse()
if err != nil {
return fmt.Errorf("invalid plugin protocol version: %q", i.PluginProtocolVersion)
}
protocolConstraint, err := protocolVersion.MinorUpgradeConstraintStr().Parse()
if err != nil {
// This should not fail if the preceding function succeeded.
return fmt.Errorf("invalid plugin protocol version: %q", protocolVersion.String())
}
for _, p := range versionMeta.Protocols {
proPro, err := VersionStr(p).Parse()
if err != nil {
// invalid protocol reported by the registry. Move along.
log.Printf("[WARN] invalid provider protocol version %q found in the registry", versionMeta.Version)
continue
}
// success!
if protocolConstraint.Allows(proPro) {
return nil
}
}
return ErrorNoVersionCompatible
}
// REVIEWER QUESTION (again): this ends up swallowing a bunch of errors from
// checkPluginProtocol. Do they need to be percolated up better, or would
// debug messages would suffice in these situations?
func (i *ProviderInstaller) findPlatformCompatibleVersion(versions []*response.TerraformProviderVersion) (*response.TerraformProviderVersion, error) {
for _, version := range versions {
if err := i.checkPlatformCompatibility(version); err == nil {
return version, nil
}
}
return nil, ErrorNoVersionCompatibleWithPlatform
}
// platformCompatibleVersions returns a list of provider versions that are
// compatible with the requested platform.
func (i *ProviderInstaller) platformCompatibleVersions(versions []*response.TerraformProviderVersion) []*response.TerraformProviderVersion {
var v []*response.TerraformProviderVersion
for _, version := range versions {
if err := i.checkPlatformCompatibility(version); err == nil {
v = append(v, version)
}
}
return v
}
func (i *ProviderInstaller) checkPlatformCompatibility(versionMeta *response.TerraformProviderVersion) error {
if len(versionMeta.Platforms) == 0 {
return fmt.Errorf("no supported provider platforms listed")
}
for _, p := range versionMeta.Platforms {
if p.Arch == i.Arch && p.OS == i.OS {
return nil
}
}
return fmt.Errorf("version %s does not support the requested platform %s_%s", versionMeta.Version, i.OS, i.Arch)
}
// take the list of available versions for a plugin, and filter out those that
// don't fit the constraints.
func allowedVersions(available *response.TerraformProviderVersions, required Constraints) []*response.TerraformProviderVersion {
var allowed []*response.TerraformProviderVersion
for _, v := range available.Versions {
version, err := VersionStr(v.Version).Parse()
if err != nil {
log.Printf("[WARN] invalid version found for %q: %s", available.ID, err)
continue
}
if required.Allows(version) {
allowed = append(allowed, v)
}
}
return allowed
}
func checksumForFile(sums []byte, name string) string {
for _, line := range strings.Split(string(sums), "\n") {
parts := strings.Fields(line)
if len(parts) > 1 && parts[1] == name {
return parts[0]
}
}
return ""
}
func getFile(url string) ([]byte, error) {
resp, err := httpClient.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%s", resp.Status)
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return data, err
}
return data, nil
}
// providerProtocolTooOld is a message sent to the CLI UI if the provider's
// supported protocol versions are too old for the user's version of terraform,
// but an older version of the provider is compatible.
const providerProtocolTooOld = `
[reset][bold][red]Provider %q v%s is not compatible with Terraform %s.[reset][red]
Provider version %s is the earliest compatible version. Select it with
the following version constraint:
version = %q
Terraform checked all of the plugin versions matching the given constraint:
%s
Consult the documentation for this provider for more information on
compatibility between provider and Terraform versions.
`
// providerProtocolTooNew is a message sent to the CLI UI if the provider's
// supported protocol versions are too new for the user's version of terraform,
// and the user could either upgrade terraform or choose an older version of the
// provider
const providerProtocolTooNew = `
[reset][bold][red]Provider %q v%s is not compatible with Terraform %s.[reset][red]
Provider version %s is the latest compatible version. Select it with
the following constraint:
version = %q
Terraform checked all of the plugin versions matching the given constraint:
%s
Consult the documentation for this provider for more information on
compatibility between provider and Terraform versions.
Alternatively, upgrade to the latest version of Terraform for compatibility with newer provider releases.
`