internal/getproviders: A new shared model for provider requirements

We've been using the models from the "moduledeps" package to represent our
provider dependencies everywhere since the idea of provider dependencies
was introduced in Terraform 0.10, but that model is not convenient to use
for any use-case other than the "terraform providers" command that needs
individual-module-level detail.

To make things easier for new codepaths working with the new-style
provider installer, here we introduce a new model type
getproviders.Requirements which is based on the type the new installer was
already taking as its input. We have new methods in the states, configs,
and earlyconfig packages to produce values of this type, and a helper
to merge Requirements together so we can combine config-derived and
state-derived requirements together during installation.

The advantage of this new model over the moduledeps one is that all of
recursive module walking is done up front and we produce a simple, flat
structure that is more convenient for the main use-cases of selecting
providers for installation and then finding providers in the local cache
to use them for other operations.

This new model is _not_ suitable for implementing "terraform providers"
because it does not retain module-specific requirement details. Therefore
we will likely keep using moduledeps for "terraform providers" for now,
and then possibly at a later time consider specializing the moduledeps
logic for only what "terraform providers" needs, because it seems to be
the only use-case that needs to retain that level of detail.

configs/config.go

@@ -7,6 +7,7 @@ import (
 	version "github.com/hashicorp/go-version"
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/internal/getproviders"
 )
 
 // A Config is a node in the tree of modules within a configuration.
@@ -163,6 +164,75 @@ func (c *Config) DescendentForInstance(path addrs.ModuleInstance) *Config {
 	return current
 }
 
+// ProviderRequirements searches the full tree of modules under the receiver
+// for both explicit and implicit dependencies on providers.
+//
+// The result is a full manifest of all of the providers that must be available
+// in order to work with the receiving configuration.
+//
+// If the returned diagnostics includes errors then the resulting Requirements
+// may be incomplete.
+func (c *Config) ProviderRequirements() (getproviders.Requirements, hcl.Diagnostics) {
+	reqs := make(getproviders.Requirements)
+	diags := c.addProviderRequirements(reqs)
+	return reqs, diags
+}
+
+// addProviderRequirements is the main part of the ProviderRequirements
+// implementation, gradually mutating a shared requirements object to
+// eventually return.
+func (c *Config) addProviderRequirements(reqs getproviders.Requirements) hcl.Diagnostics {
+	var diags hcl.Diagnostics
+
+	// First we'll deal with the requirements directly in _our_ module...
+	for _, providerReqs := range c.Module.ProviderRequirements {
+		fqn := providerReqs.Type
+		if _, ok := reqs[fqn]; !ok {
+			// We'll at least have an unconstrained dependency then, but might
+			// add to this in the loop below.
+			reqs[fqn] = nil
+		}
+		for _, constraintsSrc := range providerReqs.VersionConstraints {
+			// The model of version constraints in this package is still the
+			// old one using a different upstream module to represent versions,
+			// so we'll need to shim that out here for now. We assume this
+			// will always succeed because these constraints already succeeded
+			// parsing with the other constraint parser, which uses the same
+			// syntax.
+			constraints := getproviders.MustParseVersionConstraints(constraintsSrc.Required.String())
+			reqs[fqn] = append(reqs[fqn], constraints...)
+		}
+	}
+	// Each resource in the configuration creates an *implicit* provider
+	// dependency, though we'll only record it if there isn't already
+	// an explicit dependency on the same provider.
+	for _, rc := range c.Module.ManagedResources {
+		fqn := rc.Provider
+		if _, exists := reqs[fqn]; exists {
+			// Explicit dependency already present
+			continue
+		}
+		reqs[fqn] = nil
+	}
+	for _, rc := range c.Module.DataResources {
+		fqn := rc.Provider
+		if _, exists := reqs[fqn]; exists {
+			// Explicit dependency already present
+			continue
+		}
+		reqs[fqn] = nil
+	}
+
+	// ...and now we'll recursively visit all of the child modules to merge
+	// in their requirements too.
+	for _, childConfig := range c.Children {
+		moreDiags := childConfig.addProviderRequirements(reqs)
+		diags = append(diags, moreDiags...)
+	}
+
+	return diags
+}
+
 // ProviderTypes returns the FQNs of each distinct provider type referenced
 // in the receiving configuration.
 //

configs/config_test.go

@@ -4,8 +4,11 @@ import (
 	"testing"
 
 	"github.com/go-test/deep"
+	"github.com/google/go-cmp/cmp"
 
+	svchost "github.com/hashicorp/terraform-svchost"
 	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/internal/getproviders"
 )
 
 func TestConfigProviderTypes(t *testing.T) {
@@ -113,6 +116,42 @@ func TestConfigResolveAbsProviderAddr(t *testing.T) {
 	})
 }
 
+func TestConfigProviderRequirements(t *testing.T) {
+	cfg, diags := testNestedModuleConfigFromDir(t, "testdata/provider-reqs")
+	assertNoDiagnostics(t, diags)
+
+	tlsProvider := addrs.NewProvider(
+		addrs.DefaultRegistryHost,
+		"hashicorp", "tls",
+	)
+	happycloudProvider := addrs.NewProvider(
+		svchost.Hostname("tf.example.com"),
+		"awesomecorp", "happycloud",
+	)
+	// FIXME: these two are legacy ones for now because the config loader
+	// isn't using default configurations fully yet.
+	// Once that changes, these should be default-shaped ones like tlsProvider
+	// above.
+	nullProvider := addrs.NewLegacyProvider("null")
+	randomProvider := addrs.NewLegacyProvider("random")
+	impliedProvider := addrs.NewLegacyProvider("implied")
+
+	got, diags := cfg.ProviderRequirements()
+	assertNoDiagnostics(t, diags)
+	want := getproviders.Requirements{
+		// the nullProvider constraints from the two modules are merged
+		nullProvider:       getproviders.MustParseVersionConstraints("~> 2.0.0, 2.0.1"),
+		randomProvider:     getproviders.MustParseVersionConstraints("~> 1.2.0"),
+		tlsProvider:        getproviders.MustParseVersionConstraints("~> 3.0"),
+		impliedProvider:    nil,
+		happycloudProvider: nil,
+	}
+
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("wrong result\n%s", diff)
+	}
+}
+
 func TestProviderForConfigAddr(t *testing.T) {
 	cfg, diags := testModuleConfigFromDir("testdata/valid-modules/providers-fqns")
 	assertNoDiagnostics(t, diags)

configs/testdata/provider-reqs/child/provider-reqs-child.tf

@@ -0,0 +1,11 @@
+terraform {
+  required_providers {
+    cloud = {
+      source = "tf.example.com/awesomecorp/happycloud"
+    }
+    null = {
+      # This should merge with the null provider constraint in the root module
+      version = "2.0.1"
+    }
+  }
+}

configs/testdata/provider-reqs/provider-reqs-root.tf

@@ -0,0 +1,21 @@
+terraform {
+  required_providers {
+    null = "~> 2.0.0"
+    random = {
+      version = "~> 1.2.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = "~> 3.0"
+    }
+  }
+}
+
+# There is no provider in required_providers called "implied", so this
+# implicitly declares a dependency on "hashicorp/implied".
+resource "implied_foo" "bar" {
+}
+
+module "child" {
+  source = "./child"
+}

internal/earlyconfig/config.go

@@ -7,6 +7,7 @@ import (
 	version "github.com/hashicorp/go-version"
 	"github.com/hashicorp/terraform-config-inspect/tfconfig"
 	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/internal/getproviders"
 	"github.com/hashicorp/terraform/moduledeps"
 	"github.com/hashicorp/terraform/plugin/discovery"
 	"github.com/hashicorp/terraform/tfdiags"
@@ -68,8 +69,79 @@ type Config struct {
 	Version *version.Version
 }
 
-// ProviderDependencies returns the provider dependencies for the recieving
-// config, including all of its descendent modules.
+// ProviderRequirements searches the full tree of modules under the receiver
+// for both explicit and implicit dependencies on providers.
+//
+// The result is a full manifest of all of the providers that must be available
+// in order to work with the receiving configuration.
+//
+// If the returned diagnostics includes errors then the resulting Requirements
+// may be incomplete.
+func (c *Config) ProviderRequirements() (getproviders.Requirements, tfdiags.Diagnostics) {
+	reqs := make(getproviders.Requirements)
+	diags := c.addProviderRequirements(reqs)
+	return reqs, diags
+}
+
+// addProviderRequirements is the main part of the ProviderRequirements
+// implementation, gradually mutating a shared requirements object to
+// eventually return.
+func (c *Config) addProviderRequirements(reqs getproviders.Requirements) tfdiags.Diagnostics {
+	var diags tfdiags.Diagnostics
+
+	// First we'll deal with the requirements directly in _our_ module...
+	for localName, providerReqs := range c.Module.RequiredProviders {
+		var fqn addrs.Provider
+		if source := providerReqs.Source; source != "" {
+			addr, moreDiags := addrs.ParseProviderSourceString(source)
+			if moreDiags.HasErrors() {
+				diags = diags.Append(tfdiags.Sourceless(
+					tfdiags.Error,
+					"Invalid provider source address",
+					fmt.Sprintf("Invalid source %q for provider %q in %s", source, localName, c.Path),
+				))
+				continue
+			}
+			fqn = addr
+		}
+		if fqn.IsZero() {
+			fqn = addrs.NewDefaultProvider(localName)
+		}
+		if _, ok := reqs[fqn]; !ok {
+			// We'll at least have an unconstrained dependency then, but might
+			// add to this in the loop below.
+			reqs[fqn] = nil
+		}
+		for _, constraintsStr := range providerReqs.VersionConstraints {
+			if constraintsStr != "" {
+				constraints, err := getproviders.ParseVersionConstraints(constraintsStr)
+				if err != nil {
+					diags = diags.Append(tfdiags.Sourceless(
+						tfdiags.Error,
+						"Invalid provider version constraint",
+						fmt.Sprintf("Provider %q in %s has invalid version constraint %q: %s.", localName, c.Path, constraintsStr, err),
+					))
+					continue
+				}
+				reqs[fqn] = append(reqs[fqn], constraints...)
+			}
+		}
+	}
+
+	// ...and now we'll recursively visit all of the child modules to merge
+	// in their requirements too.
+	for _, childConfig := range c.Children {
+		moreDiags := childConfig.addProviderRequirements(reqs)
+		diags = diags.Append(moreDiags)
+	}
+
+	return diags
+}
+
+// ProviderDependencies is a deprecated variant of ProviderRequirements which
+// uses the moduledeps models for representation. This is preserved to allow
+// a gradual transition over to ProviderRequirements, but note that its
+// support for fully-qualified provider addresses has some idiosyncracies.
 func (c *Config) ProviderDependencies() (*moduledeps.Module, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 

internal/earlyconfig/config_test.go

@@ -0,0 +1,84 @@
+package earlyconfig
+
+import (
+	"log"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	version "github.com/hashicorp/go-version"
+	"github.com/hashicorp/terraform-config-inspect/tfconfig"
+	svchost "github.com/hashicorp/terraform-svchost"
+	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/internal/getproviders"
+	"github.com/hashicorp/terraform/tfdiags"
+)
+
+func TestConfigProviderRequirements(t *testing.T) {
+	cfg := testConfig(t, "testdata/provider-reqs")
+
+	impliedProvider := addrs.NewProvider(
+		addrs.DefaultRegistryHost,
+		"hashicorp", "implied",
+	)
+	nullProvider := addrs.NewProvider(
+		addrs.DefaultRegistryHost,
+		"hashicorp", "null",
+	)
+	randomProvider := addrs.NewProvider(
+		addrs.DefaultRegistryHost,
+		"hashicorp", "random",
+	)
+	tlsProvider := addrs.NewProvider(
+		addrs.DefaultRegistryHost,
+		"hashicorp", "tls",
+	)
+	happycloudProvider := addrs.NewProvider(
+		svchost.Hostname("tf.example.com"),
+		"awesomecorp", "happycloud",
+	)
+
+	got, diags := cfg.ProviderRequirements()
+	if diags.HasErrors() {
+		t.Fatalf("unexpected diagnostics: %s", diags.Err().Error())
+	}
+	want := getproviders.Requirements{
+		// the nullProvider constraints from the two modules are merged
+		nullProvider:       getproviders.MustParseVersionConstraints("~> 2.0.0, 2.0.1"),
+		randomProvider:     getproviders.MustParseVersionConstraints("~> 1.2.0"),
+		tlsProvider:        getproviders.MustParseVersionConstraints("~> 3.0"),
+		impliedProvider:    nil,
+		happycloudProvider: nil,
+	}
+
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("wrong result\n%s", diff)
+	}
+}
+
+func testConfig(t *testing.T, baseDir string) *Config {
+	rootMod, diags := LoadModule(baseDir)
+	if diags.HasErrors() {
+		t.Fatalf("unexpected diagnostics: %s", diags.Err().Error())
+	}
+
+	cfg, diags := BuildConfig(rootMod, ModuleWalkerFunc(testModuleWalkerFunc))
+	if diags.HasErrors() {
+		t.Fatalf("unexpected diagnostics: %s", diags.Err().Error())
+	}
+
+	return cfg
+}
+
+// testModuleWalkerFunc is a simple implementation of ModuleWalkerFunc that
+// only understands how to resolve relative filesystem paths, using source
+// location information from the call.
+func testModuleWalkerFunc(req *ModuleRequest) (*tfconfig.Module, *version.Version, tfdiags.Diagnostics) {
+	callFilename := req.CallPos.Filename
+	sourcePath := req.SourceAddr
+	finalPath := filepath.Join(filepath.Dir(callFilename), sourcePath)
+	log.Printf("[TRACE] %s in %s -> %s", sourcePath, callFilename, finalPath)
+
+	newMod, diags := LoadModule(finalPath)
+	return newMod, version.Must(version.NewVersion("0.0.0")), diags
+}

internal/earlyconfig/testdata/provider-reqs/child/provider-reqs-child.tf

@@ -0,0 +1,11 @@
+terraform {
+  required_providers {
+    cloud = {
+      source = "tf.example.com/awesomecorp/happycloud"
+    }
+    null = {
+      # This should merge with the null provider constraint in the root module
+      version = "2.0.1"
+    }
+  }
+}

internal/earlyconfig/testdata/provider-reqs/provider-reqs-root.tf

@@ -0,0 +1,21 @@
+terraform {
+  required_providers {
+    null = "~> 2.0.0"
+    random = {
+      version = "~> 1.2.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = "~> 3.0"
+    }
+  }
+}
+
+# There is no provider in required_providers called "implied", so this
+# implicitly declares a dependency on "hashicorp/implied".
+resource "implied_foo" "bar" {
+}
+
+module "child" {
+  source = "./child"
+}

internal/getproviders/types.go

@@ -29,18 +29,72 @@ type VersionSet = versions.Set
 // define the membership of a VersionSet by exclusion.
 type VersionConstraints = constraints.IntersectionSpec
 
+// Requirements gathers together requirements for many different providers
+// into a single data structure, as a convenient way to represent the full
+// set of requirements for a particular configuration or state or both.
+//
+// If an entry in a Requirements has a zero-length VersionConstraints then
+// that indicates that the provider is required but that any version is
+// acceptable. That's different than a provider being absent from the map
+// altogether, which means that it is not required at all.
+type Requirements map[addrs.Provider]VersionConstraints
+
+// Merge takes the requirements in the receiever and the requirements in the
+// other given value and produces a new set of requirements that combines
+// all of the requirements of both.
+//
+// The resulting requirements will permit only selections that both of the
+// source requirements would've allowed.
+func (r Requirements) Merge(other Requirements) Requirements {
+	ret := make(Requirements)
+	for addr, constraints := range r {
+		ret[addr] = constraints
+	}
+	for addr, constraints := range other {
+		ret[addr] = append(ret[addr], constraints...)
+	}
+	return ret
+}
+
+// Selections gathers together version selections for many different providers.
+//
+// This is the result of provider installation: a specific version selected
+// for each provider given in the requested Requirements, selected based on
+// the given version constraints.
+type Selections map[addrs.Provider]Version
+
 // ParseVersion parses a "semver"-style version string into a Version value,
 // which is the version syntax we use for provider versions.
 func ParseVersion(str string) (Version, error) {
 	return versions.ParseVersion(str)
 }
 
+// MustParseVersion is a variant of ParseVersion that panics if it encounters
+// an error while parsing.
+func MustParseVersion(str string) Version {
+	ret, err := ParseVersion(str)
+	if err != nil {
+		panic(err)
+	}
+	return ret
+}
+
 // ParseVersionConstraints parses a "Ruby-like" version constraint string
 // into a VersionConstraints value.
 func ParseVersionConstraints(str string) (VersionConstraints, error) {
 	return constraints.ParseRubyStyleMulti(str)
 }
 
+// MustParseVersionConstraints is a variant of ParseVersionConstraints that
+// panics if it encounters an error while parsing.
+func MustParseVersionConstraints(str string) VersionConstraints {
+	ret, err := ParseVersionConstraints(str)
+	if err != nil {
+		panic(err)
+	}
+	return ret
+}
+
 // Platform represents a target platform that a provider is or might be
 // available for.
 type Platform struct {

internal/providercache/installer.go

@@ -88,7 +88,7 @@ func (i *Installer) SetGlobalCacheDir(cacheDir *Dir) {
 // failures then those notifications will be redundant with the ones included
 // in the final returned error value so callers should show either one or the
 // other, and not both.
-func (i *Installer) EnsureProviderVersions(ctx context.Context, reqs map[addrs.Provider]getproviders.VersionConstraints, mode InstallMode) (map[addrs.Provider]getproviders.Version, error) {
+func (i *Installer) EnsureProviderVersions(ctx context.Context, reqs getproviders.Requirements, mode InstallMode) (getproviders.Selections, error) {
 	// FIXME: Currently the context isn't actually propagated into all of the
 	// other functions we call here, because they are not context-aware.
 	// Anything that could be making network requests here should take a

states/state.go

@@ -6,6 +6,7 @@ import (
 	"github.com/zclconf/go-cty/cty"
 
 	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/internal/getproviders"
 )
 
 // State is the top-level type of a Terraform state.
@@ -223,6 +224,22 @@ func (s *State) ProviderAddrs() []addrs.AbsProviderConfig {
 	return ret
 }
 
+// ProviderRequirements returns a description of all of the providers that
+// are required to work with the receiving state.
+//
+// Because the state does not track specific version information for providers,
+// the requirements returned by this method will always be unconstrained.
+// The result should usually be merged with a Requirements derived from the
+// current configuration in order to apply some constraints.
+func (s *State) ProviderRequirements() getproviders.Requirements {
+	configAddrs := s.ProviderAddrs()
+	ret := make(getproviders.Requirements, len(configAddrs))
+	for _, configAddr := range configAddrs {
+		ret[configAddr.Provider] = nil // unconstrained dependency
+	}
+	return ret
+}
+
 // PruneResourceHusks is a specialized method that will remove any Resource
 // objects that do not contain any instances, even if they have an EachMode.
 //