command: terraform providers mirror

This new command is intended to make it easy to create or update a mirror
directory containing suitable providers for the current configuration,
producing a layout that is appropriate both for a filesystem mirror or,
if copied into the document root of an HTTP server, a network mirror.

This initial version is not customizable aside from being able to select
multiple platforms to install packages for.

Future iterations of this could include commands to turn the JSON index
generation on and off, or to instruct it to produce the unpacked directory
layout instead of the packed directory layout as it currently does. Both
of those options would make the generated directory unsuitable to be
a network mirror, but it would still work as a filesystem mirror.

In the long run this will hopefully form part of a replacement workflow to
terraform-bundle as a way to put copies of providers somewhere so we don't
need to re-download them every time, but some other changes will be needed
outside of just this command before that'd be true, such as adding support
for network and/or filesystem mirrors in Terraform Enterprise.

command/e2etest/providers_mirror_test.go

@@ -0,0 +1,76 @@
+package e2etest
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/hashicorp/terraform/e2e"
+)
+
+// The tests in this file are for the "terraform providers mirror" command,
+// which is tested in an e2etest mode rather than a unit test mode because it
+// interacts directly with Terraform Registry and the full details of that are
+// tricky to mock. Such a mock is _possible_, but we're using e2etest as a
+// compromise for now to keep these tests relatively simple.
+
+func TestTerraformProvidersMirror(t *testing.T) {
+	// This test reaches out to releases.hashicorp.com to download the
+	// template and null providers, so it can only run if network access is
+	// allowed.
+	skipIfCannotAccessNetwork(t)
+
+	outputDir, err := ioutil.TempDir("", "terraform-e2etest-providers-mirror")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(outputDir)
+	t.Logf("creating mirror directory in %s", outputDir)
+
+	fixturePath := filepath.Join("testdata", "terraform-providers-mirror")
+	tf := e2e.NewBinary(terraformBin, fixturePath)
+	defer tf.Close()
+
+	stdout, stderr, err := tf.Run("providers", "mirror", "-platform=linux_amd64", "-platform=windows_386", outputDir)
+	if err != nil {
+		t.Fatalf("unexpected error: %s\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr)
+	}
+
+	// The test fixture includes exact version constraints for the two
+	// providers it depends on so that the following should remain stable.
+	// In the (unlikely) event that these particular versions of these
+	// providers are removed from the registry, this test will start to fail.
+	want := []string{
+		"registry.terraform.io/hashicorp/null/2.1.0.json",
+		"registry.terraform.io/hashicorp/null/index.json",
+		"registry.terraform.io/hashicorp/null/terraform-provider-null_2.1.0_linux_amd64.zip",
+		"registry.terraform.io/hashicorp/null/terraform-provider-null_2.1.0_windows_386.zip",
+		"registry.terraform.io/hashicorp/template/2.1.1.json",
+		"registry.terraform.io/hashicorp/template/index.json",
+		"registry.terraform.io/hashicorp/template/terraform-provider-template_2.1.1_linux_amd64.zip",
+		"registry.terraform.io/hashicorp/template/terraform-provider-template_2.1.1_windows_386.zip",
+	}
+	var got []string
+	err = filepath.Walk(outputDir, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil // we only care about leaf files for this test
+		}
+		relPath, err := filepath.Rel(outputDir, path)
+		if err != nil {
+			return err
+		}
+		got = append(got, filepath.ToSlash(relPath))
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	sort.Strings(got)
+
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("unexpected files in result\n%s", diff)
+	}
+}

command/e2etest/testdata/terraform-providers-mirror/terraform-providers-mirror.tf

@@ -0,0 +1,7 @@
+terraform {
+  required_providers {
+    template  = { version = "2.1.1" }
+    null      = { source = "hashicorp/null", version = "2.1.0" }
+    terraform = { source = "terraform.io/builtin/terraform" }
+  }
+}

command/providers_mirror.go

@@ -0,0 +1,356 @@
+package command
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"path/filepath"
+
+	"github.com/apparentlymart/go-versions/versions"
+	"github.com/hashicorp/go-getter"
+	"github.com/hashicorp/terraform/httpclient"
+	"github.com/hashicorp/terraform/internal/getproviders"
+	"github.com/hashicorp/terraform/tfdiags"
+)
+
+// ProvidersMirrorCommand is a Command implementation that implements the
+// "terraform providers mirror" command, which populates a directory with
+// local copies of provider plugins needed by the current configuration so
+// that the mirror can be used to work offline, or similar.
+type ProvidersMirrorCommand struct {
+	Meta
+}
+
+func (c *ProvidersMirrorCommand) Synopsis() string {
+	return "Mirrors the provider plugins needed for the current configuration"
+}
+
+func (c *ProvidersMirrorCommand) Run(args []string) int {
+	args = c.Meta.process(args)
+	cmdFlags := c.Meta.defaultFlagSet("providers mirror")
+	var optPlatforms FlagStringSlice
+	cmdFlags.Var(&optPlatforms, "platform", "target platform")
+	cmdFlags.Usage = func() { c.Ui.Error(c.Help()) }
+	if err := cmdFlags.Parse(args); err != nil {
+		c.Ui.Error(fmt.Sprintf("Error parsing command-line flags: %s\n", err.Error()))
+		return 1
+	}
+
+	var diags tfdiags.Diagnostics
+
+	args = cmdFlags.Args()
+	if len(args) != 1 {
+		diags = diags.Append(tfdiags.Sourceless(
+			tfdiags.Error,
+			"No output directory specified",
+			"The providers mirror command requires an output directory as a command-line argument.",
+		))
+		c.showDiagnostics(diags)
+		return 1
+	}
+	outputDir := args[0]
+
+	var platforms []getproviders.Platform
+	if len(optPlatforms) == 0 {
+		platforms = []getproviders.Platform{getproviders.CurrentPlatform}
+	} else {
+		platforms = make([]getproviders.Platform, 0, len(optPlatforms))
+		for _, platformStr := range optPlatforms {
+			platform, err := getproviders.ParsePlatform(platformStr)
+			if err != nil {
+				diags = diags.Append(tfdiags.Sourceless(
+					tfdiags.Error,
+					"Invalid target platform",
+					fmt.Sprintf("The string %q given in the -platform option is not a valid target platform: %s.", platformStr, err),
+				))
+				continue
+			}
+			platforms = append(platforms, platform)
+		}
+	}
+
+	config, confDiags := c.loadConfig(".")
+	diags = diags.Append(confDiags)
+	reqs, moreDiags := config.ProviderRequirements()
+	diags = diags.Append(moreDiags)
+
+	// If we have any error diagnostics already then we won't proceed further.
+	if diags.HasErrors() {
+		c.showDiagnostics(diags)
+		return 1
+	}
+
+	// Unlike other commands, this command always consults the origin registry
+	// for every provider so that it can be used to update a local mirror
+	// directory without needing to first disable that local mirror
+	// in the CLI configuration.
+	source := getproviders.NewMemoizeSource(
+		getproviders.NewRegistrySource(c.Services),
+	)
+
+	// Providers from registries always use HTTP, so we don't need the full
+	// generality of go-getter but it's still handy to use the HTTP getter
+	// as an easy way to download over HTTP into a file on disk.
+	httpGetter := getter.HttpGetter{
+		Client: httpclient.New(),
+		Netrc:  true,
+	}
+
+	// The following logic is similar to that used by the provider installer
+	// in package providercache, but different in a few ways:
+	// - It produces the packed directory layout rather than the unpacked
+	//   layout we require in provider cache directories.
+	// - It generates JSON index files that can be read by the
+	//   getproviders.HTTPMirrorSource installation method if the result were
+	//   copied into the docroot of an HTTP server.
+	// - It can mirror packages for potentially many different target platforms,
+	//   so that we can construct a multi-platform mirror regardless of which
+	//   platform we run this command on.
+	// - It ignores what's already present and just always downloads everything
+	//   that the configuration requires. This is a command intended to be run
+	//   infrequently to update a mirror, so it doesn't need to optimize away
+	//   fetches of packages that might already be present.
+
+	for provider, constraints := range reqs {
+		if provider.IsBuiltIn() {
+			c.Ui.Output(fmt.Sprintf("- Skipping %s because it is built in to Terraform CLI", provider.ForDisplay()))
+			continue
+		}
+		constraintsStr := getproviders.VersionConstraintsString(constraints)
+		c.Ui.Output(fmt.Sprintf("- Mirroring %s...", provider.ForDisplay()))
+		// First we'll look for the latest version that matches the given
+		// constraint, which we'll then try to mirror for each target platform.
+		acceptable := versions.MeetingConstraints(constraints)
+		avail, err := source.AvailableVersions(provider)
+		candidates := avail.Filter(acceptable)
+		if err == nil && len(candidates) == 0 {
+			err = fmt.Errorf("no releases match the given constraints %s", constraintsStr)
+		}
+		if err != nil {
+			diags = diags.Append(tfdiags.Sourceless(
+				tfdiags.Error,
+				"Provider not available",
+				fmt.Sprintf("Failed to download %s from its origin registry: %s.", provider.String(), err),
+			))
+			continue
+		}
+		selected := candidates.Newest()
+		if len(constraintsStr) > 0 {
+			c.Ui.Output(fmt.Sprintf("  - Selected v%s to meet constraints %s", selected.String(), constraintsStr))
+		} else {
+			c.Ui.Output(fmt.Sprintf("  - Selected v%s with no constraints", selected.String()))
+		}
+		for _, platform := range platforms {
+			c.Ui.Output(fmt.Sprintf("  - Downloading package for %s...", platform.String()))
+			meta, err := source.PackageMeta(provider, selected, platform)
+			if err != nil {
+				diags = diags.Append(tfdiags.Sourceless(
+					tfdiags.Error,
+					"Provider release not available",
+					fmt.Sprintf("Failed to download %s v%s for %s: %s.", provider.String(), selected.String(), platform.String(), err),
+				))
+				continue
+			}
+			urlStr, ok := meta.Location.(getproviders.PackageHTTPURL)
+			if !ok {
+				// We don't expect to get non-HTTP locations here because we're
+				// using the registry source, so this seems like a bug in the
+				// registry source.
+				diags = diags.Append(tfdiags.Sourceless(
+					tfdiags.Error,
+					"Provider release not available",
+					fmt.Sprintf("Failed to download %s v%s for %s: Terraform's provider registry client returned unexpected location type %T. This is a bug in Terraform.", provider.String(), selected.String(), platform.String(), meta.Location),
+				))
+				continue
+			}
+			urlObj, err := url.Parse(string(urlStr))
+			if err != nil {
+				// We don't expect to get non-HTTP locations here because we're
+				// using the registry source, so this seems like a bug in the
+				// registry source.
+				diags = diags.Append(tfdiags.Sourceless(
+					tfdiags.Error,
+					"Invalid URL for provider release",
+					fmt.Sprintf("The origin registry for %s returned an invalid URL for v%s on %s: %s.", provider.String(), selected.String(), platform.String(), err),
+				))
+				continue
+			}
+			// targetPath is the path where we ultimately want to place the
+			// downloaded archive, but we'll place it initially at stagingPath
+			// so we can verify its checksums and signatures before making
+			// it discoverable to mirror clients. (stagingPath intentionally
+			// does not follow the filesystem mirror file naming convention.)
+			targetPath := meta.PackedFilePath(outputDir)
+			stagingPath := filepath.Join(filepath.Dir(targetPath), "."+filepath.Base(targetPath))
+			err = httpGetter.GetFile(stagingPath, urlObj)
+			if err != nil {
+				diags = diags.Append(tfdiags.Sourceless(
+					tfdiags.Error,
+					"Cannot download provider release",
+					fmt.Sprintf("Failed to download %s v%s for %s: %s.", provider.String(), selected.String(), platform.String(), err),
+				))
+				continue
+			}
+			if meta.Authentication != nil {
+				result, err := meta.Authentication.AuthenticatePackage(getproviders.PackageLocalArchive(stagingPath))
+				if err != nil {
+					diags = diags.Append(tfdiags.Sourceless(
+						tfdiags.Error,
+						"Invalid provider package",
+						fmt.Sprintf("Failed to authenticate %s v%s for %s: %s.", provider.String(), selected.String(), platform.String(), err),
+					))
+					continue
+				}
+				c.Ui.Output(fmt.Sprintf("  - Package authenticated: %s", result))
+			}
+			os.Remove(targetPath) // okay if it fails because we're going to try to rename over it next anyway
+			err = os.Rename(stagingPath, targetPath)
+			if err != nil {
+				diags = diags.Append(tfdiags.Sourceless(
+					tfdiags.Error,
+					"Cannot download provider release",
+					fmt.Sprintf("Failed to place %s package into mirror directory: %s.", provider.String(), err),
+				))
+				continue
+			}
+		}
+	}
+
+	// Now we'll generate or update the JSON index files in the directory.
+	// We do this by scanning the directory to see what is present, rather than
+	// by relying on the selections we made above, because we want to still
+	// include in the indices any packages that were already present and
+	// not affected by the changes we just made.
+	available, err := getproviders.SearchLocalDirectory(outputDir)
+	if err != nil {
+		diags = diags.Append(tfdiags.Sourceless(
+			tfdiags.Error,
+			"Failed to update indexes",
+			fmt.Sprintf("Could not scan the output directory to get package metadata for the JSON indexes: %s.", err),
+		))
+		available = nil // the following loop will be a no-op
+	}
+	for provider, metas := range available {
+		if len(metas) == 0 {
+			continue // should never happen, but we'll be resilient
+		}
+		// The index files live in the same directory as the package files,
+		// so to figure that out without duplicating the path-building logic
+		// we'll ask the getproviders package to build an archive filename
+		// for a fictitious package and then use the directory portion of it.
+		indexDir := filepath.Dir(getproviders.PackedFilePathForPackage(
+			outputDir, provider, versions.Unspecified, getproviders.CurrentPlatform,
+		))
+		indexVersions := map[string]interface{}{}
+		indexArchives := map[getproviders.Version]map[string]interface{}{}
+		for _, meta := range metas {
+			archivePath, ok := meta.Location.(getproviders.PackageLocalArchive)
+			if !ok {
+				// only archive files are eligible to be included in JSON
+				// indices for a network mirror.
+				continue
+			}
+			archiveFilename := filepath.Base(string(archivePath))
+			version := meta.Version
+			platform := meta.TargetPlatform
+			hash, err := meta.Hash()
+			if err != nil {
+				diags = diags.Append(tfdiags.Sourceless(
+					tfdiags.Error,
+					"Failed to update indexes",
+					fmt.Sprintf("Failed to determine a hash value for %s v%s on %s: %s.", provider, version, platform, err),
+				))
+				continue
+			}
+			indexVersions[meta.Version.String()] = map[string]interface{}{}
+			if _, ok := indexArchives[version]; !ok {
+				indexArchives[version] = map[string]interface{}{}
+			}
+			indexArchives[version][platform.String()] = map[string]interface{}{
+				"url":    archiveFilename, // a relative URL from the index file's URL
+				"hashes": []string{hash},  // an array to allow for additional hash formats in future
+			}
+		}
+		mainIndex := map[string]interface{}{
+			"versions": indexVersions,
+		}
+		mainIndexJSON, err := json.MarshalIndent(mainIndex, "", "  ")
+		if err != nil {
+			// Should never happen because the input here is entirely under
+			// our control.
+			panic(fmt.Sprintf("failed to encode main index: %s", err))
+		}
+		// TODO: Ideally we would do these updates as atomic swap operations by
+		// creating a new file and then renaming it over the old one, in case
+		// this directory is the docroot of a live mirror. An atomic swap
+		// requires platform-specific code though: os.Rename alone can't do it
+		// when running on Windows as of Go 1.13. We should revisit this once
+		// we're supporting network mirrors, to avoid having them briefly
+		// become corrupted during updates.
+		err = ioutil.WriteFile(filepath.Join(indexDir, "index.json"), mainIndexJSON, 0644)
+		if err != nil {
+			diags = diags.Append(tfdiags.Sourceless(
+				tfdiags.Error,
+				"Failed to update indexes",
+				fmt.Sprintf("Failed to write an updated JSON index for %s: %s.", provider, err),
+			))
+		}
+		for version, archiveIndex := range indexArchives {
+			versionIndex := map[string]interface{}{
+				"archives": archiveIndex,
+			}
+			versionIndexJSON, err := json.MarshalIndent(versionIndex, "", "  ")
+			if err != nil {
+				// Should never happen because the input here is entirely under
+				// our control.
+				panic(fmt.Sprintf("failed to encode version index: %s", err))
+			}
+			err = ioutil.WriteFile(filepath.Join(indexDir, version.String()+".json"), versionIndexJSON, 0644)
+			if err != nil {
+				diags = diags.Append(tfdiags.Sourceless(
+					tfdiags.Error,
+					"Failed to update indexes",
+					fmt.Sprintf("Failed to write an updated JSON index for %s v%s: %s.", provider, version, err),
+				))
+			}
+		}
+	}
+
+	c.showDiagnostics(diags)
+	if diags.HasErrors() {
+		return 1
+	}
+	return 0
+}
+
+func (c *ProvidersMirrorCommand) Help() string {
+	return `
+Usage: terraform providers mirror [options] <target-dir>
+
+  Populates a local directory with copies of the provider plugins needed for
+  the current configuration, so that the directory can be used either directly
+  as a filesystem mirror or as the basis for a network mirror and thus obtain
+  those providers without access to their origin registries in future.
+
+  The mirror directory will contain JSON index files that can be published
+  along with the mirrored packages on a static HTTP file server to produce
+  a network mirror. Those index files will be ignored if the directory is
+  used instead as a local filesystem mirror.
+
+Options:
+
+  -platform=os_arch  Choose which target platform to build a mirror for.
+                     By default Terraform will obtain plugin packages
+                     suitable for the platform where you run this command.
+                     Use this flag multiple times to include packages for
+                     multiple target systems.
+
+                     Target names consist of an operating system and a CPU
+                     architecture. For example, "linux_amd64" selects the
+                     Linux operating system running on an AMD64 or x86_64
+                     CPU. Each provider is available only for a limited
+                     set of target platforms.
+`
+}

commands.go

@@ -218,6 +218,12 @@ func initCommands(config *cliconfig.Config, services *disco.Disco, providerSrc g
 			}, nil
 		},
 
+		"providers mirror": func() (cli.Command, error) {
+			return &command.ProvidersMirrorCommand{
+				Meta: meta,
+			}, nil
+		},
+
 		"providers schema": func() (cli.Command, error) {
 			return &command.ProvidersSchemaCommand{
 				Meta: meta,

website/docs/commands/providers/mirror.html.md

@@ -0,0 +1,59 @@
+---
+layout: "commands-providers"
+page_title: "Command: providers mirror"
+sidebar_current: "docs-commands-providers-mirror"
+description: |-
+  The `terraform providers mirror` command downloads the providers required
+  for the current configuration and copies them into a directory in the local
+  filesystem.
+---
+
+# Command: terraform providers mirror
+
+The `terraform providers mirror` command downloads the providers required
+for the current configuration and copies them into a directory in the local
+filesystem.
+
+In normal use, `terraform init` will automatically download needed providers
+from provider registries as part of initializing the current working directory.
+Sometimes Terraform is running in an environment where that isn't possible,
+such as on an isolated network without access to the Terraform Registry. In
+that case,
+[explicit installation method configuration](../cli-config.html#explicit-installation-method-configuration)
+allows you to configure Terraform, when running on a particular system, to
+consult only a local filesystem directory where you've created a local mirror
+of the necessary plugins, and to skip accessing the upstream registry at all.
+
+The `terraform providers mirror` command can automatically populate a directory
+that will be used as a local filesystem mirror in the provider installation
+configuration.
+
+-> `terraform providers mirror` is available only in Terraform v0.13 or later.
+
+## Usage
+
+Usage: `terraform providers mirror [options] <target-dir>`
+
+A single target directory is required. Terraform will create under that
+directory the path structure that is expected for filesystem-based provider
+plugin mirrors, populating it both with `.zip` files containing the plugins
+themselves and `.json` index files that describe what is available in the
+directory.
+
+This command supports the following additional option:
+
+* `-platform=OS_ARCH` - Choose which target platform to build a mirror for.
+  By default Terraform will obtain plugin packages suitable for the platform
+  where you run this command. Use this flag multiple times to include packages
+  for multiple target systems.
+
+  Target platform names names consist of an operating system and a CPU
+  architecture. For example, `linux_amd64` selects the Linux operating system
+  running on an AMD64 or x86_64 CPU.
+
+You can run `terraform providers mirror` again on an existing mirror directory
+to update it with new packages. For example, you can add packages for a new
+target platform by re-running the command with the desired new `-platform=...`
+option, and it will place the packages for that new platform without removing
+packages you previously downloaded, merging the resulting set of packages
+together to update the JSON index files.