tools/terraform-bundle: refactor to use new provider installer and provider directory layouts (#24629)

* tools/terraform-bundle: refactor to use new provider installer and provider directory layouts terraform-bundle now supports a "source" attribute for providers, uses the new provider installer, and the archive it creates preserves the new (required) directory hierarchy for providers, under a "plugins" directory. This is a breaking change in many ways: source is required for any non-HashiCorp provider, locally-installed providers must be given a source (can be arbitrary, see docs) and placed in the expected directory hierarchy, and the unzipped archive is no longer flat; there is a new "plugins" directory created with providers in the new directory layout. This PR also extends the existing test to check the contents of the zip file. TODO: Re-enable e2e tests (currently suppressed with a t.Skip) This commit includes an update to our travis configuration, so the terraform-bundle e2e tests run. It also turns off the e2e tests, which will fail until we have a terraform 0.13.* release under releases.hashicorp.com. We decided it was better to merge this now instead of waiting when we started seeing issues opened from users who built terraform-bundle from 0.13 and found it didn't work with 0.12 - better that they get an immediate error message from the binary directing them to build from the appropriate release.
2020-04-21 17:09:29 -04:00 · 2020-04-21 17:09:29 -04:00 · a43f141f9e
parent 1750994af1
commit a43f141f9e
13 changed files with 501 additions and 220 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -81,10 +81,7 @@ jobs:
      - run:
          name: Run Go E2E Tests
          command: |
-            PACKAGE_NAMES=$(go list ./... | circleci tests split --split-by=timings --timings-type=classname)
-            echo "Running $(echo $PACKAGE_NAMES | wc -w) packages"
-            echo $PACKAGE_NAMES
-            gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- -p 2 -cover -coverprofile=cov_e2e.part ./command/e2etest
+            gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- -p 2 -cover -coverprofile=cov_e2e.part ./command/e2etest ./tools/terraform-bundle/e2etest

      # save coverage report parts
      - persist_to_workspace:
--- a/e2e/e2e.go
+++ b/e2e/e2e.go
@ -270,3 +270,8 @@ func GoBuild(pkgPath, tmpPrefix string) string {

 	return tmpFilename
 }
+
+// WorkDir() returns the binary workdir
+func (b *binary) WorkDir() string {
+	return b.workDir
+}
--- a/internal/providercache/dir.go
+++ b/internal/providercache/dir.go
@ -55,13 +55,13 @@ func NewDir(baseDir string) *Dir {
 	}
 }

-// newDirWithPlatform is a variant of NewDir that allows selecting a specific
+// NewDirWithPlatform is a variant of NewDir that allows selecting a specific
 // target platform, rather than taking the current one where this code is
 // running.
 //
 // This is primarily intended for portable unit testing and not particularly
-// useful in "real" callers.
-func newDirWithPlatform(baseDir string, platform getproviders.Platform) *Dir {
+// useful in "real" callers, with the exception of terraform-bundle.
+func NewDirWithPlatform(baseDir string, platform getproviders.Platform) *Dir {
 	return &Dir{
 		baseDir:        baseDir,
 		targetPlatform: platform,
--- a/internal/providercache/dir_modify_test.go
+++ b/internal/providercache/dir_modify_test.go
@ -84,8 +84,8 @@ func TestLinkFromOtherCache(t *testing.T) {
 		addrs.DefaultRegistryHost, "hashicorp", "null",
 	)

-	srcDir := newDirWithPlatform(srcDirPath, windowsPlatform)
-	tmpDir := newDirWithPlatform(tmpDirPath, windowsPlatform)
+	srcDir := NewDirWithPlatform(srcDirPath, windowsPlatform)
+	tmpDir := NewDirWithPlatform(tmpDirPath, windowsPlatform)

 	// First we'll check our preconditions: srcDir should have only the
 	// null provider version 2.0.0 in it, because we're faking that we're on
--- a/internal/providercache/dir_test.go
+++ b/internal/providercache/dir_test.go
@ -40,7 +40,7 @@ func TestDirReading(t *testing.T) {

 	t.Run("ProviderLatestVersion", func(t *testing.T) {
 		t.Run("exists", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)

 			got := dir.ProviderLatestVersion(nullProvider)
 			want := &CachedProvider{
@ -59,7 +59,7 @@ func TestDirReading(t *testing.T) {
 			}
 		})
 		t.Run("no package for current platform", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)

 			// random provider is only cached for linux_amd64 in our fixtures dir
 			got := dir.ProviderLatestVersion(randomProvider)
@ -70,7 +70,7 @@ func TestDirReading(t *testing.T) {
 			}
 		})
 		t.Run("no versions available at all", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)

 			// nonexist provider is not present in our fixtures dir at all
 			got := dir.ProviderLatestVersion(nonExistProvider)
@ -84,7 +84,7 @@ func TestDirReading(t *testing.T) {

 	t.Run("ProviderVersion", func(t *testing.T) {
 		t.Run("exists", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)

 			got := dir.ProviderVersion(nullProvider, versions.MustParseVersion("2.0.0"))
 			want := &CachedProvider{
@ -100,7 +100,7 @@ func TestDirReading(t *testing.T) {
 			}
 		})
 		t.Run("specified version is not cached", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)

 			// there is no v5.0.0 package in our fixtures dir
 			got := dir.ProviderVersion(nullProvider, versions.MustParseVersion("5.0.0"))
@ -111,7 +111,7 @@ func TestDirReading(t *testing.T) {
 			}
 		})
 		t.Run("no package for current platform", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)

 			// random provider 1.2.0 is only cached for linux_amd64 in our fixtures dir
 			got := dir.ProviderVersion(randomProvider, versions.MustParseVersion("1.2.0"))
@ -122,7 +122,7 @@ func TestDirReading(t *testing.T) {
 			}
 		})
 		t.Run("no versions available at all", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)

 			// nonexist provider is not present in our fixtures dir at all
 			got := dir.ProviderVersion(nonExistProvider, versions.MustParseVersion("1.0.0"))
@ -135,7 +135,7 @@ func TestDirReading(t *testing.T) {
 	})

 	t.Run("AllAvailablePackages", func(t *testing.T) {
-		dir := newDirWithPlatform(testDir, linuxPlatform)
+		dir := NewDirWithPlatform(testDir, linuxPlatform)

 		got := dir.AllAvailablePackages()
 		want := map[addrs.Provider][]CachedProvider{
--- a/tools/terraform-bundle/README.md
+++ b/tools/terraform-bundle/README.md
@ -34,12 +34,8 @@ the rest of this README to be in `PATH`.

 `terraform-bundle` is a repackaging of the module installation functionality
 from Terraform itself, so for best results you should build from the tag
-relating to the version of Terraform you plan to use. There is some slack in
-this requirement due to the fact that the module installation behavior changes
-rarely, but please note that in particular bundles for versions of
-Terraform before v0.12 must be built from a `terraform-bundle` built against
-a Terraform v0.11 tag at the latest, since Terraform v0.12 installs plugins
-in a different way that is not compatible.
+relating to the version of Terraform you plan to use. For example, use the v0.12
+tag to build a version of terraform-bundle compatible with Terraform v0.12*.

 ## Usage

@ -59,19 +55,26 @@ terraform {
 # Define which provider plugins are to be included
 providers {
  # Include the newest "aws" provider version in the 1.0 series.
-  aws = ["~> 1.0"]
+  aws = {
+    versions = ["~> 1.0"]
+  }

  # Include both the newest 1.0 and 2.0 versions of the "google" provider.
  # Each item in these lists allows a distinct version to be added. If the
  # two expressions match different versions then _both_ are included in
  # the bundle archive.
-  google = ["~> 1.0", "~> 2.0"]
+  google = {
+    versions = ["~> 1.0", "~> 2.0"]
+  }

  # Include a custom plugin to the bundle. Will search for the plugin in the
-  # plugins directory, and package it with the bundle archive. Plugin must have
-  # a name of the form: terraform-provider-*, and must be build with the operating
-  # system and architecture that terraform enterprise is running, e.g. linux and amd64
-  customplugin = ["0.1"]
+  # plugins directory and package it with the bundle archive. Plugin must have
+  # a name of the form: terraform-provider-*, and must be built with the operating
+  # system and architecture that terraform enterprise is running, e.g. linux and amd64.
+  customplugin = {
+    versions = ["0.1"]
+    source = "myorg/customplugin"
+  }
 }

 ```
@ -80,10 +83,10 @@ The `terraform` block defines which version of Terraform will be included
 in the bundle. An exact version is required here.

 The `providers` block defines zero or more providers to include in the bundle
-along with core Terraform. Each attribute in this block is a provider name,
-and its value is a list of version constraints. For each given constraint,
-`terraform-bundle` will find the newest available version matching the
-constraint and include it in the bundle.
+along with core Terraform. Each attribute is a provider name, and its value is a
+block with the list of version constraints and (optional) source. For each given
+constraint, `terraform-bundle` will find the newest available version matching
+the constraint and include it in the bundle.

 It is allowed to specify multiple constraints for the same provider, in which
 case multiple versions can be included in the resulting bundle. Each constraint
@ -119,13 +122,44 @@ this composite version number so that bundle archives can be easily
 distinguished from official release archives and from each other when multiple
 bundles contain the same core Terraform version.

-To include custom plugins in the bundle file, create a local directory "./plugins"
-and put all the plugins you want to include there. Optionally, you can use the
-`-plugin-dir` flag to specify a location where to find the plugins. To be recognized
-as a valid plugin, the file must have a name of the form
-`terraform-provider-<NAME>_v<VERSION>`. In
-addition, ensure that the plugin is built using the same operating system and
-architecture used for Terraform Enterprise. Typically this will be `linux` and `amd64`.
+## Custom Plugins
+To include custom plugins in the bundle file, create a local directory
+"./plugins" and put all the plugins you want to include there, under the
+required [sub directory](#plugins-directory-layout). Optionally, you can use the
+`-plugin-dir` flag to specify a location where to find the plugins. To be
+recognized as a valid plugin, the file must have a name of the form
+`terraform-provider-<NAME>`. In addition, ensure that the plugin is built using
+the same operating system and architecture used for Terraform Enterprise.
+Typically this will be `linux` and `amd64`.
+
+### Plugins Directory Layout
+To include custom plugins in the bundle file, you must specify a "source"
+attribute in the configuration and place the plugin in the appropriate
+subdirectory under "./plugins". The directory must have the following layout:
+
+```
+./plugins/$SOURCEHOST/$SOURCENAMESPACE/$NAME/$VERSION/$OS_$ARCH/
+```
+
+When installing custom plugins, you may choose any arbitrary identifier for the
+$SOURCEHOST and $SOURCENAMESPACE subdirectories. 
+
+For example, given the following configuration and a plugin built for Terraform Enterprise:
+
+```
+providers {
+  customplugin = {
+    versions = ["0.1"]
+    source = "example.com/myorg/customplugin"
+  }
+}
+```
+
+The binary must be placed in the following directory:
+
+```
+./plugins/example.com/myorg/customplugin/0.1/linux_amd64/
+```

 ## Provider Resolution Behavior

--- a/tools/terraform-bundle/config.go
+++ b/tools/terraform-bundle/config.go
@ -5,20 +5,27 @@ import (
 	"io/ioutil"

 	"github.com/hashicorp/hcl"
+	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/internal/getproviders"
 	"github.com/hashicorp/terraform/plugin/discovery"
 )

-var zeroTwelve = discovery.ConstraintStr(">= 0.12.0").MustParse()
+var zeroThirteen = discovery.ConstraintStr(">= 0.13.0").MustParse()

 type Config struct {
-	Terraform TerraformConfig                      `hcl:"terraform"`
-	Providers map[string][]discovery.ConstraintStr `hcl:"providers"`
+	Terraform TerraformConfig           `hcl:"terraform"`
+	Providers map[string]ProviderConfig `hcl:"providers"`
 }

 type TerraformConfig struct {
 	Version discovery.VersionStr `hcl:"version"`
 }

+type ProviderConfig struct {
+	Versions []string `hcl:"versions"`
+	Source   string   `hcl:"source"`
+}
+
 func LoadConfig(src []byte, filename string) (*Config, error) {
 	config := &Config{}
 	err := hcl.Decode(config, string(src))
@ -49,17 +56,23 @@ func (c *Config) validate() error {
 	if v, err = c.Terraform.Version.Parse(); err != nil {
 		return fmt.Errorf("terraform.version: %s", err)
 	}
-	if !zeroTwelve.Allows(v) {
-		return fmt.Errorf("this version of terraform-bundle can only build bundles for Terraform v0.12 and later; build terraform-bundle from the v0.11 branch or a v0.11.* tag to construct bundles for earlier versions")
+	if !zeroThirteen.Allows(v) {
+		return fmt.Errorf("this version of terraform-bundle can only build bundles for Terraform v0.13 and later; build terraform-bundle from a release tag (such as v0.12.*) to construct bundles for earlier versions")
 	}

 	if c.Providers == nil {
-		c.Providers = map[string][]discovery.ConstraintStr{}
+		c.Providers = map[string]ProviderConfig{}
 	}

 	for k, cs := range c.Providers {
-		for _, c := range cs {
-			if _, err := c.Parse(); err != nil {
+		if cs.Source != "" {
+			_, diags := addrs.ParseProviderSourceString(cs.Source)
+			if diags.HasErrors() {
+				return fmt.Errorf("providers.%s: %s", k, diags.Err().Error())
+			}
+		}
+		for _, c := range cs.Versions {
+			if _, err := getproviders.ParseVersionConstraints(c); err != nil {
 				return fmt.Errorf("providers.%s: %s", k, err)
 			}
 		}
--- a/tools/terraform-bundle/e2etest/package_test.go
+++ b/tools/terraform-bundle/e2etest/package_test.go
@ -1,7 +1,11 @@
 package e2etest

 import (
+	"archive/zip"
+	"fmt"
+	"io/ioutil"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"

@ -10,6 +14,9 @@ import (

 func TestPackage_empty(t *testing.T) {
 	t.Parallel()
+	// The e2etests can be reenabled when there is a terraform v0.13* release
+	// available on releases.hashicorp.com.
+	t.Skip("terraform-bundle e2e tests are temporarily paused")

 	// This test reaches out to releases.hashicorp.com to download the
 	// template provider, so it can only run if network access is allowed.
@ -45,11 +52,15 @@ func TestPackage_empty(t *testing.T) {
 func TestPackage_manyProviders(t *testing.T) {
 	t.Parallel()

-	// This test reaches out to releases.hashicorp.com to download the
-	// template provider, so it can only run if network access is allowed.
-	// We intentionally don't try to stub this here, because there's already
-	// a stubbed version of this in the "command" package and so the goal here
-	// is to test the interaction with the real repository.
+	// The e2etests can be reenabled when there is a terraform v0.13* release
+	// available on releases.hashicorp.com.
+	t.Skip("terraform-bundle e2e tests are temporarily paused")
+
+	// This test reaches out to releases.hashicorp.com to download providers, so
+	// it can only run if network access is allowed. We intentionally don't try
+	// to stub this here, because there's already a stubbed version of this in
+	// the "command" package and so the goal here is to test the interaction
+	// with the real repository.
 	skipIfCannotAccessNetwork(t)

 	fixturePath := filepath.Join("testdata", "many-providers")
@ -68,27 +79,22 @@ func TestPackage_manyProviders(t *testing.T) {
 	// Here we have to check each provider separately
 	// because it's internally held in a map (i.e. not guaranteed order)

-	if !strings.Contains(stdout, `- Resolving "aws" provider (~> 2.26.0)...
- Checking for provider plugin on https://releases.hashicorp.com...
- Downloading plugin for provider "aws" (hashicorp/aws) 2.26.0...`) {
+	if !strings.Contains(stdout, `- Finding hashicorp/aws versions matching "~> 2.26.0"...
+- Installing hashicorp/aws v2.26.0...`) {
 		t.Errorf("success message is missing from output:\n%s", stdout)
 	}

-	if !strings.Contains(stdout, `- Resolving "kubernetes" provider (1.8.0)...
- Checking for provider plugin on https://releases.hashicorp.com...
- Downloading plugin for provider "kubernetes" (hashicorp/kubernetes) 1.8.0...
- Resolving "kubernetes" provider (1.8.1)...
- Checking for provider plugin on https://releases.hashicorp.com...
- Downloading plugin for provider "kubernetes" (hashicorp/kubernetes) 1.8.1...
- Resolving "kubernetes" provider (1.9.0)...
- Checking for provider plugin on https://releases.hashicorp.com...
- Downloading plugin for provider "kubernetes" (hashicorp/kubernetes) 1.9.0...`) {
+	if !strings.Contains(stdout, `- Finding hashicorp/kubernetes versions matching "1.8.0"...
+- Installing hashicorp/kubernetes v1.8.0...
+- Finding hashicorp/kubernetes versions matching "1.8.1"...
+- Installing hashicorp/kubernetes v1.8.1...
+- Finding hashicorp/kubernetes versions matching "1.9.0"...
+- Installing hashicorp/kubernetes v1.9.0...`) {
 		t.Errorf("success message is missing from output:\n%s", stdout)
 	}

-	if !strings.Contains(stdout, `- Resolving "null" provider (2.1.0)...
- Checking for provider plugin on https://releases.hashicorp.com...
- Downloading plugin for provider "null" (hashicorp/null) 2.1.0...`) {
+	if !strings.Contains(stdout, `- Finding hashicorp/null versions matching "2.1.0"...
+- Installing hashicorp/null v2.1.0...`) {
 		t.Errorf("success message is missing from output:\n%s", stdout)
 	}

@ -101,4 +107,129 @@ func TestPackage_manyProviders(t *testing.T) {
 	if !strings.Contains(stdout, "All done!") {
 		t.Errorf("success message is missing from output:\n%s", stdout)
 	}
+
+	// check the contents of the created zipfile
+	files, err := ioutil.ReadDir(tfBundle.WorkDir())
+	if err != nil {
+		t.Fatalf("error reading workdir: %s", err)
+	}
+	for _, file := range files {
+		if strings.Contains(file.Name(), "terraform_0.12.0-bundle") {
+			read, err := zip.OpenReader(filepath.Join(tfBundle.WorkDir(), file.Name()))
+			if err != nil {
+				t.Fatalf("Failed to open archive: %s", err)
+			}
+			defer read.Close()
+
+			expectedFiles := map[string]struct{}{
+				"terraform":                                   {},
+				testProviderBinaryPath("null", "2.1.0"):       {},
+				testProviderBinaryPath("aws", "2.26.0"):       {},
+				testProviderBinaryPath("kubernetes", "1.8.0"): {},
+				testProviderBinaryPath("kubernetes", "1.8.1"): {},
+				testProviderBinaryPath("kubernetes", "1.9.0"): {},
+			}
+			extraFiles := make(map[string]struct{})
+
+			for _, file := range read.File {
+				if _, exists := expectedFiles[file.Name]; exists {
+					delete(expectedFiles, file.Name)
+				} else {
+					extraFiles[file.Name] = struct{}{}
+				}
+			}
+			if len(expectedFiles) != 0 {
+				t.Errorf("missing expected file(s): %#v", expectedFiles)
+			}
+			if len(extraFiles) != 0 {
+				t.Errorf("found extra unexpected file(s): %#v", extraFiles)
+			}
+		}
+	}
+}
+
+func TestPackage_localProviders(t *testing.T) {
+	t.Parallel()
+
+	// The e2etests can be reenabled when there is a terraform v0.13* release
+	// available on releases.hashicorp.com.
+	t.Skip("terraform-bundle e2e tests are temporarily paused")
+
+	// This test reaches out to releases.hashicorp.com to download terrafrom, so
+	// it can only run if network access is allowed. The providers are installed
+	// from the local cache.
+	skipIfCannotAccessNetwork(t)
+
+	fixturePath := filepath.Join("testdata", "local-providers")
+	tfBundle := e2e.NewBinary(bundleBin, fixturePath)
+	defer tfBundle.Close()
+
+	// we explicitly specify the platform so that tests can find the local binary under the expected directory
+	stdout, stderr, err := tfBundle.Run("package", "-os=darwin", "-arch=amd64", "terraform-bundle.hcl")
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	if stderr != "" {
+		t.Errorf("unexpected stderr output:\n%s", stderr)
+	}
+
+	// Here we have to check each provider separately
+	// because it's internally held in a map (i.e. not guaranteed order)
+	if !strings.Contains(stdout, "Fetching Terraform 0.12.0 core package...") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+	if !strings.Contains(stdout, "Creating terraform_0.12.0-bundle") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+	if !strings.Contains(stdout, "All done!") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+
+	// check the contents of the created zipfile
+	files, err := ioutil.ReadDir(tfBundle.WorkDir())
+	if err != nil {
+		t.Fatalf("error reading workdir: %s", err)
+	}
+	for _, file := range files {
+		if strings.Contains(file.Name(), "terraform_0.12.0-bundle") {
+			read, err := zip.OpenReader(filepath.Join(tfBundle.WorkDir(), file.Name()))
+			if err != nil {
+				t.Fatalf("Failed to open archive: %s", err)
+			}
+			defer read.Close()
+
+			expectedFiles := map[string]struct{}{
+				"terraform": {},
+				"plugins/example.com/myorg/mycloud/0.1.0/darwin_amd64/terraform-provider-mycloud": {},
+			}
+			extraFiles := make(map[string]struct{})
+
+			for _, file := range read.File {
+				if _, exists := expectedFiles[file.Name]; exists {
+					delete(expectedFiles, file.Name)
+				} else {
+					extraFiles[file.Name] = struct{}{}
+				}
+			}
+			if len(expectedFiles) != 0 {
+				t.Errorf("missing expected file(s): %#v", expectedFiles)
+			}
+			if len(extraFiles) != 0 {
+				t.Errorf("found extra unexpected file(s): %#v", extraFiles)
+			}
+		}
+	}
+}
+
+// testProviderBinaryPath takes a provider name (assumed to be a hashicorp
+// provider) and version and returns the expected binary path, relative to the
+// archive, for the plugin.
+func testProviderBinaryPath(provider, version string) string {
+	os := runtime.GOOS
+	arch := runtime.GOARCH
+	return fmt.Sprintf(
+		"plugins/registry.terraform.io/hashicorp/%s/%s/%s_%s/terraform-provider-%s_v%s_x4",
+		provider, version, os, arch, provider, version,
+	)
 }
--- a/tools/terraform-bundle/e2etest/testdata/empty/terraform-bundle.hcl
+++ b/tools/terraform-bundle/e2etest/testdata/empty/terraform-bundle.hcl
@ -1,3 +1,3 @@
 terraform {
-  version = "0.12.0"
+  version = "0.13.0"
 }
--- a/tools/terraform-bundle/e2etest/testdata/local-providers/.plugins/example.com/myorg/mycloud/0.1/darwin_amd64/terraform-provider-mycloud
+++ b/tools/terraform-bundle/e2etest/testdata/local-providers/.plugins/example.com/myorg/mycloud/0.1/darwin_amd64/terraform-provider-mycloud
@ -0,0 +1 @@
+I am a fake binary.
--- a/tools/terraform-bundle/e2etest/testdata/local-providers/terraform-bundle.hcl
+++ b/tools/terraform-bundle/e2etest/testdata/local-providers/terraform-bundle.hcl
@ -0,0 +1,11 @@
+terraform {
+  version = "0.3.0"
+}
+
+providers {
+  // this provider is installed in .plugins
+  mycloud = {
+    versions = ["0.1"]
+    source   = "example.com/myorg/mycloud"
+  }
+}
--- a/tools/terraform-bundle/e2etest/testdata/many-providers/terraform-bundle.hcl
+++ b/tools/terraform-bundle/e2etest/testdata/many-providers/terraform-bundle.hcl
@ -1,9 +1,17 @@
 terraform {
-  version = "0.12.0"
+  version = "0.13.0"
 }

 providers {
-  aws        = ["~> 2.26.0"]
-  kubernetes = ["1.8.0", "1.8.1", "1.9.0"]
-  null       = ["2.1.0"]
+  aws = {
+    versions = ["~> 2.26.0"]
+  }
+
+  kubernetes = {
+    versions = ["1.8.0", "1.8.1", "1.9.0"]
+  }
+
+  null = {
+    versions = ["2.1.0"]
+  }
 }
--- a/tools/terraform-bundle/package.go
+++ b/tools/terraform-bundle/package.go
@ -2,85 +2,37 @@ package main

 import (
 	"archive/zip"
+	"context"
+	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"time"

-	"flag"
-
-	"io"
-
 	getter "github.com/hashicorp/go-getter"
+	"github.com/hashicorp/terraform-svchost/disco"
 	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/httpclient"
+	"github.com/hashicorp/terraform/internal/getproviders"
+	"github.com/hashicorp/terraform/internal/providercache"
 	discovery "github.com/hashicorp/terraform/plugin/discovery"
+	"github.com/hashicorp/terraform/tfdiags"
+	"github.com/hashicorp/terraform/version"
 	"github.com/mitchellh/cli"
 )

 var releaseHost = "https://releases.hashicorp.com"

+var pluginDir = ".plugins"
+
 type PackageCommand struct {
 	ui cli.Ui
 }

-// shameless stackoverflow copy + pasta https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
-func CopyFile(src, dst string) (err error) {
-	sfi, err := os.Stat(src)
-	if err != nil {
-		return
-	}
-	if !sfi.Mode().IsRegular() {
-		// cannot copy non-regular files (e.g., directories,
-		// symlinks, devices, etc.)
-		return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String())
-	}
-	dfi, err := os.Stat(dst)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return
-		}
-	} else {
-		if !(dfi.Mode().IsRegular()) {
-			return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String())
-		}
-		if os.SameFile(sfi, dfi) {
-			return
-		}
-	}
-	if err = os.Link(src, dst); err == nil {
-		return
-	}
-	err = copyFileContents(src, dst)
-	os.Chmod(dst, sfi.Mode())
-	return
-}
-
-// see above
-func copyFileContents(src, dst string) (err error) {
-	in, err := os.Open(src)
-	if err != nil {
-		return
-	}
-	defer in.Close()
-	out, err := os.Create(dst)
-	if err != nil {
-		return
-	}
-	defer func() {
-		cerr := out.Close()
-		if err == nil {
-			err = cerr
-		}
-	}()
-	if _, err = io.Copy(out, in); err != nil {
-		return
-	}
-	err = out.Sync()
-	return
-}
-
 func (c *PackageCommand) Run(args []string) int {
 	flags := flag.NewFlagSet("package", flag.ExitOnError)
 	osPtr := flags.String("os", "", "Target operating system")
@ -94,7 +46,6 @@ func (c *PackageCommand) Run(args []string) int {

 	osName := runtime.GOOS
 	archName := runtime.GOARCH
-	pluginDir := "./plugins"
 	if *osPtr != "" {
 		osName = *osPtr
 	}
@ -117,86 +68,102 @@ func (c *PackageCommand) Run(args []string) int {
 		return 1
 	}

-	if discovery.ConstraintStr("< 0.10.0-beta1").MustParse().Allows(config.Terraform.Version.MustParse()) {
-		c.ui.Error("Bundles can be created only for Terraform 0.10 or newer")
-		return 1
-	}
-
-	workDir, err := ioutil.TempDir("", "terraform-bundle")
+	tmpDir, err := ioutil.TempDir("", "terraform-bundle")
 	if err != nil {
 		c.ui.Error(fmt.Sprintf("Could not create temporary dir: %s", err))
 		return 1
 	}
+	// symlinked tmp directories can cause odd behaviors.
+	workDir, err := filepath.EvalSymlinks(tmpDir)
 	defer os.RemoveAll(workDir)

 	c.ui.Info(fmt.Sprintf("Fetching Terraform %s core package...", config.Terraform.Version))

 	coreZipURL := c.coreURL(config.Terraform.Version, osName, archName)
 	err = getter.Get(workDir, coreZipURL)
-
 	if err != nil {
 		c.ui.Error(fmt.Sprintf("Failed to fetch core package from %s: %s", coreZipURL, err))
 		return 1
 	}

-	c.ui.Info(fmt.Sprintf("Fetching 3rd party plugins in directory: %s", pluginDir))
-	dirs := []string{pluginDir} //FindPlugins requires an array
-	localPlugins := discovery.FindPlugins("provider", dirs)
-	for k, _ := range localPlugins {
-		c.ui.Info(fmt.Sprintf("plugin: %s (%s)", k.Name, k.Version))
+	// get the list of required providers from the config
+	reqs := make(map[addrs.Provider][]string)
+	for name, provider := range config.Providers {
+		var fqn addrs.Provider
+		var diags tfdiags.Diagnostics
+		if provider.Source != "" {
+			fqn, diags = addrs.ParseProviderSourceString(provider.Source)
+			if diags.HasErrors() {
+				c.ui.Error(fmt.Sprintf("Invalid provider source string: %s", provider.Source))
+				return 1
+			}
+		} else {
+			fqn = addrs.NewDefaultProvider(name)
+		}
+		reqs[fqn] = provider.Versions
 	}
-	installer := &discovery.ProviderInstaller{
-		Dir: workDir,
-
-		// FIXME: This is incorrect because it uses the protocol version of
-		// this tool, rather than of the Terraform binary we just downloaded.
-		// But we can't get this information from a Terraform binary, so
-		// we'll just ignore this for now and use the same plugin installer
-		// protocol version for terraform-bundle as the terraform shipped
-		// with this release.
-		//
-		// NOTE: To target older versions of terraform, use the terraform-bundle
-		// from the same tag.
-		PluginProtocolVersion: discovery.PluginInstallProtocolVersion,

+	// set up the provider installer
+	platform := getproviders.Platform{
 		OS:   osName,
 		Arch: archName,
-		Ui:   c.ui,
 	}
+	installdir := providercache.NewDirWithPlatform(filepath.Join(workDir, "plugins"), platform)

-	for name, constraintStrs := range config.Providers {
-		for _, constraintStr := range constraintStrs {
-			c.ui.Output(fmt.Sprintf("- Resolving %q provider (%s)...",
-				name, constraintStr))
-			foundPlugins := discovery.PluginMetaSet{}
-			constraint := constraintStr.MustParse()
-			for plugin, _ := range localPlugins {
-				if plugin.Name == name && constraint.Allows(plugin.Version.MustParse()) {
-					foundPlugins.Add(plugin)
+	services := disco.New()
+	services.SetUserAgent(httpclient.TerraformUserAgent(version.String()))
+	var sources []getproviders.MultiSourceSelector
+
+	// Find any local providers first so we can exclude these from the registry
+	// install. We'll just silently ignore any errors and assume it would fail
+	// real installation later too.
+	foundLocally := map[addrs.Provider]struct{}{}
+
+	if absPluginDir, err := filepath.Abs(pluginDir); err == nil {
+		c.ui.Info(fmt.Sprintf("Local plugin directory %q found; scanning for provider binaries.", pluginDir))
+		if _, err := os.Stat(absPluginDir); err == nil {
+			localSource := getproviders.NewFilesystemMirrorSource(absPluginDir)
+			if available, err := localSource.AllAvailablePackages(); err == nil {
+				for found := range available {
+					c.ui.Info(fmt.Sprintf("Found provider %q in %q. p", found.String(), pluginDir))
+					foundLocally[found] = struct{}{}
 				}
 			}
-
-			if len(foundPlugins) > 0 {
-				plugin := foundPlugins.Newest()
-				CopyFile(plugin.Path, workDir+"/terraform-provider-"+plugin.Name+"_v"+plugin.Version.MustParse().String()) //put into temp dir
-			} else { //attempt to get from the public registry if not found locally
-				c.ui.Output(fmt.Sprintf("- Checking for provider plugin on %s...",
-					releaseHost))
-				_, _, err := installer.Get(addrs.NewLegacyProvider(name), constraint)
-				if err != nil {
-					c.ui.Error(fmt.Sprintf("- Failed to resolve %s provider %s: %s", name, constraint, err))
-					return 1
-				}
+			sources = append(sources, getproviders.MultiSourceSelector{
+				Source: localSource,
+			})
+			if len(foundLocally) == 0 {
+				c.ui.Info(fmt.Sprintf("No local providers found in %q.", pluginDir))
 			}
+		} else {
+			c.ui.Info(fmt.Sprintf("No %q directory found, skipping local provider discovery.", pluginDir))
 		}
 	}

-	files, err := ioutil.ReadDir(workDir)
+	// Anything we found in local directories above is excluded from being
+	// looked up via the registry source we're about to construct.
+	var directExcluded getproviders.MultiSourceMatchingPatterns
+	for addr := range foundLocally {
+		directExcluded = append(directExcluded, addr)
+	}
+
+	// Add the registry source, minus any providers found in the local pluginDir.
+	sources = append(sources, getproviders.MultiSourceSelector{
+		Source:  getproviders.NewMemoizeSource(getproviders.NewRegistrySource(services)),
+		Exclude: directExcluded,
+	})
+
+	installer := providercache.NewInstaller(installdir, getproviders.MultiSource(sources))
+
+	err = c.ensureProviderVersions(installer, reqs)
 	if err != nil {
-		c.ui.Error(fmt.Sprintf("Failed to read work directory %s: %s", workDir, err))
+		c.ui.Error(err.Error())
 		return 1
 	}

+	// remove the selections.json file created by the provider installer
+	os.Remove(filepath.Join(workDir, "plugins", "selections.json"))
+
 	// If we get this far then our workDir now contains the union of the
 	// contents of all the zip files we downloaded above. We can now create
 	// our output file.
@ -221,41 +188,99 @@ func (c *PackageCommand) Run(args []string) int {
 		}
 	}()

-	for _, file := range files {
-		if file.IsDir() {
-			// should never happen unless something tampers with our tmpdir
-			continue
-		}
+	// recursively walk the workDir to get a list of all binary filepaths
+	err = filepath.Walk(workDir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}

-		fn := filepath.Join(workDir, file.Name())
-		r, err := os.Open(fn)
-		if err != nil {
-			c.ui.Error(fmt.Sprintf("Failed to open %s: %s", fn, err))
-			return 1
-		}
-		hdr, err := zip.FileInfoHeader(file)
-		if err != nil {
-			c.ui.Error(fmt.Sprintf("Failed to add zip entry for %s: %s", fn, err))
-			return 1
-		}
-		hdr.Method = zip.Deflate // be sure to compress files
-		w, err := outZ.CreateHeader(hdr)
-		if err != nil {
-			c.ui.Error(fmt.Sprintf("Failed to add zip entry for %s: %s", fn, err))
-			return 1
-		}
-		_, err = io.Copy(w, r)
-		if err != nil {
-			c.ui.Error(fmt.Sprintf("Failed to write %s to bundle: %s", fn, err))
-			return 1
-		}
+			// maybe symlinks
+			linkPath, err := filepath.EvalSymlinks(path)
+			if err != nil {
+				return err
+			}
+			linkInfo, err := os.Stat(linkPath)
+			if err != nil {
+				return err
+			}
+
+			if linkInfo.IsDir() {
+				// The only time we should encounter a symlink directory is when we
+				// have a locally-installed provider, so we will grab the provider
+				// binary from that file.
+				files, err := ioutil.ReadDir(linkPath)
+				if err != nil {
+					return err
+				}
+				for _, file := range files {
+					if strings.Contains(file.Name(), "terraform-provider") {
+						relPath, _ := filepath.Rel(workDir, path)
+						return addZipFile(
+							filepath.Join(linkPath, file.Name()), // the link to this provider binary
+							filepath.Join(relPath, file.Name()),  // the expected directory for the binary
+							info, outZ,
+						)
+					}
+				}
+				// This shouldn't happen - we should always find a provider
+				// binary and exit the loop - but on the chance it does not,
+				// just continue.
+				return nil
+			}
+
+			// provider plugins need to be created in the same relative directory structure
+			absPath, err := filepath.Abs(linkPath)
+			if err != nil {
+				return err
+			}
+			relPath, err := filepath.Rel(workDir, absPath)
+			if err != nil {
+				return err
+			}
+
+			return addZipFile(path, relPath, info, outZ)
+
+		})
+
+	if err != nil {
+		c.ui.Error(err.Error())
+		return 1
 	}
-
 	c.ui.Info("All done!")

 	return 0
 }

+// addZipFile is a helper function intneded to simplify customizing the file
+// path when adding a file to the zip archive. The relPath is specified for
+// provider binaries, which need to be zipped into the full directory hierarchy.
+func addZipFile(fn, relPath string, info os.FileInfo, outZ *zip.Writer) error {
+	hdr, err := zip.FileInfoHeader(info)
+	if err != nil {
+		return fmt.Errorf("Failed to add zip entry for %s: %s", fn, err)
+	}
+	hdr.Method = zip.Deflate // be sure to compress files
+	hdr.Name = relPath       // we need the full, relative path to the provider binary
+	w, err := outZ.CreateHeader(hdr)
+	if err != nil {
+		return fmt.Errorf("Failed to add zip entry for %s: %s", fn, err)
+	}
+
+	r, err := os.Open(fn)
+	if err != nil {
+		return fmt.Errorf("Failed to open %s: %s", fn, err)
+	}
+	_, err = io.Copy(w, r)
+	if err != nil {
+		return fmt.Errorf("Failed to write %s to bundle: %s", fn, err)
+	}
+	return nil
+}
+
 func (c *PackageCommand) bundleFilename(version discovery.VersionStr, time time.Time, osName, archName string) string {
 	time = time.UTC()
 	return fmt.Sprintf(
@ -306,26 +331,82 @@ not a normal Terraform configuration file. The file format looks like this:
  terraform {
    # Version of Terraform to include in the bundle. An exact version number
 	# is required.
-    version = "0.10.0"
+    version = "0.13.0"
  }

  # Define which provider plugins are to be included
  providers {
    # Include the newest "aws" provider version in the 1.0 series.
-    aws = ["~> 1.0"]
+    aws = {
+		versions = ["~> 1.0"]
+	}

    # Include both the newest 1.0 and 2.0 versions of the "google" provider.
    # Each item in these lists allows a distinct version to be added. If the
 	# two expressions match different versions then _both_ are included in
 	# the bundle archive.
-	google = ["~> 1.0", "~> 2.0"]
-	
-	#Include a custom plugin to the bundle. Will search for the plugin in the 
-	#plugins directory, and package it with the bundle archive. Plugin must have
-	#a name of the form: terraform-provider-*-v*, and must be built with the operating
-	#system and architecture that terraform enterprise is running, e.g. linux and amd64
-	customplugin = ["0.1"]
+	google = {
+		versions = ["~> 1.0", "~> 2.0"]
+	}
+
+	# Include a custom plugin to the bundle. Will search for the plugin in the 
+	# plugins directory, and package it with the bundle archive. Plugin must 
+	# have a name of the form: terraform-provider-*, and must be built with
+	# the operating system and architecture that terraform enterprise is running,
+	# e.g. linux and amd64.
+	# See the README for more information on the source attribute and plugin
+	# directory layout.
+	customplugin = {
+		versions = ["0.1"]
+		source = "example.com/myorg/customplugin"
+	}
  }

 `
 }
+
+// ensureProviderVersions is a wrapper around
+// providercache.EnsureProviderVersions which allows installing multiple
+// versions of a given provider.
+func (c *PackageCommand) ensureProviderVersions(installer *providercache.Installer, reqs map[addrs.Provider][]string) error {
+	mode := providercache.InstallNewProvidersOnly
+	evts := &providercache.InstallerEvents{
+		ProviderAlreadyInstalled: func(provider addrs.Provider, selectedVersion getproviders.Version) {
+			c.ui.Info(fmt.Sprintf("- Using previously-installed %s v%s", provider.ForDisplay(), selectedVersion))
+		},
+		QueryPackagesBegin: func(provider addrs.Provider, versionConstraints getproviders.VersionConstraints) {
+			if len(versionConstraints) > 0 {
+				c.ui.Info(fmt.Sprintf("- Finding %s versions matching %q...", provider.ForDisplay(), getproviders.VersionConstraintsString(versionConstraints)))
+			} else {
+				c.ui.Info(fmt.Sprintf("- Finding latest version of %s...", provider.ForDisplay()))
+			}
+		},
+		FetchPackageBegin: func(provider addrs.Provider, version getproviders.Version, location getproviders.PackageLocation) {
+			c.ui.Info(fmt.Sprintf("- Installing %s v%s...", provider.ForDisplay(), version))
+		},
+		QueryPackagesFailure: func(provider addrs.Provider, err error) {
+			c.ui.Error(fmt.Sprintf("Could not retrieve the list of available versions for provider %s: %s.", provider.ForDisplay(), err))
+		},
+		FetchPackageFailure: func(provider addrs.Provider, version getproviders.Version, err error) {
+			c.ui.Error(fmt.Sprintf("Error while installing %s v%s: %s.", provider.ForDisplay(), version, err))
+		},
+	}
+
+	ctx := evts.OnContext(context.TODO())
+	for provider, versions := range reqs {
+		for _, constraint := range versions {
+			req := make(getproviders.Requirements, 1)
+			cstr, err := getproviders.ParseVersionConstraints(constraint)
+			if err != nil {
+				return err
+			}
+			req[provider] = cstr
+			_, err = installer.EnsureProviderVersions(ctx, req, mode)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}