basic plugin getter

Add discovery.GetProviders to fetch plugins from the relases site.

This is an early version, with no tests, that only (probably) fetches
plugins from the default location. The URLs are still subject to change,
and since there are no plugin releases, it doesn't work at all yet.

command/init.go

@@ -6,10 +6,13 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/hashicorp/go-getter"
+	getter "github.com/hashicorp/go-getter"
+	"github.com/hashicorp/terraform/backend"
 	"github.com/hashicorp/terraform/config"
 	"github.com/hashicorp/terraform/config/module"
 	"github.com/hashicorp/terraform/helper/variables"
+	"github.com/hashicorp/terraform/plugin/discovery"
+	"github.com/hashicorp/terraform/terraform"
 )
 
 // InitCommand is a Command implementation that takes a Terraform
@@ -19,7 +22,7 @@ type InitCommand struct {
 }
 
 func (c *InitCommand) Run(args []string) int {
-	var flagBackend, flagGet bool
+	var flagBackend, flagGet, flagGetPlugins bool
 	var flagConfigExtra map[string]interface{}
 
 	args = c.Meta.process(args, false)
@@ -27,6 +30,7 @@ func (c *InitCommand) Run(args []string) int {
 	cmdFlags.BoolVar(&flagBackend, "backend", true, "")
 	cmdFlags.Var((*variables.FlagAny)(&flagConfigExtra), "backend-config", "")
 	cmdFlags.BoolVar(&flagGet, "get", true, "")
+	cmdFlags.BoolVar(&flagGetPlugins, "get-plugins", true, "")
 	cmdFlags.BoolVar(&c.forceInitCopy, "force-copy", false, "suppress prompts about copying state data")
 	cmdFlags.BoolVar(&c.Meta.stateLock, "lock", true, "lock state")
 	cmdFlags.DurationVar(&c.Meta.stateLockTimeout, "lock-timeout", 0, "lock timeout")
@@ -103,6 +107,8 @@ func (c *InitCommand) Run(args []string) int {
 		return 0
 	}
 
+	var back backend.Backend
+
 	// If we're performing a get or loading the backend, then we perform
 	// some extra tasks.
 	if flagGet || flagBackend {
@@ -125,10 +131,12 @@ func (c *InitCommand) Run(args []string) int {
 					"Error downloading modules: %s", err))
 				return 1
 			}
+
 		}
 
-		// If we're requesting backend configuration and configure it
-		if flagBackend {
+		// If we're requesting backend configuration or looking for required
+		// plugins, load the backend
+		if flagBackend || flagGetPlugins {
 			header = true
 
 			// Only output that we're initializing a backend if we have
@@ -145,13 +153,36 @@ func (c *InitCommand) Run(args []string) int {
 				ConfigExtra: flagConfigExtra,
 				Init:        true,
 			}
-			if _, err := c.Backend(opts); err != nil {
+			if back, err = c.Backend(opts); err != nil {
 				c.Ui.Error(err.Error())
 				return 1
 			}
 		}
 	}
 
+	// Now that we have loaded all modules, check the module tree for missing providers
+	if flagGetPlugins {
+		sMgr, err := back.State(c.Env())
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf(
+				"Error loading state: %s", err))
+			return 1
+		}
+
+		if err := sMgr.RefreshState(); err != nil {
+			c.Ui.Error(fmt.Sprintf(
+				"Error refreshing state: %s", err))
+			return 1
+		}
+
+		err = c.getProviders(path, sMgr.State())
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf(
+				"Error getting plugins: %s", err))
+			return 1
+		}
+	}
+
 	// If we outputted information, then we need to output a newline
 	// so that our success message is nicely spaced out from prior text.
 	if header {
@@ -163,6 +194,31 @@ func (c *InitCommand) Run(args []string) int {
 	return 0
 }
 
+// load the complete module tree, and fetch any missing providers
+func (c *InitCommand) getProviders(path string, state *terraform.State) error {
+	mod, err := c.Module(path)
+	if err != nil {
+		return err
+	}
+
+	if err := mod.Validate(); err != nil {
+		return err
+	}
+
+	requirements := terraform.ModuleTreeDependencies(mod, state).AllPluginRequirements()
+	missing := c.missingProviders(requirements)
+
+	dst := c.pluginDir()
+	for provider, reqd := range missing {
+		err := discovery.GetProvider(dst, provider, reqd)
+		// TODO: return all errors
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (c *InitCommand) copySource(dst, src, pwd string) error {
 	// Verify the directory is empty
 	if empty, err := config.IsEmptyDir(dst); err != nil {
@@ -226,6 +282,8 @@ Options:
 
   -get=true            Download any modules for this configuration.
 
+  -get-plugins=true    Download any missing plugins for this configuration.
+
   -input=true          Ask for input if necessary. If false, will error if
                        input was required.
 

command/plugins.go

@@ -40,7 +40,9 @@ func (r *multiVersionProviderResolver) ResolveProviders(
 	return factories, errs
 }
 
-func (m *Meta) providerResolver() terraform.ResourceProviderResolver {
+// providerPluginSet returns the set of valid providers that were discovered in
+// the defined search paths.
+func (m *Meta) providerPluginSet() discovery.PluginMetaSet {
 	var dirs []string
 
 	// When searching the following directories, earlier entries get precedence
@@ -54,11 +56,30 @@ func (m *Meta) providerResolver() terraform.ResourceProviderResolver {
 	plugins := discovery.FindPlugins("provider", dirs)
 	plugins, _ = plugins.ValidateVersions()
 
+	return plugins
+}
+
+func (m *Meta) providerResolver() terraform.ResourceProviderResolver {
 	return &multiVersionProviderResolver{
-		Available: plugins,
+		Available: m.providerPluginSet(),
 	}
 }
 
+// filter the requirements returning only the providers that we can't resolve
+func (m *Meta) missingProviders(reqd discovery.PluginRequirements) discovery.PluginRequirements {
+	missing := make(discovery.PluginRequirements)
+
+	candidates := m.providerPluginSet().ConstrainVersions(reqd)
+
+	for name, versionSet := range reqd {
+		if metas := candidates[name]; metas == nil {
+			missing[name] = versionSet
+		}
+	}
+
+	return missing
+}
+
 func (m *Meta) provisionerFactories() map[string]terraform.ResourceProvisionerFactory {
 	var dirs []string
 

plugin/discovery/get.go

@@ -0,0 +1,170 @@
+package discovery
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"runtime"
+	"strings"
+
+	"golang.org/x/net/html"
+
+	getter "github.com/hashicorp/go-getter"
+)
+
+const releasesURL = "https://releases.hashicorp.com/"
+
+// pluginURL generates URLs to lookup the versions of a plugin, or the file path.
+//
+// The URL for releases follows the pattern:
+//    https://releases.hashicorp.com/terraform-providers/terraform-provider-name/ +
+//        terraform-provider-name_<x.y.z>/terraform-provider-name_<x.y.z>_<os>_<arch>.<ext>
+//
+// The name prefix common to all plugins of this type.
+// This is either `terraform-provider` or `terraform-provisioner`.
+type pluginBaseName string
+
+// base returns the top level directory for all plugins of this type
+func (p pluginBaseName) base() string {
+	// the top level directory is the plural form of the plugin type
+	return releasesURL + string(p) + "s"
+}
+
+// versions returns the url to the directory to list available versions for this plugin
+func (p pluginBaseName) versions(name string) string {
+	return fmt.Sprintf("%s/%s-%s", p.base(), p, name)
+}
+
+// file returns the full path to a plugin based on the plugin name,
+// version, GOOS and GOARCH.
+func (p pluginBaseName) file(name, version string) string {
+	releasesDir := fmt.Sprintf("%s-%s_%s/", p, name, version)
+	fileName := fmt.Sprintf("%s-%s_%s_%s_%s.zip", p, name, version, runtime.GOOS, runtime.GOARCH)
+	return fmt.Sprintf("%s/%s/%s", p.versions(name), releasesDir, fileName)
+}
+
+var providersURL = pluginBaseName("terraform-provider")
+var provisionersURL = pluginBaseName("terraform-provisioners")
+
+// GetProvider fetches a provider plugin based on the version constraints, and
+// copies it to the dst directory.
+//
+// TODO: verify checksum and signature
+func GetProvider(dst, provider string, req Constraints) error {
+	versions, err := listProviderVersions(provider)
+	// TODO: return multiple errors
+	if err != nil {
+		return err
+	}
+
+	version := newestVersion(versions, req)
+	if version == nil {
+		return fmt.Errorf("no version of %q available that fulfills constraints %s", provider, req)
+	}
+
+	url := providersURL.file(provider, version.String())
+
+	log.Printf("[DEBUG] getting provider %q version %q at %s", provider, version, url)
+	return getter.Get(dst, url)
+}
+
+// take the list of available versions for a plugin, and the required
+// Constraints, and return the latest available version that satisfies the
+// constraints.
+func newestVersion(available []*Version, required Constraints) *Version {
+	var latest *Version
+	for _, v := range available {
+		if required.Has(*v) {
+			if latest == nil {
+				latest = v
+				continue
+			}
+
+			if v.NewerThan(*latest) {
+				latest = v
+			}
+		}
+	}
+
+	return latest
+}
+
+// list the version available for the named plugin
+func listProviderVersions(name string) ([]*Version, error) {
+	versions, err := listPluginVersions(providersURL.versions(name))
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch versions for provider %q: %s", name, err)
+	}
+	return versions, nil
+}
+
+func listProvisionerVersions(name string) ([]*Version, error) {
+	versions, err := listPluginVersions(provisionersURL.versions(name))
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch versions for provisioner %q: %s", name, err)
+	}
+
+	return versions, nil
+}
+
+// return a list of the plugin versions at the given URL
+// TODO: This doesn't yet take into account plugin protocol version.
+//       That may have to be checked via an http header via a separate request
+//       to each plugin file.
+func listPluginVersions(url string) ([]*Version, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := ioutil.ReadAll(resp.Body)
+		log.Printf("[ERROR] failed to fetch plugin versions from %s\n%s\n%s", url, resp.Status, body)
+		return nil, errors.New(resp.Status)
+	}
+
+	body, err := html.Parse(resp.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	names := []string{}
+
+	// all we need to do is list links on the directory listing page that look like plugins
+	var f func(*html.Node)
+	f = func(n *html.Node) {
+		if n.Type == html.ElementNode && n.Data == "a" {
+			c := n.FirstChild
+			if c != nil && c.Type == html.TextNode && strings.HasPrefix(c.Data, "terraform-") {
+				names = append(names, c.Data)
+				fmt.Println(c.Data)
+				return
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			f(c)
+		}
+	}
+	f(body)
+
+	var versions []*Version
+
+	for _, name := range names {
+		parts := strings.SplitN(name, "_", 2)
+		if len(parts) == 2 && parts[1] != "" {
+			v, err := VersionStr(parts[1]).Parse()
+			if err != nil {
+				// filter invalid versions scraped from the page
+				log.Printf("[WARN] invalid version found for %q: %s", name, err)
+				continue
+			}
+
+			versions = append(versions, &v)
+		}
+	}
+
+	return versions, nil
+}