diff --git a/command/init.go b/command/init.go index 3023d9f14..7008bb6e0 100644 --- a/command/init.go +++ b/command/init.go @@ -6,10 +6,13 @@ import ( "path/filepath" "strings" - "github.com/hashicorp/go-getter" + getter "github.com/hashicorp/go-getter" + "github.com/hashicorp/terraform/backend" "github.com/hashicorp/terraform/config" "github.com/hashicorp/terraform/config/module" "github.com/hashicorp/terraform/helper/variables" + "github.com/hashicorp/terraform/plugin/discovery" + "github.com/hashicorp/terraform/terraform" ) // InitCommand is a Command implementation that takes a Terraform @@ -19,7 +22,7 @@ type InitCommand struct { } func (c *InitCommand) Run(args []string) int { - var flagBackend, flagGet bool + var flagBackend, flagGet, flagGetPlugins bool var flagConfigExtra map[string]interface{} args = c.Meta.process(args, false) @@ -27,6 +30,7 @@ func (c *InitCommand) Run(args []string) int { cmdFlags.BoolVar(&flagBackend, "backend", true, "") cmdFlags.Var((*variables.FlagAny)(&flagConfigExtra), "backend-config", "") cmdFlags.BoolVar(&flagGet, "get", true, "") + cmdFlags.BoolVar(&flagGetPlugins, "get-plugins", true, "") cmdFlags.BoolVar(&c.forceInitCopy, "force-copy", false, "suppress prompts about copying state data") cmdFlags.BoolVar(&c.Meta.stateLock, "lock", true, "lock state") cmdFlags.DurationVar(&c.Meta.stateLockTimeout, "lock-timeout", 0, "lock timeout") @@ -103,6 +107,8 @@ func (c *InitCommand) Run(args []string) int { return 0 } + var back backend.Backend + // If we're performing a get or loading the backend, then we perform // some extra tasks. if flagGet || flagBackend { @@ -125,10 +131,12 @@ func (c *InitCommand) Run(args []string) int { "Error downloading modules: %s", err)) return 1 } + } - // If we're requesting backend configuration and configure it - if flagBackend { + // If we're requesting backend configuration or looking for required + // plugins, load the backend + if flagBackend || flagGetPlugins { header = true // Only output that we're initializing a backend if we have @@ -145,13 +153,36 @@ func (c *InitCommand) Run(args []string) int { ConfigExtra: flagConfigExtra, Init: true, } - if _, err := c.Backend(opts); err != nil { + if back, err = c.Backend(opts); err != nil { c.Ui.Error(err.Error()) return 1 } } } + // Now that we have loaded all modules, check the module tree for missing providers + if flagGetPlugins { + sMgr, err := back.State(c.Env()) + if err != nil { + c.Ui.Error(fmt.Sprintf( + "Error loading state: %s", err)) + return 1 + } + + if err := sMgr.RefreshState(); err != nil { + c.Ui.Error(fmt.Sprintf( + "Error refreshing state: %s", err)) + return 1 + } + + err = c.getProviders(path, sMgr.State()) + if err != nil { + c.Ui.Error(fmt.Sprintf( + "Error getting plugins: %s", err)) + return 1 + } + } + // If we outputted information, then we need to output a newline // so that our success message is nicely spaced out from prior text. if header { @@ -163,6 +194,31 @@ func (c *InitCommand) Run(args []string) int { return 0 } +// load the complete module tree, and fetch any missing providers +func (c *InitCommand) getProviders(path string, state *terraform.State) error { + mod, err := c.Module(path) + if err != nil { + return err + } + + if err := mod.Validate(); err != nil { + return err + } + + requirements := terraform.ModuleTreeDependencies(mod, state).AllPluginRequirements() + missing := c.missingProviders(requirements) + + dst := c.pluginDir() + for provider, reqd := range missing { + err := discovery.GetProvider(dst, provider, reqd) + // TODO: return all errors + if err != nil { + return err + } + } + return nil +} + func (c *InitCommand) copySource(dst, src, pwd string) error { // Verify the directory is empty if empty, err := config.IsEmptyDir(dst); err != nil { @@ -226,6 +282,8 @@ Options: -get=true Download any modules for this configuration. + -get-plugins=true Download any missing plugins for this configuration. + -input=true Ask for input if necessary. If false, will error if input was required. diff --git a/command/plugins.go b/command/plugins.go index a0f94bc1f..b36f52929 100644 --- a/command/plugins.go +++ b/command/plugins.go @@ -40,7 +40,9 @@ func (r *multiVersionProviderResolver) ResolveProviders( return factories, errs } -func (m *Meta) providerResolver() terraform.ResourceProviderResolver { +// providerPluginSet returns the set of valid providers that were discovered in +// the defined search paths. +func (m *Meta) providerPluginSet() discovery.PluginMetaSet { var dirs []string // When searching the following directories, earlier entries get precedence @@ -54,11 +56,30 @@ func (m *Meta) providerResolver() terraform.ResourceProviderResolver { plugins := discovery.FindPlugins("provider", dirs) plugins, _ = plugins.ValidateVersions() + return plugins +} + +func (m *Meta) providerResolver() terraform.ResourceProviderResolver { return &multiVersionProviderResolver{ - Available: plugins, + Available: m.providerPluginSet(), } } +// filter the requirements returning only the providers that we can't resolve +func (m *Meta) missingProviders(reqd discovery.PluginRequirements) discovery.PluginRequirements { + missing := make(discovery.PluginRequirements) + + candidates := m.providerPluginSet().ConstrainVersions(reqd) + + for name, versionSet := range reqd { + if metas := candidates[name]; metas == nil { + missing[name] = versionSet + } + } + + return missing +} + func (m *Meta) provisionerFactories() map[string]terraform.ResourceProvisionerFactory { var dirs []string diff --git a/plugin/discovery/get.go b/plugin/discovery/get.go new file mode 100644 index 000000000..779a02e38 --- /dev/null +++ b/plugin/discovery/get.go @@ -0,0 +1,170 @@ +package discovery + +import ( + "errors" + "fmt" + "io/ioutil" + "log" + "net/http" + "runtime" + "strings" + + "golang.org/x/net/html" + + getter "github.com/hashicorp/go-getter" +) + +const releasesURL = "https://releases.hashicorp.com/" + +// pluginURL generates URLs to lookup the versions of a plugin, or the file path. +// +// The URL for releases follows the pattern: +// https://releases.hashicorp.com/terraform-providers/terraform-provider-name/ + +// terraform-provider-name_/terraform-provider-name___. +// +// The name prefix common to all plugins of this type. +// This is either `terraform-provider` or `terraform-provisioner`. +type pluginBaseName string + +// base returns the top level directory for all plugins of this type +func (p pluginBaseName) base() string { + // the top level directory is the plural form of the plugin type + return releasesURL + string(p) + "s" +} + +// versions returns the url to the directory to list available versions for this plugin +func (p pluginBaseName) versions(name string) string { + return fmt.Sprintf("%s/%s-%s", p.base(), p, name) +} + +// file returns the full path to a plugin based on the plugin name, +// version, GOOS and GOARCH. +func (p pluginBaseName) file(name, version string) string { + releasesDir := fmt.Sprintf("%s-%s_%s/", p, name, version) + fileName := fmt.Sprintf("%s-%s_%s_%s_%s.zip", p, name, version, runtime.GOOS, runtime.GOARCH) + return fmt.Sprintf("%s/%s/%s", p.versions(name), releasesDir, fileName) +} + +var providersURL = pluginBaseName("terraform-provider") +var provisionersURL = pluginBaseName("terraform-provisioners") + +// GetProvider fetches a provider plugin based on the version constraints, and +// copies it to the dst directory. +// +// TODO: verify checksum and signature +func GetProvider(dst, provider string, req Constraints) error { + versions, err := listProviderVersions(provider) + // TODO: return multiple errors + if err != nil { + return err + } + + version := newestVersion(versions, req) + if version == nil { + return fmt.Errorf("no version of %q available that fulfills constraints %s", provider, req) + } + + url := providersURL.file(provider, version.String()) + + log.Printf("[DEBUG] getting provider %q version %q at %s", provider, version, url) + return getter.Get(dst, url) +} + +// take the list of available versions for a plugin, and the required +// Constraints, and return the latest available version that satisfies the +// constraints. +func newestVersion(available []*Version, required Constraints) *Version { + var latest *Version + for _, v := range available { + if required.Has(*v) { + if latest == nil { + latest = v + continue + } + + if v.NewerThan(*latest) { + latest = v + } + } + } + + return latest +} + +// list the version available for the named plugin +func listProviderVersions(name string) ([]*Version, error) { + versions, err := listPluginVersions(providersURL.versions(name)) + if err != nil { + return nil, fmt.Errorf("failed to fetch versions for provider %q: %s", name, err) + } + return versions, nil +} + +func listProvisionerVersions(name string) ([]*Version, error) { + versions, err := listPluginVersions(provisionersURL.versions(name)) + if err != nil { + return nil, fmt.Errorf("failed to fetch versions for provisioner %q: %s", name, err) + } + + return versions, nil +} + +// return a list of the plugin versions at the given URL +// TODO: This doesn't yet take into account plugin protocol version. +// That may have to be checked via an http header via a separate request +// to each plugin file. +func listPluginVersions(url string) ([]*Version, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + log.Printf("[ERROR] failed to fetch plugin versions from %s\n%s\n%s", url, resp.Status, body) + return nil, errors.New(resp.Status) + } + + body, err := html.Parse(resp.Body) + if err != nil { + log.Fatal(err) + } + + names := []string{} + + // all we need to do is list links on the directory listing page that look like plugins + var f func(*html.Node) + f = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "a" { + c := n.FirstChild + if c != nil && c.Type == html.TextNode && strings.HasPrefix(c.Data, "terraform-") { + names = append(names, c.Data) + fmt.Println(c.Data) + return + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + f(body) + + var versions []*Version + + for _, name := range names { + parts := strings.SplitN(name, "_", 2) + if len(parts) == 2 && parts[1] != "" { + v, err := VersionStr(parts[1]).Parse() + if err != nil { + // filter invalid versions scraped from the page + log.Printf("[WARN] invalid version found for %q: %s", name, err) + continue + } + + versions = append(versions, &v) + } + } + + return versions, nil +} diff --git a/plugin/getter/get.go b/plugin/getter/get.go deleted file mode 100644 index e69de29bb..000000000