resilien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

terraform: HCL2-flavored module dependency resolver 

For the moment this is just a lightly-adapted copy of
ModuleTreeDependencies named ConfigTreeDependencies, with the goal that
the two can live concurrently for the moment while not all callers are yet
updated and then we can drop ModuleTreeDependencies and its helper
functions altogether in a later commit.

This can then be used to make "terraform init" and "terraform providers"
work properly with the HCL2-powered configuration loader.
This commit is contained in:
Martin Atkins 2018-03-27 17:22:51 -07:00
parent ebafa51723
commit 4ed06a9227
5 changed files with 285 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
command/init.go
View File

@ -7,7 +7,6 @@ import (
"sort"
"strings"
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/hcl2/hcl"
"github.com/hashicorp/terraform/backend"
backendinit "github.com/hashicorp/terraform/backend/init"
@ -145,6 +144,8 @@ func (c *InitCommand) Run(args []string) int {
c.showDiagnostics(diags)
return 1
}
c.Ui.Output("")
}
// If our directory is empty, then we're done. We can't get or setup
@ -289,10 +290,10 @@ func (c *InitCommand) Run(args []string) int {
}
// Now that we have loaded all modules, check the module tree for missing providers.
err = c.getProviders(path, state, flagUpgrade)
if err != nil {
// this function provides its own output
log.Printf("[ERROR] %s", err)
providerDiags := c.getProviders(path, state, flagUpgrade)
diags = diags.Append(providerDiags)
if providerDiags.HasErrors() {
c.showDiagnostics(diags)
return 1
}
@ -302,6 +303,11 @@ func (c *InitCommand) Run(args []string) int {
c.Ui.Output("")
}
// If we accumulated any warnings along the way that weren't accompanied
// by errors then we'll output them here so that the success message is
// still the final thing shown.
c.showDiagnostics(diags)
c.Ui.Output(c.Colorize().Color(strings.TrimSpace(outputInitSuccess)))
if !c.RunningInAutomation {
// If we're not running in an automation wrapper, give the user
@ -384,24 +390,25 @@ func (c *InitCommand) backendConfigOverrideBody(flags rawFlags, schema *configsc
// Load the complete module tree, and fetch any missing providers.
// This method outputs its own Ui.
func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade bool) error {
mod, diags := c.Module(path)
func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade bool) tfdiags.Diagnostics {
config, diags := c.loadConfig(path)
if diags.HasErrors() {
c.showDiagnostics(diags)
return diags.Err()
return diags
}
if err := terraform.CheckStateVersion(state); err != nil {
diags = diags.Append(err)
c.showDiagnostics(diags)
return err
return diags
}
if err := terraform.CheckRequiredVersion(mod); err != nil {
diags = diags.Append(err)
c.showDiagnostics(diags)
return err
}
// FIXME: Restore this once terraform.CheckRequiredVersion is updated to
// work with a configs.Config instead of a legacy module.Tree.
/*
if err := terraform.CheckRequiredVersion(mod); err != nil {
diags = diags.Append(err)
return diags
}
*/
var available discovery.PluginMetaSet
if upgrade {
@ -412,7 +419,7 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
available = c.providerPluginSet()
}
requirements := terraform.ModuleTreeDependencies(mod, state).AllPluginRequirements()
requirements := terraform.ConfigTreeDependencies(config, state).AllPluginRequirements()
if len(requirements) == 0 {
// nothing to initialize
return nil
@ -424,7 +431,6 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
missing := c.missingPlugins(available, requirements)
var errs error
if c.getPlugins {
if len(missing) > 0 {
c.Ui.Output(fmt.Sprintf("- Checking for available provider plugins on %s...",
@ -461,12 +467,12 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
c.Ui.Error(fmt.Sprintf(errProviderInstallError, provider, err.Error(), DefaultPluginVendorDir))
}
errs = multierror.Append(errs, err)
diags = diags.Append(err)
}
}
if errs != nil {
return errs
if diags.HasErrors() {
return diags
}
} else if len(missing) > 0 {
// we have missing providers, but aren't going to try and download them
@ -477,11 +483,11 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
} else {
lines = append(lines, fmt.Sprintf("* %s (%s)\n", provider, reqd.Versions))
}
errs = multierror.Append(errs, fmt.Errorf("missing provider %q", provider))
diags = diags.Append(fmt.Errorf("missing provider %q", provider))
}
sort.Strings(lines)
c.Ui.Error(fmt.Sprintf(errMissingProvidersNoInstall, strings.Join(lines, ""), DefaultPluginVendorDir))
return errs
return diags
}
// With all the providers downloaded, we'll generate our lock file
@ -497,8 +503,8 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
for name, meta := range chosen {
digest, err := meta.SHA256()
if err != nil {
c.Ui.Error(fmt.Sprintf("failed to read provider plugin %s: %s", meta.Path, err))
return err
diags = diags.Append(fmt.Errorf("Failed to read provider plugin %s: %s", meta.Path, err))
return diags
}
digests[name] = digest
if c.ignorePluginChecksum {
@ -507,8 +513,8 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
}
err := c.providerPluginsLock().Write(digests)
if err != nil {
c.Ui.Error(fmt.Sprintf("failed to save provider manifest: %s", err))
return err
diags = diags.Append(fmt.Errorf("failed to save provider manifest: %s", err))
return diags
}
{
@ -556,7 +562,7 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
}
}
return nil
return diags
}
func (c *InitCommand) AutocompleteArgs() complete.Predictor {

8
command/providers.go
View File

@ -5,6 +5,7 @@ import (
"sort"
"github.com/hashicorp/terraform/moduledeps"
"github.com/hashicorp/terraform/terraform"
"github.com/hashicorp/terraform/tfdiags"
"github.com/xlab/treeprint"
)
@ -69,11 +70,8 @@ func (c *ProvidersCommand) Run(args []string) int {
return 1
}
// FIXME: Restore this once the "terraform" package is updated to deal
// with HCL2 config types.
//s := state.State()
//depTree := terraform.ModuleTreeDependencies(config, s)
var depTree *moduledeps.Module
s := state.State()
depTree := terraform.ConfigTreeDependencies(config, s)
depTree.SortDescendents()
printRoot := treeprint.New()

43
configs/resource.go
View File

@ -2,6 +2,7 @@ package configs
import (
"fmt"
"strings"
"github.com/hashicorp/hcl2/gohcl"
"github.com/hashicorp/hcl2/hcl"
@ -39,6 +40,22 @@ func (r *ManagedResource) moduleUniqueKey() string {
return fmt.Sprintf("%s.%s", r.Name, r.Type)
}
// ProviderConfigKey returns a string key for the provider configuration
// that should be used for this resource. This function implements the
// default behavior of extracting the type from the resource type name if
// an explicit "provider" argument was not provided.
func (r *ManagedResource) ProviderConfigKey() string {
if r.ProviderConfigRef == nil {
typeName := r.Type
if under := strings.Index(typeName, "_"); under != -1 {
return typeName[:under]
}
return typeName
}
return r.ProviderConfigRef.String()
}
func decodeResourceBlock(block *hcl.Block) (*ManagedResource, hcl.Diagnostics) {
r := &ManagedResource{
Type: block.Labels[0],
@ -234,6 +251,22 @@ func (r *DataResource) moduleUniqueKey() string {
return fmt.Sprintf("data.%s.%s", r.Name, r.Type)
}
// ProviderConfigKey returns a string key for the provider configuration
// that should be used for this resource. This function implements the
// default behavior of extracting the type from the resource type name if
// an explicit "provider" argument was not provided.
func (r *DataResource) ProviderConfigKey() string {
if r.ProviderConfigRef == nil {
typeName := r.Type
if under := strings.Index(typeName, "_"); under != -1 {
return typeName[:under]
}
return typeName
}
return r.ProviderConfigRef.String()
}
func decodeDataBlock(block *hcl.Block) (*DataResource, hcl.Diagnostics) {
r := &DataResource{
Type: block.Labels[0],
@ -370,6 +403,16 @@ func decodeProviderConfigRef(attr *hcl.Attribute) (*ProviderConfigRef, hcl.Diagn
return ret, diags
}
func (r *ProviderConfigRef) String() string {
if r == nil {
return "<nil>"
}
if r.Alias != "" {
return fmt.Sprintf("%s.%s", r.Name, r.Alias)
}
return r.Name
}
var commonResourceAttributes = []hcl.AttributeSchema{
{
Name: "count",

5
plugin/discovery/version_set.go
View File

@ -36,6 +36,11 @@ type Constraints struct {
raw version.Constraints
}
// NewConstraints creates a Constraints based on a version.Constraints.
func NewConstraints(c version.Constraints) Constraints {
return Constraints{c}
}
// AllVersions is a Constraints containing all versions
var AllVersions Constraints

246
terraform/module_dependencies.go
View File

@ -1,12 +1,207 @@
package terraform
import (
version "github.com/hashicorp/go-version"
"github.com/hashicorp/terraform/config"
"github.com/hashicorp/terraform/config/module"
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/moduledeps"
"github.com/hashicorp/terraform/plugin/discovery"
)
// ConfigTreeDependencies returns the dependencies of the tree of modules
// described by the given configuration and state.
//
// Both configuration and state are required because there can be resources
// implied by instances in the state that no longer exist in config.
func ConfigTreeDependencies(root *configs.Config, state *State) *moduledeps.Module {
// First we walk the configuration tree to build the overall structure
// and capture the explicit/implicit/inherited provider dependencies.
deps := configTreeConfigDependencies(root, nil)
// Next we walk over the resources in the state to catch any additional
// dependencies created by existing resources that are no longer in config.
// Most things we find in state will already be present in 'deps', but
// we're interested in the rare thing that isn't.
configTreeMergeStateDependencies(deps, state)
return deps
}
func configTreeConfigDependencies(root *configs.Config, inheritProviders map[string]*configs.Provider) *moduledeps.Module {
if root == nil {
// If no config is provided, we'll make a synthetic root.
// This isn't necessarily correct if we're called with a nil that
// *isn't* at the root, but in practice that can never happen.
return &moduledeps.Module{
Name: "root",
}
}
name := "root"
if len(root.Path) != 0 {
name = root.Path[len(root.Path)-1]
}
ret := &moduledeps.Module{
Name: name,
}
module := root.Module
// Provider dependencies
{
providers := make(moduledeps.Providers)
// The main way to declare a provider dependency is explicitly inside
// the "terraform" block, which allows declaring a requirement without
// also creating a configuration.
for fullName, constraints := range module.ProviderRequirements {
inst := moduledeps.ProviderInstance(fullName)
// The handling here is a bit fiddly because the moduledeps package
// was designed around the legacy (pre-0.12) configuration model
// and hasn't yet been revised to handle the new model. As a result,
// we need to do some translation here.
// FIXME: Eventually we should adjust the underlying model so we
// can also retain the source location of each constraint, for
// more informative output from the "terraform providers" command.
var rawConstraints version.Constraints
for _, constraint := range constraints {
rawConstraints = append(rawConstraints, constraint.Required...)
}
discoConstraints := discovery.NewConstraints(rawConstraints)
providers[inst] = moduledeps.ProviderDependency{
Constraints: discoConstraints,
Reason: moduledeps.ProviderDependencyExplicit,
}
}
// Provider configurations can also include version constraints,
// allowing for more terse declaration in situations where both a
// configuration and a constraint are defined in the same module.
for fullName, pCfg := range module.ProviderConfigs {
inst := moduledeps.ProviderInstance(fullName)
discoConstraints := discovery.AllVersions
if pCfg.Version.Required != nil {
discoConstraints = discovery.NewConstraints(pCfg.Version.Required)
}
if existing, exists := providers[inst]; exists {
existing.Constraints = existing.Constraints.Append(discoConstraints)
} else {
providers[inst] = moduledeps.ProviderDependency{
Constraints: discoConstraints,
Reason: moduledeps.ProviderDependencyExplicit,
}
}
}
// Each resource in the configuration creates an *implicit* provider
// dependency, though we'll only record it if there isn't already
// an explicit dependency on the same provider.
for _, rc := range module.ManagedResources {
fullName := rc.ProviderConfigKey()
inst := moduledeps.ProviderInstance(fullName)
if _, exists := providers[inst]; exists {
// Explicit dependency already present
continue
}
reason := moduledeps.ProviderDependencyImplicit
if _, inherited := inheritProviders[fullName]; inherited {
reason = moduledeps.ProviderDependencyInherited
}
providers[inst] = moduledeps.ProviderDependency{
Constraints: discovery.AllVersions,
Reason: reason,
}
}
for _, rc := range module.DataResources {
fullName := rc.ProviderConfigKey()
inst := moduledeps.ProviderInstance(fullName)
if _, exists := providers[inst]; exists {
// Explicit dependency already present
continue
}
reason := moduledeps.ProviderDependencyImplicit
if _, inherited := inheritProviders[fullName]; inherited {
reason = moduledeps.ProviderDependencyInherited
}
providers[inst] = moduledeps.ProviderDependency{
Constraints: discovery.AllVersions,
Reason: reason,
}
}
ret.Providers = providers
}
childInherit := make(map[string]*configs.Provider)
for k, v := range inheritProviders {
childInherit[k] = v
}
for k, v := range module.ProviderConfigs {
childInherit[k] = v
}
for _, c := range root.Children {
ret.Children = append(ret.Children, configTreeConfigDependencies(c, childInherit))
}
return ret
}
func configTreeMergeStateDependencies(root *moduledeps.Module, state *State) {
if state == nil {
return
}
findModule := func(path []string) *moduledeps.Module {
module := root
for _, name := range path[1:] { // skip initial "root"
var next *moduledeps.Module
for _, cm := range module.Children {
if cm.Name == name {
next = cm
break
}
}
if next == nil {
// If we didn't find a next node, we'll need to make one
next = &moduledeps.Module{
Name: name,
}
module.Children = append(module.Children, next)
}
module = next
}
return module
}
for _, ms := range state.Modules {
module := findModule(ms.Path)
for _, is := range ms.Resources {
fullName := config.ResourceProviderFullName(is.Type, is.Provider)
inst := moduledeps.ProviderInstance(fullName)
if _, exists := module.Providers[inst]; !exists {
if module.Providers == nil {
module.Providers = make(moduledeps.Providers)
}
module.Providers[inst] = moduledeps.ProviderDependency{
Constraints: discovery.AllVersions,
Reason: moduledeps.ProviderDependencyFromState,
}
}
}
}
}
// ModuleTreeDependencies returns the dependencies of the tree of modules
// described by the given configuration tree and state.
//
@ -106,50 +301,9 @@ func moduleTreeConfigDependencies(root *module.Tree, inheritProviders map[string
}
func moduleTreeMergeStateDependencies(root *moduledeps.Module, state *State) {
if state == nil {
return
}
findModule := func(path []string) *moduledeps.Module {
module := root
for _, name := range path[1:] { // skip initial "root"
var next *moduledeps.Module
for _, cm := range module.Children {
if cm.Name == name {
next = cm
break
}
}
if next == nil {
// If we didn't find a next node, we'll need to make one
next = &moduledeps.Module{
Name: name,
}
module.Children = append(module.Children, next)
}
module = next
}
return module
}
for _, ms := range state.Modules {
module := findModule(ms.Path)
for _, is := range ms.Resources {
fullName := config.ResourceProviderFullName(is.Type, is.Provider)
inst := moduledeps.ProviderInstance(fullName)
if _, exists := module.Providers[inst]; !exists {
if module.Providers == nil {
module.Providers = make(moduledeps.Providers)
}
module.Providers[inst] = moduledeps.ProviderDependency{
Constraints: discovery.AllVersions,
Reason: moduledeps.ProviderDependencyFromState,
}
}
}
}
// This is really just the same logic as configTreeMergeStateDependencies
// but we retain this old name just to keep the symmetry until we've
// removed all of these "moduleTree..." versions that use the legacy
// configuration structs.
configTreeMergeStateDependencies(root, state)
}