terraform/command/013_config_upgrade.go

package command

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"sort"
	"strings"

	"github.com/hashicorp/hcl/v2"
	"github.com/hashicorp/hcl/v2/hclsyntax"
	"github.com/hashicorp/hcl/v2/hclwrite"
	"github.com/hashicorp/terraform/addrs"
	"github.com/hashicorp/terraform/configs"
	"github.com/hashicorp/terraform/internal/getproviders"
	"github.com/hashicorp/terraform/tfdiags"
	"github.com/zclconf/go-cty/cty"
)

// ZeroThirteenUpgradeCommand upgrades configuration files for a module
// to include explicit provider source settings
type ZeroThirteenUpgradeCommand struct {
	Meta
}

// Warning diagnostic detail message used for JSON and override config files
const skippedConfigurationFileWarning = "The %s configuration file %q was skipped, because %s files are assumed to be generated. The program that generated this file may need to be updated for changes to the configuration language."

func (c *ZeroThirteenUpgradeCommand) Run(args []string) int {
	args = c.Meta.process(args)
	flags := c.Meta.defaultFlagSet("0.13upgrade")
	flags.Usage = func() { c.Ui.Error(c.Help()) }
	if err := flags.Parse(args); err != nil {
		return 1
	}

	var diags tfdiags.Diagnostics

	var dir string
	args = flags.Args()
	switch len(args) {
	case 0:
		dir = "."
	case 1:
		dir = args[0]
	default:
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Too many arguments",
			"The command 0.13upgrade expects only a single argument, giving the directory containing the module to upgrade.",
		))
		c.showDiagnostics(diags)
		return 1
	}

	// Check for user-supplied plugin path
	var err error
	if c.pluginPath, err = c.loadPluginPath(); err != nil {
		c.Ui.Error(fmt.Sprintf("Error loading plugin path: %s", err))
		return 1
	}

	dir = c.normalizePath(dir)

	// Upgrade only if some configuration is present
	empty, err := configs.IsEmptyDir(dir)
	if err != nil {
		diags = diags.Append(fmt.Errorf("Error checking configuration: %s", err))
		return 1
	}
	if empty {
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Not a module directory",
			fmt.Sprintf("The given directory %s does not contain any Terraform configuration files.", dir),
		))
		c.showDiagnostics(diags)
		return 1
	}

	// Set up the config loader and find all the config files
	loader, err := c.initConfigLoader()
	if err != nil {
		diags = diags.Append(err)
		c.showDiagnostics(diags)
		return 1
	}
	parser := loader.Parser()
	primary, overrides, hclDiags := parser.ConfigDirFiles(dir)
	diags = diags.Append(hclDiags)
	if diags.HasErrors() {
		c.Ui.Error(strings.TrimSpace("Failed to load configuration"))
		c.showDiagnostics(diags)
		return 1
	}

	// Load and parse all primary files
	files := make(map[string]*configs.File)
	for _, path := range primary {
		// Skip JSON configuration files, because we can't rewrite them and
		// they're probably generated anyway.
		if strings.HasSuffix(strings.ToLower(path), ".json") {
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Warning,
				"JSON configuration file was not rewritten",
				fmt.Sprintf(
					skippedConfigurationFileWarning,
					"JSON",
					path,
					"JSON",
				),
			))
			continue
		}
		file, fileDiags := parser.LoadConfigFile(path)
		diags = diags.Append(fileDiags)
		if file != nil {
			files[path] = file
		}
	}
	if diags.HasErrors() {
		c.Ui.Error(strings.TrimSpace("Failed to load configuration"))
		c.showDiagnostics(diags)
		return 1
	}

	// It's not clear what the correct behaviour is for upgrading override
	// files. For now, just log that we're ignoring the file.
	for _, path := range overrides {
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Warning,
			"Override configuration file was not rewritten",
			fmt.Sprintf(
				skippedConfigurationFileWarning,
				"override",
				path,
				"override",
			),
		))
	}

	// Build up a list of required providers, uniquely by local name
	requiredProviders := make(map[string]*configs.RequiredProvider)
	var rewritePaths []string

	// Step 1: copy all explicit provider requirements across
	for path, file := range files {
		for _, rps := range file.RequiredProviders {
			rewritePaths = append(rewritePaths, path)
			for _, rp := range rps.RequiredProviders {
				if previous, exist := requiredProviders[rp.Name]; exist {
					diags = diags.Append(&hcl.Diagnostic{
						Summary:  "Duplicate required provider configuration",
						Detail:   fmt.Sprintf("Found duplicate required provider configuration for %q.Previously configured at %s", rp.Name, previous.DeclRange),
						Severity: hcl.DiagWarning,
						Context:  rps.DeclRange.Ptr(),
						Subject:  rp.DeclRange.Ptr(),
					})
				} else {
					// We're copying the struct here to ensure that any
					// mutation does not affect the original, if we rewrite
					// this file
					requiredProviders[rp.Name] = &configs.RequiredProvider{
						Name:        rp.Name,
						Source:      rp.Source,
						Type:        rp.Type,
						Requirement: rp.Requirement,
						DeclRange:   rp.DeclRange,
					}
				}
			}
		}
	}

	for _, file := range files {
		// Step 2: add missing provider requirements from provider blocks
		for _, p := range file.ProviderConfigs {
			// If no explicit provider configuration exists for the
			// provider configuration's local name, add one with a legacy
			// provider address.
			if _, exist := requiredProviders[p.Name]; !exist {
				requiredProviders[p.Name] = &configs.RequiredProvider{
					Name: p.Name,
				}
			}
		}

		// Step 3: add missing provider requirements from resources
		resources := [][]*configs.Resource{file.ManagedResources, file.DataResources}
		for _, rs := range resources {
			for _, r := range rs {
				// Find the appropriate provider local name for this resource
				var localName string

				// If there's a provider config, use that to determine the
				// local name. Otherwise use the implied provider local name
				// based on the resource's address.
				if r.ProviderConfigRef != nil {
					localName = r.ProviderConfigRef.Name
				} else {
					localName = r.Addr().ImpliedProvider()
				}

				// If no explicit provider configuration exists for this local
				// name, add one with a legacy provider address.
				if _, exist := requiredProviders[localName]; !exist {
					requiredProviders[localName] = &configs.RequiredProvider{
						Name: localName,
					}
				}
			}
		}
	}

	// We should now have a complete understanding of the provider requirements
	// stated in the config.  If there are any providers, attempt to detect
	// their sources, and rewrite the config.
	if len(requiredProviders) > 0 {
		detectDiags := c.detectProviderSources(requiredProviders)
		diags = diags.Append(detectDiags)
		if diags.HasErrors() {
			c.Ui.Error("Unable to detect sources for providers")
			c.showDiagnostics(diags)
			return 1
		}

		// Default output filename is "providers.tf"
		filename := path.Join(dir, "providers.tf")

		// Special case: if we only have one file with a required providers
		// block, output to that file instead.
		if len(rewritePaths) == 1 {
			filename = rewritePaths[0]
		}

		// Remove the output file from the list of paths we want to rewrite
		// later. Otherwise we'd delete the required providers block after
		// writing it.
		for i, path := range rewritePaths {
			if path == filename {
				rewritePaths = append(rewritePaths[:i], rewritePaths[i+1:]...)
				break
			}
		}

		var out *hclwrite.File

		// If the output file doesn't exist, just create a new empty file
		if _, err := os.Stat(filename); os.IsNotExist(err) {
			out = hclwrite.NewEmptyFile()
		} else if err != nil {
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Unable to read configuration file",
				fmt.Sprintf("Error when reading configuration file %q: %s", filename, err),
			))
			c.showDiagnostics(diags)
			return 1
		} else {
			// Configuration file already exists, so load and parse it
			config, err := ioutil.ReadFile(filename)
			if err != nil {
				diags = diags.Append(tfdiags.Sourceless(
					tfdiags.Error,
					"Unable to read configuration file",
					fmt.Sprintf("Error when reading configuration file %q: %s", filename, err),
				))
				c.showDiagnostics(diags)
				return 1
			}
			var parseDiags hcl.Diagnostics
			out, parseDiags = hclwrite.ParseConfig(config, filename, hcl.InitialPos)
			diags = diags.Append(parseDiags)
		}

		if diags.HasErrors() {
			c.showDiagnostics(diags)
			return 1
		}

		// Find all required_providers blocks, and store them alongside a map
		// back to the parent terraform block.
		var requiredProviderBlocks []*hclwrite.Block
		parentBlocks := make(map[*hclwrite.Block]*hclwrite.Block)
		root := out.Body()
		for _, rootBlock := range root.Blocks() {
			if rootBlock.Type() != "terraform" {
				continue
			}
			for _, childBlock := range rootBlock.Body().Blocks() {
				if childBlock.Type() == "required_providers" {
					requiredProviderBlocks = append(requiredProviderBlocks, childBlock)
					parentBlocks[childBlock] = rootBlock
				}
			}
		}

		// First required provider block, and the rest found in this file.
		var first *hclwrite.Block
		var rest []*hclwrite.Block

		if len(requiredProviderBlocks) > 0 {
			// If we already have one or more required provider blocks, we'll rewrite
			// the first one, and remove the rest.
			first, rest = requiredProviderBlocks[0], requiredProviderBlocks[1:]
		} else {
			// Otherwise, find or a create a terraform block, and add a new
			// empty required providers block to it.
			var tfBlock *hclwrite.Block
			for _, rootBlock := range root.Blocks() {
				if rootBlock.Type() == "terraform" {
					tfBlock = rootBlock
					break
				}
			}
			if tfBlock == nil {
				tfBlock = root.AppendNewBlock("terraform", nil)
			}
			first = tfBlock.Body().AppendNewBlock("required_providers", nil)
		}

		// Find the body of the first block to prepare for rewriting it
		body := first.Body()

		// Build a sorted list of provider local names, for consistent ordering
		var localNames []string
		for localName := range requiredProviders {
			localNames = append(localNames, localName)
		}
		sort.Strings(localNames)

		// Populate the required providers block
		for _, localName := range localNames {
			requiredProvider := requiredProviders[localName]
			var attributes = make(map[string]cty.Value)

			if !requiredProvider.Type.IsZero() {
				attributes["source"] = cty.StringVal(requiredProvider.Type.ForDisplay())
			}

			if version := requiredProvider.Requirement.Required.String(); version != "" {
				attributes["version"] = cty.StringVal(version)
			}

			var attributesObject cty.Value
			if len(attributes) > 0 {
				attributesObject = cty.ObjectVal(attributes)
			} else {
				attributesObject = cty.EmptyObjectVal
			}
			body.SetAttributeValue(localName, attributesObject)

			// If we don't have a source attribute, manually construct a commented
			// block explaining what to do
			if _, hasSource := attributes["source"]; !hasSource {
				// Generate the token stream for the required provider
				rp := body.GetAttribute(localName)
				expr := rp.Expr().BuildTokens(nil)

				// Partition the tokens into before and after the opening brace
				before, after := partitionTokensAfter(expr, hclsyntax.TokenOBrace)

				// If the value is an empty object, add a newline between the
				// braces so that the comment is not on the same line as either
				// brace.
				if len(before) == 1 && len(after) == 1 {
					newline := &hclwrite.Token{
						Type:  hclsyntax.TokenNewline,
						Bytes: []byte{'\n'},
					}
					after = append(hclwrite.Tokens{newline}, after...)
				}

				// Generate the comment and insert it at the start of the object
				comment := noSourceDetectedComment(localName)
				commentedBlock := append(before, comment...)
				commentedBlock = append(commentedBlock, after...)

				// Set the required provider object to this raw token stream
				body.SetAttributeRaw(localName, commentedBlock)
			}
		}

		// Remove the rest of the blocks (and the parent block, if it's empty)
		for _, rpBlock := range rest {
			tfBlock := parentBlocks[rpBlock]
			tfBody := tfBlock.Body()
			tfBody.RemoveBlock(rpBlock)

			// If the terraform block has no blocks and no attributes, it's
			// basically empty (aside from comments and whitespace), so it's
			// more useful to remove it than leave it in.
			if len(tfBody.Blocks()) == 0 && len(tfBody.Attributes()) == 0 {
				root.RemoveBlock(tfBlock)
			}
		}

		// Write the config back to the file
		f, err := os.OpenFile(filename, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
		if err != nil {
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Unable to open configuration file for writing",
				fmt.Sprintf("Error when reading configuration file %q: %s", filename, err),
			))
			c.showDiagnostics(diags)
			return 1
		}
		_, err = out.WriteTo(f)
		if err != nil {
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Unable to rewrite configuration file",
				fmt.Sprintf("Error when rewriting configuration file %q: %s", filename, err),
			))
			c.showDiagnostics(diags)
			return 1
		}

		// After successfully writing the new configuration, remove all other
		// required provider blocks from remaining configuration files.
		for _, path := range rewritePaths {
			// Read and parse the existing file
			config, err := ioutil.ReadFile(path)
			if err != nil {
				diags = diags.Append(tfdiags.Sourceless(
					tfdiags.Error,
					"Unable to read configuration file",
					fmt.Sprintf("Error when reading configuration file %q: %s", filename, err),
				))
				c.showDiagnostics(diags)
				return 1
			}
			file, parseDiags := hclwrite.ParseConfig(config, filename, hcl.InitialPos)
			diags = diags.Append(parseDiags)
			if diags.HasErrors() {
				c.showDiagnostics(diags)
				return 1
			}

			// Find and remove all terraform.required_providers blocks
			root := file.Body()
			for _, rootBlock := range root.Blocks() {
				if rootBlock.Type() != "terraform" {
					continue
				}
				tfBody := rootBlock.Body()
				for _, childBlock := range tfBody.Blocks() {
					if childBlock.Type() == "required_providers" {
						rootBlock.Body().RemoveBlock(childBlock)

						// If the terraform block is now empty, remove it
						if len(tfBody.Blocks()) == 0 && len(tfBody.Attributes()) == 0 {
							root.RemoveBlock(rootBlock)
						}
					}
				}
			}

			// Write the config back to the file
			f, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
			if err != nil {
				diags = diags.Append(tfdiags.Sourceless(
					tfdiags.Error,
					"Unable to open configuration file for writing",
					fmt.Sprintf("Error when reading configuration file %q: %s", filename, err),
				))
				c.showDiagnostics(diags)
				return 1
			}
			_, err = file.WriteTo(f)
			if err != nil {
				diags = diags.Append(tfdiags.Sourceless(
					tfdiags.Error,
					"Unable to rewrite configuration file",
					fmt.Sprintf("Error when rewriting configuration file %q: %s", filename, err),
				))
				c.showDiagnostics(diags)
				return 1
			}
		}
	}

	c.showDiagnostics(diags)
	if diags.HasErrors() {
		return 1
	}

	if len(diags) != 0 {
		c.Ui.Output(`-----------------------------------------------------------------------------`)
	}
	c.Ui.Output(c.Colorize().Color(`
[bold][green]Upgrade complete![reset]

Use your version control system to review the proposed changes, make any
necessary adjustments, and then commit.
`))

	return 0
}

// For providers which need a source attribute, detect the source
func (c *ZeroThirteenUpgradeCommand) detectProviderSources(requiredProviders map[string]*configs.RequiredProvider) tfdiags.Diagnostics {
	source := c.providerInstallSource()
	var diags tfdiags.Diagnostics

	for name, rp := range requiredProviders {
		// If there's already an explicit source, skip it
		if rp.Source != "" {
			continue
		}

		// Construct a legacy provider FQN using the required provider local
		// name.  This ignores any auto-generated provider FQN from the load &
		// parse process, because we know that without an explicit source it is
		// not explicitly specified.
		addr := addrs.NewLegacyProvider(name)
		p, err := getproviders.LookupLegacyProvider(addr, source)
		if err == nil {
			rp.Type = p
		} else {
			if _, ok := err.(getproviders.ErrProviderNotKnown); ok {
				// Setting the provider address to a zero value struct
				// indicates that there is no known FQN for this provider,
				// which will cause us to write an explanatory comment in the
				// HCL output advising the user what to do about this.
				rp.Type = addrs.Provider{}
			}
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Warning,
				"Could not detect provider source",
				fmt.Sprintf("Error looking up provider source for %q: %s", name, err),
			))
		}
	}

	return diags
}

// Take a list of tokens and a separator token, and return two lists: one up to
// and including the first instance of the separator, and the rest of the
// tokens. If the separator is not present, return the entire list in the first
// return value.
func partitionTokensAfter(tokens hclwrite.Tokens, separator hclsyntax.TokenType) (hclwrite.Tokens, hclwrite.Tokens) {
	for i := 0; i < len(tokens); i++ {
		if tokens[i].Type == separator {
			return tokens[0 : i+1], tokens[i+1:]
		}
	}

	return tokens, nil
}

// Generate a list of tokens for a comment explaining that a provider source
// could not be detected.
func noSourceDetectedComment(name string) hclwrite.Tokens {
	comment := fmt.Sprintf(`# TF-UPGRADE-TODO
#
# No source detected for this provider. You must add a source address
# in the following format:
#
# source = "your-registry.example.com/organization/%s"
#
# For more information, see the provider source documentation:
#
# https://www.terraform.io/docs/configuration/providers.html#provider-source`, name)

	var tokens hclwrite.Tokens
	for _, line := range strings.Split(comment, "\n") {
		tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenNewline, Bytes: []byte{'\n'}})
		tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenComment, Bytes: []byte(line)})
	}
	return tokens
}

func (c *ZeroThirteenUpgradeCommand) Help() string {
	helpText := `
Usage: terraform 0.13upgrade [module-dir]

  Generates a "providers.tf" configuration file which includes source
  configuration for every non-default provider.
`
	return strings.TrimSpace(helpText)
}

func (c *ZeroThirteenUpgradeCommand) Synopsis() string {
	return "Rewrites pre-0.13 module source code for v0.13"
}