diff --git a/command/cliconfig/cliconfig.go b/command/cliconfig/cliconfig.go index ea0bf1e57..2aecfb9d1 100644 --- a/command/cliconfig/cliconfig.go +++ b/command/cliconfig/cliconfig.go @@ -17,7 +17,7 @@ import ( "github.com/hashicorp/hcl" - "github.com/hashicorp/terraform-svchost" + svchost "github.com/hashicorp/terraform-svchost" "github.com/hashicorp/terraform/tfdiags" ) @@ -42,6 +42,12 @@ type Config struct { Credentials map[string]map[string]interface{} `hcl:"credentials"` CredentialsHelpers map[string]*ConfigCredentialsHelper `hcl:"credentials_helper"` + + // ProviderInstallation represents any provider_installation blocks + // in the configuration. Only one of these is allowed across the whole + // configuration, but we decode into a slice here so that we can handle + // that validation at validation time rather than initial decode time. + ProviderInstallation []*ProviderInstallation } // ConfigHost is the structure of the "host" nested block within the CLI @@ -91,12 +97,23 @@ func LoadConfig() (*Config, tfdiags.Diagnostics) { } } - if configDir, err := ConfigDir(); err == nil { - if info, err := os.Stat(configDir); err == nil && info.IsDir() { - dirConfig, dirDiags := loadConfigDir(configDir) - diags = diags.Append(dirDiags) - config = config.Merge(dirConfig) + // Unless the user has specifically overridden the configuration file + // location using an environment variable, we'll also load what we find + // in the config directory. We skip the config directory when source + // file override is set because we interpret the environment variable + // being set as an intention to ignore the default set of CLI config + // files because we're doing something special, like running Terraform + // in automation with a locally-customized configuration. + if cliConfigFileOverride() == "" { + if configDir, err := ConfigDir(); err == nil { + if info, err := os.Stat(configDir); err == nil && info.IsDir() { + dirConfig, dirDiags := loadConfigDir(configDir) + diags = diags.Append(dirDiags) + config = config.Merge(dirConfig) + } } + } else { + log.Printf("[DEBUG] Not reading CLI config directory because config location is overridden by environment variable") } if envConfig := EnvConfig(); envConfig != nil { @@ -136,6 +153,13 @@ func loadConfigFile(path string) (*Config, tfdiags.Diagnostics) { return result, diags } + // Deal with the provider_installation block, which is not handled using + // DecodeObject because its structure is not compatible with the + // limitations of that function. + providerInstBlocks, moreDiags := decodeProviderInstallationFromConfig(obj) + diags = diags.Append(moreDiags) + result.ProviderInstallation = providerInstBlocks + // Replace all env vars for k, v := range result.Providers { result.Providers[k] = os.ExpandEnv(v) @@ -242,6 +266,13 @@ func (c *Config) Validate() tfdiags.Diagnostics { ) } + // Should have zero or one "provider_installation" blocks + if len(c.ProviderInstallation) > 1 { + diags = diags.Append( + fmt.Errorf("No more than one provider_installation block may be specified"), + ) + } + return diags } @@ -310,17 +341,18 @@ func (c1 *Config) Merge(c2 *Config) *Config { } } + if (len(c1.ProviderInstallation) + len(c2.ProviderInstallation)) > 0 { + result.ProviderInstallation = append(result.ProviderInstallation, c1.ProviderInstallation...) + result.ProviderInstallation = append(result.ProviderInstallation, c2.ProviderInstallation...) + } + return &result } func cliConfigFile() (string, error) { mustExist := true - configFilePath := os.Getenv("TF_CLI_CONFIG_FILE") - if configFilePath == "" { - configFilePath = os.Getenv("TERRAFORM_CONFIG") - } - + configFilePath := cliConfigFileOverride() if configFilePath == "" { var err error configFilePath, err = ConfigFile() @@ -347,3 +379,11 @@ func cliConfigFile() (string, error) { log.Println("[DEBUG] File doesn't exist, but doesn't need to. Ignoring.") return "", nil } + +func cliConfigFileOverride() string { + configFilePath := os.Getenv("TF_CLI_CONFIG_FILE") + if configFilePath == "" { + configFilePath = os.Getenv("TERRAFORM_CONFIG") + } + return configFilePath +} diff --git a/command/cliconfig/cliconfig_test.go b/command/cliconfig/cliconfig_test.go index 194296c4c..a84e825b3 100644 --- a/command/cliconfig/cliconfig_test.go +++ b/command/cliconfig/cliconfig_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/davecgh/go-spew/spew" + "github.com/google/go-cmp/cmp" ) // This is the directory where our test fixtures are. @@ -169,6 +170,29 @@ func TestConfigValidate(t *testing.T) { }, 1, // no more than one credentials_helper block allowed }, + "provider_installation good none": { + &Config{ + ProviderInstallation: nil, + }, + 0, + }, + "provider_installation good one": { + &Config{ + ProviderInstallation: []*ProviderInstallation{ + {}, + }, + }, + 0, + }, + "provider_installation too many": { + &Config{ + ProviderInstallation: []*ProviderInstallation{ + {}, + {}, + }, + }, + 1, // no more than one provider_installation block allowed + }, } for name, test := range tests { @@ -209,6 +233,19 @@ func TestConfig_Merge(t *testing.T) { CredentialsHelpers: map[string]*ConfigCredentialsHelper{ "buz": {}, }, + ProviderInstallation: []*ProviderInstallation{ + { + Methods: []*ProviderInstallationMethod{ + {Location: ProviderInstallationFilesystemMirror("a")}, + {Location: ProviderInstallationFilesystemMirror("b")}, + }, + }, + { + Methods: []*ProviderInstallationMethod{ + {Location: ProviderInstallationFilesystemMirror("c")}, + }, + }, + }, } c2 := &Config{ @@ -234,6 +271,13 @@ func TestConfig_Merge(t *testing.T) { CredentialsHelpers: map[string]*ConfigCredentialsHelper{ "biz": {}, }, + ProviderInstallation: []*ProviderInstallation{ + { + Methods: []*ProviderInstallationMethod{ + {Location: ProviderInstallationFilesystemMirror("d")}, + }, + }, + }, } expected := &Config{ @@ -270,11 +314,29 @@ func TestConfig_Merge(t *testing.T) { "buz": {}, "biz": {}, }, + ProviderInstallation: []*ProviderInstallation{ + { + Methods: []*ProviderInstallationMethod{ + {Location: ProviderInstallationFilesystemMirror("a")}, + {Location: ProviderInstallationFilesystemMirror("b")}, + }, + }, + { + Methods: []*ProviderInstallationMethod{ + {Location: ProviderInstallationFilesystemMirror("c")}, + }, + }, + { + Methods: []*ProviderInstallationMethod{ + {Location: ProviderInstallationFilesystemMirror("d")}, + }, + }, + }, } actual := c1.Merge(c2) - if !reflect.DeepEqual(actual, expected) { - t.Fatalf("bad: %#v", actual) + if diff := cmp.Diff(expected, actual); diff != "" { + t.Fatalf("wrong result\n%s", diff) } } diff --git a/command/cliconfig/provider_installation.go b/command/cliconfig/provider_installation.go new file mode 100644 index 000000000..6ef8833fc --- /dev/null +++ b/command/cliconfig/provider_installation.go @@ -0,0 +1,265 @@ +package cliconfig + +import ( + "fmt" + + "github.com/hashicorp/hcl" + hclast "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/terraform/tfdiags" +) + +// ProviderInstallation is the structure of the "provider_installation" +// nested block within the CLI configuration. +type ProviderInstallation struct { + Methods []*ProviderInstallationMethod +} + +// decodeProviderInstallationFromConfig uses the HCL AST API directly to +// decode "provider_installation" blocks from the given file. +// +// This uses the HCL AST directly, rather than HCL's decoder, because the +// intended configuration structure can't be represented using the HCL +// decoder's struct tags. This structure is intended as something that would +// be relatively easier to deal with in HCL 2 once we eventually migrate +// CLI config over to that, and so this function is stricter than HCL 1's +// decoder would be in terms of exactly what configuration shape it is +// expecting. +// +// Note that this function wants the top-level file object which might or +// might not contain provider_installation blocks, not a provider_installation +// block directly itself. +func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInstallation, tfdiags.Diagnostics) { + var ret []*ProviderInstallation + var diags tfdiags.Diagnostics + + root := hclFile.Node.(*hclast.ObjectList) + + // This is a rather odd hybrid: it's a HCL 2-like decode implemented using + // the HCL 1 AST API. That makes it a bit awkward in places, but it allows + // us to mimick the strictness of HCL 2 (making a later migration easier) + // and to support a block structure that the HCL 1 decoder can't represent. + for _, block := range root.Items { + if block.Keys[0].Token.Value() != "provider_installation" { + continue + } + // HCL only tracks whether the input was JSON or native syntax inside + // individual tokens, so we'll use our block type token to decide + // and assume that the rest of the block must be written in the same + // syntax, because syntax is a whole-file idea. + isJSON := block.Keys[0].Token.JSON + if block.Assign.Line != 0 && !isJSON { + // Seems to be an attribute rather than a block + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation block", + fmt.Sprintf("The provider_installation block at %s must not be introduced with an equals sign.", block.Pos()), + )) + continue + } + if len(block.Keys) > 1 && !isJSON { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation block", + fmt.Sprintf("The provider_installation block at %s must not have any labels.", block.Pos()), + )) + } + + pi := &ProviderInstallation{} + + body, ok := block.Val.(*hclast.ObjectType) + if !ok { + // We can't get in here with native HCL syntax because we + // already checked above that we're using block syntax, but + // if we're reading JSON then our value could potentially be + // anything. + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation block", + fmt.Sprintf("The provider_installation block at %s must not be introduced with an equals sign.", block.Pos()), + )) + continue + } + + for _, methodBlock := range body.List.Items { + if methodBlock.Assign.Line != 0 && !isJSON { + // Seems to be an attribute rather than a block + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation method block", + fmt.Sprintf("The items inside the provider_installation block at %s must all be blocks.", block.Pos()), + )) + continue + } + if len(methodBlock.Keys) > 1 && !isJSON { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation method block", + fmt.Sprintf("The blocks inside the provider_installation block at %s may not have any labels.", block.Pos()), + )) + } + + methodBody, ok := methodBlock.Val.(*hclast.ObjectType) + if !ok { + // We can't get in here with native HCL syntax because we + // already checked above that we're using block syntax, but + // if we're reading JSON then our value could potentially be + // anything. + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation method block", + fmt.Sprintf("The items inside the provider_installation block at %s must all be blocks.", block.Pos()), + )) + continue + } + + methodTypeStr := methodBlock.Keys[0].Token.Value().(string) + var location ProviderInstallationLocation + var include, exclude []string + switch methodTypeStr { + case "direct": + type BodyContent struct { + Include []string `hcl:"include"` + Exclude []string `hcl:"exclude"` + } + var bodyContent BodyContent + err := hcl.DecodeObject(&bodyContent, methodBody) + if err != nil { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation method block", + fmt.Sprintf("Invalid %s block at %s: %s.", methodTypeStr, block.Pos(), err), + )) + continue + } + location = ProviderInstallationDirect + include = bodyContent.Include + exclude = bodyContent.Exclude + case "filesystem_mirror": + type BodyContent struct { + Path string `hcl:"path"` + Include []string `hcl:"include"` + Exclude []string `hcl:"exclude"` + } + var bodyContent BodyContent + err := hcl.DecodeObject(&bodyContent, methodBody) + if err != nil { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation method block", + fmt.Sprintf("Invalid %s block at %s: %s.", methodTypeStr, block.Pos(), err), + )) + continue + } + if bodyContent.Path == "" { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation method block", + fmt.Sprintf("Invalid %s block at %s: \"path\" argument is required.", methodTypeStr, block.Pos()), + )) + continue + } + location = ProviderInstallationFilesystemMirror(bodyContent.Path) + include = bodyContent.Include + exclude = bodyContent.Exclude + case "network_mirror": + type BodyContent struct { + URL string `hcl:"url"` + Include []string `hcl:"include"` + Exclude []string `hcl:"exclude"` + } + var bodyContent BodyContent + err := hcl.DecodeObject(&bodyContent, methodBody) + if err != nil { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation method block", + fmt.Sprintf("Invalid %s block at %s: %s.", methodTypeStr, block.Pos(), err), + )) + continue + } + if bodyContent.URL == "" { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation method block", + fmt.Sprintf("Invalid %s block at %s: \"url\" argument is required.", methodTypeStr, block.Pos()), + )) + continue + } + location = ProviderInstallationNetworkMirror(bodyContent.URL) + include = bodyContent.Include + exclude = bodyContent.Exclude + default: + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation method block", + fmt.Sprintf("Unknown provider installation method %q at %s.", methodTypeStr, methodBlock.Pos()), + )) + continue + } + + pi.Methods = append(pi.Methods, &ProviderInstallationMethod{ + Location: location, + Include: include, + Exclude: exclude, + }) + } + + ret = append(ret, pi) + } + + return ret, diags +} + +// ProviderInstallationMethod represents an installation method block inside +// a provider_installation block. +type ProviderInstallationMethod struct { + Location ProviderInstallationLocation + Include []string `hcl:"include"` + Exclude []string `hcl:"exclude"` +} + +// ProviderInstallationLocation is an interface type representing the +// different installation location types. The concrete implementations of +// this interface are: +// +// ProviderInstallationDirect: install from the provider's origin registry +// ProviderInstallationFilesystemMirror(dir): install from a local filesystem mirror +// ProviderInstallationNetworkMirror(host): install from a network mirror +type ProviderInstallationLocation interface { + providerInstallationLocation() +} + +type providerInstallationDirect [0]byte + +func (i providerInstallationDirect) providerInstallationLocation() {} + +// ProviderInstallationDirect is a ProviderInstallationSourceLocation +// representing installation from a provider's origin registry. +var ProviderInstallationDirect ProviderInstallationLocation = providerInstallationDirect{} + +func (i providerInstallationDirect) GoString() string { + return "cliconfig.ProviderInstallationDirect" +} + +// ProviderInstallationFilesystemMirror is a ProviderInstallationSourceLocation +// representing installation from a particular local filesystem mirror. The +// string value is the filesystem path to the mirror directory. +type ProviderInstallationFilesystemMirror string + +func (i ProviderInstallationFilesystemMirror) providerInstallationLocation() {} + +func (i ProviderInstallationFilesystemMirror) GoString() string { + return fmt.Sprintf("cliconfig.ProviderInstallationFilesystemMirror(%q)", i) +} + +// ProviderInstallationNetworkMirror is a ProviderInstallationSourceLocation +// representing installation from a particular local network mirror. The +// string value is the HTTP base URL exactly as written in the configuration, +// without any normalization. +type ProviderInstallationNetworkMirror string + +func (i ProviderInstallationNetworkMirror) providerInstallationLocation() {} + +func (i ProviderInstallationNetworkMirror) GoString() string { + return fmt.Sprintf("cliconfig.ProviderInstallationNetworkMirror(%q)", i) +} diff --git a/command/cliconfig/provider_installation_test.go b/command/cliconfig/provider_installation_test.go new file mode 100644 index 000000000..5eb0b412f --- /dev/null +++ b/command/cliconfig/provider_installation_test.go @@ -0,0 +1,70 @@ +package cliconfig + +import ( + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestLoadConfig_providerInstallation(t *testing.T) { + for _, configFile := range []string{"provider-installation", "provider-installation.json"} { + t.Run(configFile, func(t *testing.T) { + got, diags := loadConfigFile(filepath.Join(fixtureDir, configFile)) + if diags.HasErrors() { + t.Errorf("unexpected diagnostics: %s", diags.Err().Error()) + } + + want := &Config{ + ProviderInstallation: []*ProviderInstallation{ + { + Methods: []*ProviderInstallationMethod{ + { + Location: ProviderInstallationFilesystemMirror("/tmp/example1"), + Include: []string{"example.com/*/*"}, + }, + { + Location: ProviderInstallationNetworkMirror("https://tf-Mirror.example.com/"), + Include: []string{"registry.terraform.io/*/*"}, + Exclude: []string{"registry.Terraform.io/foobar/*"}, + }, + { + Location: ProviderInstallationFilesystemMirror("/tmp/example2"), + }, + { + Location: ProviderInstallationDirect, + Exclude: []string{"example.com/*/*"}, + }, + }, + }, + }, + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("wrong result\n%s", diff) + } + }) + } +} + +func TestLoadConfig_providerInstallationErrors(t *testing.T) { + _, diags := loadConfigFile(filepath.Join(fixtureDir, "provider-installation-errors")) + want := `7 problems: + +- Invalid provider_installation method block: Unknown provider installation method "not_a_thing" at 2:3. +- Invalid provider_installation method block: Invalid filesystem_mirror block at 1:1: "path" argument is required. +- Invalid provider_installation method block: Invalid network_mirror block at 1:1: "url" argument is required. +- Invalid provider_installation method block: The items inside the provider_installation block at 1:1 must all be blocks. +- Invalid provider_installation method block: The blocks inside the provider_installation block at 1:1 may not have any labels. +- Invalid provider_installation block: The provider_installation block at 9:1 must not have any labels. +- Invalid provider_installation block: The provider_installation block at 11:1 must not be introduced with an equals sign.` + + // The above error messages include only line/column location information + // and not file location information because HCL 1 does not store + // information about the filename a location belongs to. (There is a field + // for it in token.Pos but it's always an empty string in practice.) + + if got := diags.Err().Error(); got != want { + t.Errorf("wrong diagnostics\ngot:\n%s\nwant:\n%s", got, want) + } +} diff --git a/command/cliconfig/testdata/provider-installation b/command/cliconfig/testdata/provider-installation new file mode 100644 index 000000000..91dc82eab --- /dev/null +++ b/command/cliconfig/testdata/provider-installation @@ -0,0 +1,17 @@ +provider_installation { + filesystem_mirror { + path = "/tmp/example1" + include = ["example.com/*/*"] + } + network_mirror { + url = "https://tf-Mirror.example.com/" + include = ["registry.terraform.io/*/*"] + exclude = ["registry.Terraform.io/foobar/*"] + } + filesystem_mirror { + path = "/tmp/example2" + } + direct { + exclude = ["example.com/*/*"] + } +} diff --git a/command/cliconfig/testdata/provider-installation-errors b/command/cliconfig/testdata/provider-installation-errors new file mode 100644 index 000000000..8cf634e50 --- /dev/null +++ b/command/cliconfig/testdata/provider-installation-errors @@ -0,0 +1,11 @@ +provider_installation { + not_a_thing {} # unknown source type + filesystem_mirror {} # missing "path" argument + network_mirror {} # missing "host" argument + direct = {} # should be a block, not an argument + direct "what" {} # should not have a label +} + +provider_installation "what" {} # should not have a label + +provider_installation = {} # should be a block, not an argument diff --git a/command/cliconfig/testdata/provider-installation.json b/command/cliconfig/testdata/provider-installation.json new file mode 100644 index 000000000..826d68d47 --- /dev/null +++ b/command/cliconfig/testdata/provider-installation.json @@ -0,0 +1,19 @@ +{ + "provider_installation": { + "filesystem_mirror": [{ + "path": "/tmp/example1", + "include": ["example.com/*/*"] + }], + "network_mirror": [{ + "url": "https://tf-Mirror.example.com/", + "include": ["registry.terraform.io/*/*"], + "exclude": ["registry.Terraform.io/foobar/*"] + }], + "filesystem_mirror": [{ + "path": "/tmp/example2" + }], + "direct": [{ + "exclude": ["example.com/*/*"] + }] + } +} diff --git a/command/e2etest/init_test.go b/command/e2etest/init_test.go index c1ad00805..7993c4e13 100644 --- a/command/e2etest/init_test.go +++ b/command/e2etest/init_test.go @@ -173,6 +173,56 @@ func TestInitProvidersLocalOnly(t *testing.T) { } } +func TestInitProvidersCustomMethod(t *testing.T) { + t.Parallel() + + // This test should not reach out to the network if it is behaving as + // intended. If it _does_ try to access an upstream registry and encounter + // an error doing so then that's a legitimate test failure that should be + // fixed. (If it incorrectly reaches out anywhere then it's likely to be + // to the host "example.com", which is the placeholder domain we use in + // the test fixture.) + + for _, configFile := range []string{"cliconfig.tfrc", "cliconfig.tfrc.json"} { + t.Run(configFile, func(t *testing.T) { + fixturePath := filepath.Join("testdata", "custom-provider-install-method") + tf := e2e.NewBinary(terraformBin, fixturePath) + defer tf.Close() + + // Our fixture dir has a generic os_arch dir, which we need to customize + // to the actual OS/arch where this test is running in order to get the + // desired result. + fixtMachineDir := tf.Path("fs-mirror/example.com/awesomecorp/happycloud/1.2.0/os_arch") + wantMachineDir := tf.Path("fs-mirror/example.com/awesomecorp/happycloud/1.2.0/", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)) + err := os.Rename(fixtMachineDir, wantMachineDir) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + // We'll use a local CLI configuration file taken from our fixture + // directory so we can force a custom installation method config. + tf.AddEnv("TF_CLI_CONFIG_FILE=" + tf.Path(configFile)) + + stdout, stderr, err := tf.Run("init") + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + if stderr != "" { + t.Errorf("unexpected stderr output:\n%s", stderr) + } + + if !strings.Contains(stdout, "Terraform has been successfully initialized!") { + t.Errorf("success message is missing from output:\n%s", stdout) + } + + if !strings.Contains(stdout, "- Installing example.com/awesomecorp/happycloud v1.2.0") { + t.Errorf("provider download message is missing from output:\n%s", stdout) + } + }) + } +} + func TestInitProviders_pluginCache(t *testing.T) { t.Parallel() diff --git a/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc b/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc new file mode 100644 index 000000000..4b4dbefa7 --- /dev/null +++ b/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc @@ -0,0 +1,8 @@ +provider_installation { + filesystem_mirror { + path = "./fs-mirror" + } + direct { + exclude = ["example.com/*/*"] + } +} diff --git a/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc.json b/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc.json new file mode 100644 index 000000000..6e5e946b8 --- /dev/null +++ b/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc.json @@ -0,0 +1,9 @@ +{ + "provider_installation": { + "filesystem_mirror": [ + { + "path": "./fs-mirror" + } + ] + } +} diff --git a/command/e2etest/testdata/custom-provider-install-method/fs-mirror/example.com/awesomecorp/happycloud/1.2.0/os_arch/terraform-provider-happycloud_v1.2.0 b/command/e2etest/testdata/custom-provider-install-method/fs-mirror/example.com/awesomecorp/happycloud/1.2.0/os_arch/terraform-provider-happycloud_v1.2.0 new file mode 100644 index 000000000..3299bec8a --- /dev/null +++ b/command/e2etest/testdata/custom-provider-install-method/fs-mirror/example.com/awesomecorp/happycloud/1.2.0/os_arch/terraform-provider-happycloud_v1.2.0 @@ -0,0 +1,2 @@ +This is not a real plugin executable. It's just here to be discovered by the +provider installation process. diff --git a/command/e2etest/testdata/custom-provider-install-method/main.tf b/command/e2etest/testdata/custom-provider-install-method/main.tf new file mode 100644 index 000000000..a521cf07b --- /dev/null +++ b/command/e2etest/testdata/custom-provider-install-method/main.tf @@ -0,0 +1,21 @@ +# The purpose of this test is to refer to a provider whose address contains +# a hostname that is only used for namespacing purposes and doesn't actually +# have a provider registry deployed at it. +# +# A user can install such a provider in one of the implied local filesystem +# directories and Terraform should accept that as the selection for that +# provider without producing any errors about the fact that example.com +# does not have a provider registry. +# +# For this test in particular we're using the "vendor" directory that is +# the documented way to include provider plugins directly inside a +# configuration uploaded to Terraform Cloud, but this functionality applies +# to all of the implicit local filesystem search directories. + +terraform { + required_providers { + happycloud = { + source = "example.com/awesomecorp/happycloud" + } + } +} diff --git a/internal/getproviders/http_mirror_source.go b/internal/getproviders/http_mirror_source.go index 5b5bb80f1..c2dedf9db 100644 --- a/internal/getproviders/http_mirror_source.go +++ b/internal/getproviders/http_mirror_source.go @@ -1,6 +1,7 @@ package getproviders import ( + "fmt" "net/url" "github.com/hashicorp/terraform/addrs" @@ -26,13 +27,11 @@ func NewHTTPMirrorSource(baseURL *url.URL) *HTTPMirrorSource { // AvailableVersions retrieves the available versions for the given provider // from the object's underlying HTTP mirror service. func (s *HTTPMirrorSource) AvailableVersions(provider addrs.Provider) (VersionList, error) { - // TODO: Implement - panic("HTTPMirrorSource.AvailableVersions not yet implemented") + return nil, fmt.Errorf("Network-based provider mirrors are not supported in this version of Terraform") } // PackageMeta retrieves metadata for the requested provider package // from the object's underlying HTTP mirror service. func (s *HTTPMirrorSource) PackageMeta(provider addrs.Provider, version Version, target Platform) (PackageMeta, error) { - // TODO: Implement - panic("HTTPMirrorSource.PackageMeta not yet implemented") + return PackageMeta{}, fmt.Errorf("Network-based provider mirrors are not supported in this version of Terraform") } diff --git a/main.go b/main.go index f9a3b9cf8..c11432c3f 100644 --- a/main.go +++ b/main.go @@ -127,6 +127,7 @@ func wrappedMain() int { log.Printf("[INFO] CLI args: %#v", os.Args) config, diags := cliconfig.LoadConfig() + if len(diags) > 0 { // Since we haven't instantiated a command.Meta yet, we need to do // some things manually here and use some "safe" defaults for things @@ -164,11 +165,22 @@ func wrappedMain() int { services := disco.NewWithCredentialsSource(credsSrc) services.SetUserAgent(httpclient.TerraformUserAgent(version.String())) - // For the moment, we just always use the registry source to install - // direct from a registry. In future there should be a mechanism to - // configure providers sources from the CLI config, which will then - // change how we construct this object. - providerSrc := providerSource(services) + providerSrc, diags := providerSource(config.ProviderInstallation, services) + if len(diags) > 0 { + Ui.Error("There are some problems with the provider_installation configuration:") + for _, diag := range diags { + earlyColor := &colorstring.Colorize{ + Colors: colorstring.DefaultColors, + Disable: true, // Disable color to be conservative until we know better + Reset: true, + } + Ui.Error(format.Diagnostic(diag, nil, earlyColor, 78)) + } + if diags.HasErrors() { + Ui.Error("As a result of the above problems, Terraform's provider installer may not behave as intended.\n\n") + // We continue to run anyway, because most commands don't do provider installation. + } + } // Initialize the backends. backendInit.Init(services) diff --git a/provider_source.go b/provider_source.go index 8e91658a8..b65dbf439 100644 --- a/provider_source.go +++ b/provider_source.go @@ -1,7 +1,9 @@ package main import ( + "fmt" "log" + "net/url" "os" "path/filepath" @@ -11,18 +13,69 @@ import ( "github.com/hashicorp/terraform/addrs" "github.com/hashicorp/terraform/command/cliconfig" "github.com/hashicorp/terraform/internal/getproviders" + "github.com/hashicorp/terraform/tfdiags" ) // providerSource constructs a provider source based on a combination of the // CLI configuration and some default search locations. This will be the // provider source used for provider installation in the "terraform init" // command, unless overridden by the special -plugin-dir option. -func providerSource(services *disco.Disco) getproviders.Source { - // We're not yet using the CLI config here because we've not implemented - // yet the new configuration constructs to customize provider search - // locations. That'll come later. For now, we just always use the - // implicit default provider source. - return implicitProviderSource(services) +func providerSource(configs []*cliconfig.ProviderInstallation, services *disco.Disco) (getproviders.Source, tfdiags.Diagnostics) { + if len(configs) == 0 { + // If there's no explicit installation configuration then we'll build + // up an implicit one with direct registry installation along with + // some automatically-selected local filesystem mirrors. + return implicitProviderSource(services), nil + } + + // There should only be zero or one configurations, which is checked by + // the validation logic in the cliconfig package. Therefore we'll just + // ignore any additional configurations in here. + config := configs[0] + return explicitProviderSource(config, services) +} + +func explicitProviderSource(config *cliconfig.ProviderInstallation, services *disco.Disco) (getproviders.Source, tfdiags.Diagnostics) { + var diags tfdiags.Diagnostics + var searchRules []getproviders.MultiSourceSelector + + log.Printf("[DEBUG] Explicit provider installation configuration is set") + for _, methodConfig := range config.Methods { + source, moreDiags := providerSourceForCLIConfigLocation(methodConfig.Location, services) + diags = diags.Append(moreDiags) + if moreDiags.HasErrors() { + continue + } + + include, err := getproviders.ParseMultiSourceMatchingPatterns(methodConfig.Include) + if err != nil { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider source inclusion patterns", + fmt.Sprintf("CLI config specifies invalid provider inclusion patterns: %s.", err), + )) + continue + } + exclude, err := getproviders.ParseMultiSourceMatchingPatterns(methodConfig.Exclude) + if err != nil { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider source exclusion patterns", + fmt.Sprintf("CLI config specifies invalid provider exclusion patterns: %s.", err), + )) + continue + } + + searchRules = append(searchRules, getproviders.MultiSourceSelector{ + Source: source, + Include: include, + Exclude: exclude, + }) + + log.Printf("[TRACE] Selected provider installation method %#v with includes %s and excludes %s", methodConfig.Location, include, exclude) + } + + return getproviders.MultiSource(searchRules), diags } // implicitProviderSource builds a default provider source to use if there's @@ -130,3 +183,45 @@ func implicitProviderSource(services *disco.Disco) getproviders.Source { return getproviders.MultiSource(searchRules) } + +func providerSourceForCLIConfigLocation(loc cliconfig.ProviderInstallationLocation, services *disco.Disco) (getproviders.Source, tfdiags.Diagnostics) { + if loc == cliconfig.ProviderInstallationDirect { + return getproviders.NewMemoizeSource( + getproviders.NewRegistrySource(services), + ), nil + } + + switch loc := loc.(type) { + + case cliconfig.ProviderInstallationFilesystemMirror: + return getproviders.NewFilesystemMirrorSource(string(loc)), nil + + case cliconfig.ProviderInstallationNetworkMirror: + url, err := url.Parse(string(loc)) + if err != nil { + var diags tfdiags.Diagnostics + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid URL for provider installation source", + fmt.Sprintf("Cannot parse %q as a URL for a network provider mirror: %s.", string(loc), err), + )) + return nil, diags + } + if url.Scheme != "https" || url.Host == "" { + var diags tfdiags.Diagnostics + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid URL for provider installation source", + fmt.Sprintf("Cannot use %q as a URL for a network provider mirror: the mirror must be at an https: URL.", string(loc)), + )) + return nil, diags + } + return getproviders.NewHTTPMirrorSource(url), nil + + default: + // We should not get here because the set of cases above should + // be comprehensive for all of the + // cliconfig.ProviderInstallationLocation implementations. + panic(fmt.Sprintf("unexpected provider source location type %T", loc)) + } +} diff --git a/website/docs/commands/cli-config.html.markdown b/website/docs/commands/cli-config.html.markdown index fd4e9a552..9e76cb5ad 100644 --- a/website/docs/commands/cli-config.html.markdown +++ b/website/docs/commands/cli-config.html.markdown @@ -51,6 +51,14 @@ disable_checkpoint = true The following settings can be set in the CLI configuration file: +- `credentials` - configures credentials for use with Terraform Cloud or + Terraform Enterprise. See [Credentials](#credentials) below for more + information. + +- `credentials_helper` - configures an external helper program for the storage + and retrieval of credentials for Terraform Cloud or Terraform Enterprise. + See [Credentials Helpers](#credentials-helpers) below for more information. + - `disable_checkpoint` — when set to `true`, disables [upgrade and security bulletin checks](/docs/commands/index.html#upgrade-and-security-bulletin-checks) that require reaching out to HashiCorp-provided network services. @@ -60,16 +68,12 @@ The following settings can be set in the CLI configuration file: id used to de-duplicate warning messages. - `plugin_cache_dir` — enables - [plugin caching](/docs/configuration/providers.html#provider-plugin-cache) + [plugin caching](#provider-plugin-cache) and specifies, as a string, the location of the plugin cache directory. -- `credentials` - configures credentials for use with Terraform Cloud or - Terraform Enterprise. See [Credentials](#credentials) below for more - information. - -- `credentials_helper` - configures an external helper program for the storage - and retrieval of credentials for Terraform Cloud or Terraform Enterprise. - See [Credentials Helpers](#credentials-helpers) below for more information. +- `provider_installation` - customizes the installation methods used by + `terraform init` when installing provider plugins. See + [Provider Installation](#provider-installation) below for more information. ## Credentials @@ -144,14 +148,200 @@ To learn how to write and install your own credentials helpers to integrate with existing in-house credentials management systems, see [the guide to Credentials Helper internals](/docs/internals/credentials-helpers.html). -## Deprecated Settings +## Provider Installation -The following settings are supported for backward compatibility but are no -longer recommended for use: +The default way to install provider plugins is from a provider registry. The +origin registry for a provider is encoded in the provider's source address, +like `registry.terraform.io/hashicorp/aws`. For convenience in the common case, +Terraform allows omitting the hostname portion for providers on +`registry.terraform.io`, so we'd normally write `hashicorp/aws` instead in +this case. + +Downloading a plugin directly from its origin registry is not always +appropriate, though. For example, the system where you are running Terraform +may not be able to access an origin registry due to firewall restrictions +within your organization or your locality. + +To allow using Terraform providers in these situations, there are some +alternative options for making provider plugins available to Terraform which +we'll describe in the following sections. + +### Explicit Installation Method Configuration + +A `provider_installation` block in the CLI configuration allows overriding +Terraform's default installation behaviors, so you can force Terraform to use +a local mirror for some or all of the providers you intend to use. + +The general structure of a `provider_installation` block is as follows: + +```hcl +provider_installation { + filesystem_mirror { + path = "/usr/share/terraform/providers" + include = ["example.com/*/*"] + } + direct { + exclude = ["example.com/*/*"] + } +} +``` + +Each of the nested blocks inside the `provider_installation` block specifies +one installation method. Each installation method can take both `include` +and `exclude` patterns that specify which providers a particular installation +method can be used for. In the example above, we specify that any provider +whose origin registry is at `example.com` can be installed only from the +filesystem mirror at `/usr/share/terraform/providers`, while all other +providers can be installed only directly from their origin registries. + +If you set both both `include` and `exclude` for a particular installation +method, the exclusion patterns take priority. For example, including +`registry.terraform.io/hashicorp/*` but also excluding +`registry.terraform.io/hashicorp/dns` will make that installation method apply +to everything in the `hashicorp` namespace with the exception of +`hashicorp/dns`. + +As with provider source addresses in the main configuration, you can omit +the `registry.terraform.io/` prefix for providers distributed through the +public Terraform registry, even when using wildcards. For example, +`registry.terraform.io/hashicorp/*` and `hashicorp/*` are equivalent. +`*/*` is a shorthand for `registry.terraform.io/*/*`, not for +`*/*/*`. + +The following are the two supported installation method types: + +* `direct`: request information about the provider directly from its origin + registry and download over the network from the location that registry + indicates. This method expects no additional arguments. + +* `filesystem_mirror`: consult a directory on the local disk for copies of + providers. This method requires the additional argument `path` to indicate + which directory to look in. + + Terraform expects the given directory to contain a nested directory structure + where the path segments together provide metadata about the available + providers. The following two directory structures are supported: + + * Packed layout: `HOSTNAME/NAMESPACE/TYPE/terraform-provider-TYPE_VERSION_TARGET.zip` + is the distribution zip file obtained from the provider's origin registry. + * Unpacked layout: `HOSTNAME/NAMESPACE/TYPE/VERSION/TARGET` is a directory + containing the result of extracting the provider's distribution zip file. + + In both layouts, the `VERSION` is a string like `2.0.0` and the `TARGET` + specifies a particular target platform using a format like `darwin_amd64`, + `linux_arm`, `windows_amd64`, etc. + + If you use the unpacked layout, Terraform will attempt to create a symbolic + link to the mirror directory when installing the provider, rather than + creating a deep copy of the directory. The packed layout prevents this + because Terraform must extract the zip file during installation. + + You can include multiple `filesystem_mirror` blocks in order to specify + several different directories to search. + +Terraform will try all of the specified methods whose include and exclude +patterns match a given provider, and select the newest version available across +all of those methods that matches the version constraint given in each +Terraform configuration. If you have a local mirror of a particular provider +and intend Terraform to use that local mirror exclusively, you must either +remove the `direct` installation method altogether or use its `exclude` +argument to disable its use for specific providers. + +### Implied Local Mirror Directories + +If your CLI configuration does not include a `provider_installation` block at +all, Terraform produces an _implied_ configuration. The implied configuration +includes a selection of `filesystem_mirror` methods and then the `direct` +method. + +The set of directories Terraform can select as filesystem mirrors depends on +the operating system where you are running Terraform: + +* **Windows:** `%APPDATA%/HashiCorp/Terraform/plugins` +* **Mac OS X:** `~/Library/Application Support/io.terraform/plugins` and + `/Library/Application Support/io.terraform/plugins` +* **Linux and other Unix-like systems**: Terraform implements the + [XDG Base Directory](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html) + specification and appends `terraform/plugins` to all of the specified + data directories. Without any XDG environment variables set, Terraform + will use `~/.local/share/terraform/plugins`, + `/usr/local/share/terraform/plugins`, and `/usr/share/terraform/plugins`. + +Terraform will create an implied `filesystem_mirror` method block for each of +the directories indicated above that exists when Terraform starts up. + +In addition to the zero or more implied `filesystem_mirror` blocks, Terraform +also creates an implied `direct` block. Terraform will scan all of the +filesystem mirror directories to see which providers are placed there and +automatically exclude all of those providers from the implied `direct` block. +(This automatic `exclude` behavior applies only to _implicit_ `direct` blocks; +if you use explicit `provider_installation` you will need to write the intended +exclusions out yourself.) + +### Provider Plugin Cache + +By default, `terraform init` downloads plugins into a subdirectory of the +working directory so that each working directory is self-contained. As a +consequence, if you have multiple configurations that use the same provider +then a separate copy of its plugin will be downloaded for each configuration. + +Given that provider plugins can be quite large (on the order of hundreds of +megabytes), this default behavior can be inconvenient for those with slow +or metered Internet connections. Therefore Terraform optionally allows the +use of a local directory as a shared plugin cache, which then allows each +distinct plugin binary to be downloaded only once. + +To enable the plugin cache, use the `plugin_cache_dir` setting in +the CLI configuration file. For example: + +```hcl +plugin_cache_dir = "$HOME/.terraform.d/plugin-cache" +``` + +This directory must already exist before Terraform will cache plugins; +Terraform will not create the directory itself. + +Please note that on Windows it is necessary to use forward slash separators +(`/`) rather than the conventional backslash (`\`) since the configuration +file parser considers a backslash to begin an escape sequence. + +Setting this in the configuration file is the recommended approach for a +persistent setting. Alternatively, the `TF_PLUGIN_CACHE_DIR` environment +variable can be used to enable caching or to override an existing cache +directory within a particular shell session: + +```bash +export TF_PLUGIN_CACHE_DIR="$HOME/.terraform.d/plugin-cache" +``` + +When a plugin cache directory is enabled, the `terraform init` command will +still use the configured or implied installation methods to obtain metadata +about which plugins are available, but once a suitable version has been +selected it will first check to see if the chosen plugin is already available +in the cache directory. If so, Terraform will use the previously-downloaded +copy. + +If the selected plugin is not already in the cache, Terraform will download +it into the cache first and then copy it from there into the correct location +under your current working directory. When possible Terraform will use +symbolic links to avoid storing a separate copy of a cached plugin in multiple +directories. + +The plugin cache directory _must not_ also be one of the configured or implied +filesystem mirror directories, since the cache management logic conflicts with +the filesystem mirror logic when operating on the same directory. + +Terraform will never itself delete a plugin from the plugin cache once it has +been placed there. Over time, as plugins are upgraded, the cache directory may +grow to contain several unused versions which you must delete manually. + +## Removed Settings + +The following settings are supported in Terraform 0.12 and earlier but are +no longer recommended for use: * `providers` - a configuration block that allows specifying the locations of specific plugins for each named provider. This mechanism is deprecated - because it is unable to specify a version number for each plugin, and thus - it does not co-operate with the plugin versioning mechanism. Instead, - place the plugin executable files in - [the third-party plugins directory](/docs/configuration/providers.html#third-party-plugins). + because it is unable to specify a version number and source for each provider. + See [Provider Installation](#provider-installation) above for the replacement + of this setting in Terraform 0.13 and later. diff --git a/website/docs/configuration/providers.html.md b/website/docs/configuration/providers.html.md index dff4f6cc9..4d06e2066 100644 --- a/website/docs/configuration/providers.html.md +++ b/website/docs/configuration/providers.html.md @@ -233,65 +233,35 @@ from their parents. Anyone can develop and distribute their own Terraform providers. (See [Writing Custom Providers](/docs/extend/writing-custom-providers.html) for more -about provider development.) These third-party providers must be manually -installed, since `terraform init` cannot automatically download them. +about provider development.) -Install third-party providers by placing their plugin executables in the user -plugins directory. The user plugins directory is in one of the following -locations, depending on the host operating system: +The main way to distribute a provider is via a provider registry, and the main +provider registry is +[part of the public Terraform Registry](https://registry.terraform.io/browse/providers), +along with public shared modules. -Operating system | User plugins directory -------------------|----------------------- -Windows | `%APPDATA%\terraform.d\plugins` -All other systems | `~/.terraform.d/plugins` +Providers distributed via a public registry to not require any special +additional configuration to use, once you know their source addresses. You can +specify both official and third-party source addresses in the +`required_providers` block in your module: -Once a plugin is installed, `terraform init` can initialize it normally. You must run this command from the directory where the configuration files are located. +```hcl +terraform { + required_providers { + # An example third-party provider. Not actually available. + example = { + source = "example.com/examplecorp/example" + } + } +} +``` -Providers distributed by HashiCorp can also go in the user plugins directory. If -a manually installed version meets the configuration's version constraints, -Terraform will use it instead of downloading that provider. This is useful in -airgapped environments and when testing pre-release provider builds. - -### Plugin Names and Versions - -The naming scheme for provider plugins is `terraform-provider-_vX.Y.Z`, -and Terraform uses the name to understand the name and version of a particular -provider binary. - -If multiple versions of a plugin are installed, Terraform will use the newest -version that meets the configuration's version constraints. - -Third-party plugins are often distributed with an appropriate filename already -set in the distribution archive, so that they can be extracted directly into the -user plugins directory. - -### OS and Architecture Directories - -Terraform plugins are compiled for a specific operating system and architecture, -and any plugins in the root of the user plugins directory must be compiled for -the current system. - -If you use the same plugins directory on multiple systems, you can install -plugins into subdirectories with a naming scheme of `_` (for example, -`darwin_amd64`). Terraform uses plugins from the root of the plugins directory -and from the subdirectory that corresponds to the current system, ignoring -other subdirectories. - -Terraform's OS and architecture strings are the standard ones used by the Go -language. The following are the most common: - -* `darwin_amd64` -* `freebsd_386` -* `freebsd_amd64` -* `freebsd_arm` -* `linux_386` -* `linux_amd64` -* `linux_arm` -* `openbsd_386` -* `openbsd_amd64` -* `solaris_amd64` -* `windows_386` -* `windows_amd64` +Installing directly from a registry is not appropriate for all situations, +though. If you are running Terraform from a system that cannot access some or +all of the necessary origin registries, you can configure Terraform to obtain +providers from a local mirror instead. For more information, see +[Provider Installation](../commands/cli-config.html#provider-installation) +in the CLI configuration documentation. ## Provider Plugin Cache @@ -308,51 +278,3 @@ distinct plugin binary to be downloaded only once. To enable the plugin cache, use the `plugin_cache_dir` setting in [the CLI configuration file](/docs/commands/cli-config.html). -For example: - -```hcl -# (Note that the CLI configuration file is _not_ the same as the .tf files -# used to configure infrastructure.) - -plugin_cache_dir = "$HOME/.terraform.d/plugin-cache" -``` - -This directory must already exist before Terraform will cache plugins; -Terraform will not create the directory itself. - -Please note that on Windows it is necessary to use forward slash separators -(`/`) rather than the conventional backslash (`\`) since the configuration -file parser considers a backslash to begin an escape sequence. - -Setting this in the configuration file is the recommended approach for a -persistent setting. Alternatively, the `TF_PLUGIN_CACHE_DIR` environment -variable can be used to enable caching or to override an existing cache -directory within a particular shell session: - -```bash -export TF_PLUGIN_CACHE_DIR="$HOME/.terraform.d/plugin-cache" -``` - -When a plugin cache directory is enabled, the `terraform init` command will -still access the plugin distribution server to obtain metadata about which -plugins are available, but once a suitable version has been selected it will -first check to see if the selected plugin is already available in the cache -directory. If so, the already-downloaded plugin binary will be used. - -If the selected plugin is not already in the cache, it will be downloaded -into the cache first and then copied from there into the correct location -under your current working directory. - -When possible, Terraform will use hardlinks or symlinks to avoid storing -a separate copy of a cached plugin in multiple directories. At present, this -is not supported on Windows and instead a copy is always created. - -The plugin cache directory must _not_ be the third-party plugin directory -or any other directory Terraform searches for pre-installed plugins, since -the cache management logic conflicts with the normal plugin discovery logic -when operating on the same directory. - -Please note that Terraform will never itself delete a plugin from the -plugin cache once it's been placed there. Over time, as plugins are upgraded, -the cache directory may grow to contain several unused versions which must be -manually deleted.