From 4d7122a0ddcc50a5ebb2ad7d1481eaad06012144 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 21 Jan 2020 16:01:49 -0800 Subject: [PATCH] internal/getproviders: LookupLegacyProvider This is a temporary helper so that we can potentially ship the new provider installer without making a breaking change by relying on the old default namespace lookup API on the default registry to find a proper FQN for a legacy provider provider address during installation. If it's given a non-legacy provider address then it just returns the given address verbatim, so any codepath using it will also correctly handle explicit full provider addresses. This also means it will automatically self-disable once we stop using addrs.NewLegacyProvider in the config loader, because there will therefore no longer be any legacy provider addresses in the config to resolve. (They'll be "default" provider addresses instead, assumed to be under registry.terraform.io/hashicorp/* ) It's not decided yet whether we will actually introduce the new provider in a minor release, but even if we don't this API function will likely be useful for a hypothetical automatic upgrade tool to introduce explicit full provider addresses into existing modules that currently rely on the equivalent to this lookup in the current provider installer. This is dead code for now, but my intent is that it would either be called as part of new provider installation to produce an address suitable to pass to Source.AvailableVersions, or it would be called from the aforementioned hypothetical upgrade tool. Whatever happens, these functions can be removed no later than one whole major release after the new provider installer is introduced, when everyone's had the opportunity to update their legacy unqualified addresses. --- internal/getproviders/legacy_lookup.go | 121 ++++++++++++++++++ internal/getproviders/legacy_lookup_test.go | 29 +++++ internal/getproviders/multi_source.go | 17 +++ internal/getproviders/registry_client.go | 58 +++++++++ internal/getproviders/registry_client_test.go | 33 ++++- internal/getproviders/registry_source.go | 23 ++++ 6 files changed, 280 insertions(+), 1 deletion(-) create mode 100644 internal/getproviders/legacy_lookup.go create mode 100644 internal/getproviders/legacy_lookup_test.go diff --git a/internal/getproviders/legacy_lookup.go b/internal/getproviders/legacy_lookup.go new file mode 100644 index 000000000..96901abab --- /dev/null +++ b/internal/getproviders/legacy_lookup.go @@ -0,0 +1,121 @@ +package getproviders + +import ( + "fmt" + + svchost "github.com/hashicorp/terraform-svchost" + + "github.com/hashicorp/terraform/addrs" +) + +// LookupLegacyProvider attempts to resolve a legacy provider address (whose +// registry host and namespace are implied, rather than explicit) into a +// fully-qualified provider address, by asking the main Terraform registry +// to resolve it. +// +// If the given address is not a legacy provider address then it will just be +// returned verbatim without making any outgoing requests. +// +// Legacy provider lookup is possible only if the given source is either a +// *RegistrySource directly or if it is a MultiSource containing a +// *RegistrySource whose selector matching patterns include the +// public registry hostname registry.terraform.io. +// +// This is a backward-compatibility mechanism for compatibility with existing +// configurations that don't include explicit provider source addresses. New +// configurations should not rely on it, and this fallback mechanism is +// likely to be removed altogether in a future Terraform version. +func LookupLegacyProvider(addr addrs.Provider, source Source) (addrs.Provider, error) { + if addr.Namespace != "-" { + return addr, nil + } + if addr.Hostname != defaultRegistryHost { // condition above assures namespace is also "-" + // Legacy providers must always belong to the default registry host. + return addrs.Provider{}, fmt.Errorf("invalid provider type %q: legacy provider addresses must always belong to %s", addr, defaultRegistryHost) + } + + // Now we need to derive a suitable *RegistrySource from the given source, + // either directly or indirectly. This will not be possible if the user + // has configured Terraform to disable direct installation from + // registry.terraform.io; in that case, fully-qualified provider addresses + // are always required. + regSource := findLegacyProviderLookupSource(addr.Hostname, source) + if regSource == nil { + // This error message is assuming that the given Source was produced + // based on the CLI configuration, which isn't necessarily true but + // is true in all cases where this error message will ultimately be + // presented to an end-user, so good enough for now. + return addrs.Provider{}, fmt.Errorf("unqualified provider type %q cannot be resolved because direct installation from %s is disabled in the CLI configuration; declare an explicit provider namespace for this provider", addr.Type, addr.Hostname) + } + + defaultNamespace, err := regSource.LookupLegacyProviderNamespace(addr.Hostname, addr.Type) + if err != nil { + return addrs.Provider{}, err + } + + return addrs.Provider{ + Hostname: addr.Hostname, + Namespace: defaultNamespace, + Type: addr.Type, + }, nil +} + +// findLegacyProviderLookupSource tries to find a *RegistrySource that can talk +// to the given registry host in the given Source. It might be given directly, +// or it might be given indirectly via a MultiSource where the selector +// includes a wildcard for registry.terraform.io. +// +// Returns nil if the given source does not have any configured way to talk +// directly to the given host. +// +// If the given source contains multiple sources that can talk to the given +// host directly, the first one in the sequence takes preference. In practice +// it's pointless to have two direct installation sources that match the same +// hostname anyway, so this shouldn't arise in normal use. +func findLegacyProviderLookupSource(host svchost.Hostname, source Source) *RegistrySource { + switch source := source.(type) { + + case *RegistrySource: + // Easy case: the source is a registry source directly, and so we'll + // just use it. + return source + + case MultiSource: + // Trickier case: if it's a multisource then we need to scan over + // its selectors until we find one that is a *RegistrySource _and_ + // that is configured to accept arbitrary providers from the + // given hostname. + + // For our matching purposes we'll use an address that would not be + // valid as a real provider FQN and thus can only match a selector + // that has no filters at all or a selector that wildcards everything + // except the hostname, like "registry.terraform.io/*/*" + matchAddr := addrs.Provider{ + Hostname: host, + // Other fields are intentionally left empty, to make this invalid + // as a specific provider address. + } + + for _, selector := range source { + // If this source has suitable matching patterns to install from + // the given hostname then we'll recursively search inside it + // for *RegistrySource objects. + if selector.CanHandleProvider(matchAddr) { + ret := findLegacyProviderLookupSource(host, selector.Source) + if ret != nil { + return ret + } + } + } + + // If we get here then there were no selectors that are both configured + // to handle modules from the given hostname and that are registry + // sources, so we fail. + return nil + + default: + // This source cannot be and cannot contain a *RegistrySource, so + // we fail. + return nil + } +} diff --git a/internal/getproviders/legacy_lookup_test.go b/internal/getproviders/legacy_lookup_test.go new file mode 100644 index 000000000..eac853b63 --- /dev/null +++ b/internal/getproviders/legacy_lookup_test.go @@ -0,0 +1,29 @@ +package getproviders + +import ( + "testing" + + "github.com/hashicorp/terraform/addrs" +) + +func TestLookupLegacyProvider(t *testing.T) { + source, _, close := testRegistrySource(t) + defer close() + + got, err := LookupLegacyProvider( + addrs.NewLegacyProvider("legacy"), + source, + ) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + want := addrs.Provider{ + Hostname: defaultRegistryHost, + Namespace: "legacycorp", + Type: "legacy", + } + if got != want { + t.Errorf("wrong result\ngot: %#v\nwant: %#v", got, want) + } +} diff --git a/internal/getproviders/multi_source.go b/internal/getproviders/multi_source.go index 6e7fa3baf..c3bd29e77 100644 --- a/internal/getproviders/multi_source.go +++ b/internal/getproviders/multi_source.go @@ -118,6 +118,23 @@ func ParseMultiSourceMatchingPatterns(strs []string) (MultiSourceMatchingPattern return ret, nil } +// CanHandleProvider returns true if and only if the given provider address +// is both included by the selector's include patterns and _not_ excluded +// by its exclude patterns. +// +// The absense of any include patterns is treated the same as a pattern +// that matches all addresses. Exclusions take priority over inclusions. +func (s MultiSourceSelector) CanHandleProvider(addr addrs.Provider) bool { + switch { + case s.Exclude.MatchesProvider(addr): + return false + case len(s.Include) > 0: + return s.Include.MatchesProvider(addr) + default: + return true + } +} + // MatchesProvider tests whether the receiving matching patterns match with // the given concrete provider address. func (ps MultiSourceMatchingPatterns) MatchesProvider(addr addrs.Provider) bool { diff --git a/internal/getproviders/registry_client.go b/internal/getproviders/registry_client.go index dbaf7e75d..6b5be18c0 100644 --- a/internal/getproviders/registry_client.go +++ b/internal/getproviders/registry_client.go @@ -238,6 +238,64 @@ func (c *registryClient) PackageMeta(provider addrs.Provider, version Version, t return ret, nil } +// LegacyProviderCanonicalAddress returns the raw address strings produced by +// the registry when asked about the given unqualified provider type name. +// The returned namespace string is taken verbatim from the registry's response. +// +// This method exists only to allow compatibility with unqualified names +// in older configurations. New configurations should be written so as not to +// depend on it. +func (c *registryClient) LegacyProviderDefaultNamespace(typeName string) (string, error) { + endpointPath, err := url.Parse(path.Join("-", typeName)) + if err != nil { + // Should never happen because we're constructing this from + // already-validated components. + return "", err + } + endpointURL := c.baseURL.ResolveReference(endpointPath) + + req, err := http.NewRequest("GET", endpointURL.String(), nil) + if err != nil { + return "", err + } + c.addHeadersToRequest(req) + + // This is just to give us something to return in error messages. It's + // not a proper provider address. + placeholderProviderAddr := addrs.NewLegacyProvider(typeName) + + resp, err := c.httpClient.Do(req) + if err != nil { + return "", c.errQueryFailed(placeholderProviderAddr, err) + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + // Great! + case http.StatusNotFound: + return "", ErrProviderNotKnown{ + Provider: placeholderProviderAddr, + } + case http.StatusUnauthorized, http.StatusForbidden: + return "", c.errUnauthorized(placeholderProviderAddr.Hostname) + default: + return "", c.errQueryFailed(placeholderProviderAddr, errors.New(resp.Status)) + } + + type ResponseBody struct { + Namespace string + } + var body ResponseBody + + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&body); err != nil { + return "", c.errQueryFailed(placeholderProviderAddr, err) + } + + return body.Namespace, nil +} + func (c *registryClient) addHeadersToRequest(req *http.Request) { if c.creds != nil { c.creds.PrepareRequest(req) diff --git a/internal/getproviders/registry_client_test.go b/internal/getproviders/registry_client_test.go index 841b551d7..2848652af 100644 --- a/internal/getproviders/registry_client_test.go +++ b/internal/getproviders/registry_client_test.go @@ -42,6 +42,15 @@ func testServices(t *testing.T) (services *disco.Disco, baseURL string, cleanup "providers.v1": server.URL + "/fails-immediately/", }) + // We'll also permit registry.terraform.io here just because it's our + // default and has some unique features that are not allowed on any other + // hostname. It behaves the same as example.com, which should be preferred + // if you're not testing something specific to the default registry in order + // to ensure that most things are hostname-agnostic. + services.ForceHostServices(svchost.Hostname("registry.terraform.io"), map[string]interface{}{ + "providers.v1": server.URL + "/providers/v1/", + }) + return services, server.URL, func() { server.Close() } @@ -89,12 +98,34 @@ func fakeRegistryHandler(resp http.ResponseWriter, req *http.Request) { } pathParts := strings.Split(path, "/")[3:] - if len(pathParts) < 3 { + if len(pathParts) < 2 { resp.WriteHeader(404) resp.Write([]byte(`unexpected number of path parts`)) return } log.Printf("[TRACE] fake provider registry request for %#v", pathParts) + if len(pathParts) == 2 { + switch pathParts[0] + "/" + pathParts[1] { + + case "-/legacy": + // NOTE: This legacy lookup endpoint is specific to + // registry.terraform.io and not expected to work on any other + // registry host. + resp.Header().Set("Content-Type", "application/json") + resp.WriteHeader(200) + resp.Write([]byte(`{"namespace":"legacycorp"}`)) + + default: + resp.WriteHeader(404) + resp.Write([]byte(`unknown namespace or provider type for direct lookup`)) + } + } + + if len(pathParts) < 3 { + resp.WriteHeader(404) + resp.Write([]byte(`unexpected number of path parts`)) + return + } if pathParts[2] == "versions" { if len(pathParts) != 3 { diff --git a/internal/getproviders/registry_source.go b/internal/getproviders/registry_source.go index 13f5e8d96..301f431a1 100644 --- a/internal/getproviders/registry_source.go +++ b/internal/getproviders/registry_source.go @@ -90,6 +90,29 @@ func (s *RegistrySource) PackageMeta(provider addrs.Provider, version Version, t return client.PackageMeta(provider, version, target) } +// LookupLegacyProviderNamespace is a special method available only on +// RegistrySource which can deal with legacy provider addresses that contain +// only a type and leave the namespace implied. +// +// It asks the registry at the given hostname to provide a default namespace +// for the given provider type, which can be combined with the given hostname +// and type name to produce a fully-qualified provider address. +// +// Not all unqualified type names can be resolved to a default namespace. If +// the request fails, this method returns an error describing the failure. +// +// This method exists only to allow compatibility with unqualified names +// in older configurations. New configurations should be written so as not to +// depend on it, and this fallback mechanism will likely be removed altogether +// in a future Terraform version. +func (s *RegistrySource) LookupLegacyProviderNamespace(hostname svchost.Hostname, typeName string) (string, error) { + client, err := s.registryClient(hostname) + if err != nil { + return "", err + } + return client.LegacyProviderDefaultNamespace(typeName) +} + func (s *RegistrySource) registryClient(hostname svchost.Hostname) (*registryClient, error) { host, err := s.services.Discover(hostname) if err != nil {