From 082af84131ba6aceea93e818ed76eb9d6fe48ae5 Mon Sep 17 00:00:00 2001 From: Kristin Laemmert Date: Fri, 27 Jul 2018 11:59:03 -0700 Subject: [PATCH] registry: adding provider functions to registry client --- registry/client.go | 131 ++++++++++++++++++++-- registry/client_test.go | 90 ++++++++++++++++ registry/errors.go | 16 +++ registry/regsrc/terraform_provider.go | 58 ++++++++++ registry/response/provider.go | 36 +++++++ registry/response/provider_list.go | 7 ++ registry/response/terraform_provider.go | 47 ++++++++ registry/test/mock_registry.go | 137 ++++++++++++++++++++++-- 8 files changed, 508 insertions(+), 14 deletions(-) create mode 100644 registry/regsrc/terraform_provider.go create mode 100644 registry/response/provider.go create mode 100644 registry/response/provider_list.go create mode 100644 registry/response/terraform_provider.go diff --git a/registry/client.go b/registry/client.go index 8e31a6a3e..be1626cde 100644 --- a/registry/client.go +++ b/registry/client.go @@ -20,10 +20,11 @@ import ( ) const ( - xTerraformGet = "X-Terraform-Get" - xTerraformVersion = "X-Terraform-Version" - requestTimeout = 10 * time.Second - serviceID = "modules.v1" + xTerraformGet = "X-Terraform-Get" + xTerraformVersion = "X-Terraform-Version" + requestTimeout = 10 * time.Second + modulesServiceID = "modules.v1" + providersServiceID = "providers.v1" ) var tfVersion = version.String() @@ -58,7 +59,7 @@ func NewClient(services *disco.Disco, client *http.Client) *Client { } // Discover qeuries the host, and returns the url for the registry. -func (c *Client) Discover(host svchost.Hostname) *url.URL { +func (c *Client) Discover(host svchost.Hostname, serviceID string) *url.URL { service := c.services.DiscoverServiceURL(host, serviceID) if service == nil { return nil @@ -76,7 +77,7 @@ func (c *Client) Versions(module *regsrc.Module) (*response.ModuleVersions, erro return nil, err } - service := c.Discover(host) + service := c.Discover(host, modulesServiceID) if service == nil { return nil, fmt.Errorf("host %s does not provide Terraform modules", host) } @@ -149,7 +150,7 @@ func (c *Client) Location(module *regsrc.Module, version string) (string, error) return "", err } - service := c.Discover(host) + service := c.Discover(host, modulesServiceID) if service == nil { return "", fmt.Errorf("host %s does not provide Terraform modules", host.ForDisplay()) } @@ -225,3 +226,119 @@ func (c *Client) Location(module *regsrc.Module, version string) (string, error) return location, nil } + +// TerraformProviderVersions queries the registry for a provider, and returns the available versions. +func (c *Client) TerraformProviderVersions(provider *regsrc.TerraformProvider) (*response.TerraformProviderVersions, error) { + host, err := provider.SvcHost() + if err != nil { + return nil, err + } + + service := c.Discover(host, providersServiceID) + if service == nil { + return nil, fmt.Errorf("host %s does not provide Terraform providers", host) + } + + p, err := url.Parse(path.Join(provider.TerraformProvider(), "versions")) + if err != nil { + return nil, err + } + + service = service.ResolveReference(p) + + log.Printf("[DEBUG] fetching provider versions from %q", service) + + req, err := http.NewRequest("GET", service.String(), nil) + if err != nil { + return nil, err + } + + c.addRequestCreds(host, req) + req.Header.Set(xTerraformVersion, tfVersion) + + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + // OK + case http.StatusNotFound: + return nil, &errProviderNotFound{addr: provider} + default: + return nil, fmt.Errorf("error looking up provider versions: %s", resp.Status) + } + + var versions response.TerraformProviderVersions + + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&versions); err != nil { + return nil, err + } + + return &versions, nil +} + +// TerraformProviderLocation queries the registry for a provider download metadata +func (c *Client) TerraformProviderLocation(provider *regsrc.TerraformProvider, version string) (*response.TerraformProviderPlatformLocation, error) { + host, err := provider.SvcHost() + if err != nil { + return nil, err + } + + service := c.Discover(host, providersServiceID) + if service == nil { + return nil, fmt.Errorf("host %s does not provide Terraform providers", host.ForDisplay()) + } + + var p *url.URL + p, err = url.Parse(path.Join( + provider.TerraformProvider(), + version, + "download", + provider.OS, + provider.Arch, + )) + if err != nil { + return nil, err + } + + download := service.ResolveReference(p) + + log.Printf("[DEBUG] looking up provider location from %q", download) + + req, err := http.NewRequest("GET", download.String(), nil) + if err != nil { + return nil, err + } + + c.addRequestCreds(host, req) + req.Header.Set(xTerraformVersion, tfVersion) + + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var loc response.TerraformProviderPlatformLocation + + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&loc); err != nil { + return nil, err + } + + switch resp.StatusCode { + case http.StatusOK, http.StatusNoContent: + // OK + case http.StatusNotFound: + return nil, fmt.Errorf("provider %q version %q not found", provider.TerraformProvider(), version) + default: + // anything else is an error: + return nil, fmt.Errorf("error getting download location for %q: %s", provider.TerraformProvider(), resp.Status) + } + + return &loc, nil +} diff --git a/registry/client_test.go b/registry/client_test.go index 5ee712f7f..f73fa6b41 100644 --- a/registry/client_test.go +++ b/registry/client_test.go @@ -1,6 +1,7 @@ package registry import ( + "fmt" "os" "strings" "testing" @@ -199,3 +200,92 @@ func TestLookupLookupModuleError(t *testing.T) { t.Fatal("error should not include the hostname. got:", err) } } + +func TestLookupProviderVersions(t *testing.T) { + server := test.Registry() + defer server.Close() + + client := NewClient(test.Disco(server), nil, nil) + + tests := []struct { + name string + }{ + {"foo"}, + {"bar"}, + } + for _, tt := range tests { + provider, err := regsrc.NewTerraformProvider(tt.name, "", "") + resp, err := client.TerraformProviderVersions(provider) + if err != nil { + t.Fatal(err) + } + + name := fmt.Sprintf("terraform-providers/%s", tt.name) + if resp.ID != name { + t.Fatalf("expected provider name %q, got %q", name, resp.ID) + } + + if len(resp.Versions) != 2 { + t.Fatal("expected 2 versions, got", len(resp.Versions)) + } + + for _, v := range resp.Versions { + _, err := version.NewVersion(v.Version) + if err != nil { + t.Fatalf("invalid version %q: %s", v, err) + } + } + } +} + +func TestLookupProviderLocation(t *testing.T) { + server := test.Registry() + defer server.Close() + + client := NewClient(test.Disco(server), nil, nil) + + tests := []struct { + Name string + Version string + Err bool + }{ + { + "foo", + "0.2.3", + false, + }, + { + "bar", + "0.1.1", + false, + }, + { + "baz", + "0.0.0", + true, + }, + } + for _, tt := range tests { + // FIXME: the tests are set up to succeed - os/arch is not being validated at this time + p, err := regsrc.NewTerraformProvider(tt.Name, "linux", "amd64") + if err != nil { + t.Fatal(err) + } + locationMetadata, err := client.TerraformProviderLocation(p, tt.Version) + if tt.Err { + if err == nil { + t.Fatal("succeeded; want error") + } + return + } else if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + downloadURL := fmt.Sprintf("https://releases.hashicorp.com/terraform-provider-%s/%s/terraform-provider-%s.zip", tt.Name, tt.Version, tt.Name) + + if locationMetadata.DownloadURL != downloadURL { + t.Fatalf("incorrect download URL: expected %q, got %q", downloadURL, locationMetadata.DownloadURL) + } + } + +} diff --git a/registry/errors.go b/registry/errors.go index b8dcd31e3..32c1792e7 100644 --- a/registry/errors.go +++ b/registry/errors.go @@ -21,3 +21,19 @@ func IsModuleNotFound(err error) bool { _, ok := err.(*errModuleNotFound) return ok } + +type errProviderNotFound struct { + addr *regsrc.TerraformProvider +} + +func (e *errProviderNotFound) Error() string { + return fmt.Sprintf("provider %s not found", e.addr) +} + +// IsProviderNotFound returns true only if the given error is a "provider not found" +// error. This allows callers to recognize this particular error condition +// as distinct from operational errors such as poor network connectivity. +func IsProviderNotFound(err error) bool { + _, ok := err.(*errProviderNotFound) + return ok +} diff --git a/registry/regsrc/terraform_provider.go b/registry/regsrc/terraform_provider.go new file mode 100644 index 000000000..545c5a65a --- /dev/null +++ b/registry/regsrc/terraform_provider.go @@ -0,0 +1,58 @@ +package regsrc + +import ( + "fmt" + "runtime" + + "github.com/hashicorp/terraform/svchost" +) + +var ( + // DefaultProviderNamespace represents the namespace for canonical + // HashiCorp-controlled providers. + // REVIEWERS: Naming things is hard. + // * HashiCorpProviderNameSpace? + // * OfficialP...? + // * CanonicalP...? + DefaultProviderNamespace = "terraform-providers" +) + +// TerraformProvider describes a Terraform Registry Provider source. +type TerraformProvider struct { + RawHost *FriendlyHost + RawNamespace string + RawName string + OS string + Arch string +} + +// NewTerraformProvider constructs a new provider source. +func NewTerraformProvider(name, os, arch string) (*TerraformProvider, error) { + if os == "" { + os = runtime.GOOS + } + if arch == "" { + arch = runtime.GOARCH + } + + p := &TerraformProvider{ + RawHost: PublicRegistryHost, + RawNamespace: DefaultProviderNamespace, + RawName: name, + OS: os, + Arch: arch, + } + + return p, nil +} + +// Provider returns just the registry ID of the provider +func (p *TerraformProvider) TerraformProvider() string { + return fmt.Sprintf("%s/%s", p.RawNamespace, p.RawName) +} + +// SvcHost returns the svchost.Hostname for this provider. The +// default PublicRegistryHost is returned. +func (p *TerraformProvider) SvcHost() (svchost.Hostname, error) { + return svchost.ForComparison(PublicRegistryHost.Raw) +} diff --git a/registry/response/provider.go b/registry/response/provider.go new file mode 100644 index 000000000..5e8bae354 --- /dev/null +++ b/registry/response/provider.go @@ -0,0 +1,36 @@ +package response + +import ( + "time" +) + +// Provider is the response structure with the data for a single provider +// version. This is just the metadata. A full provider response will be +// ProviderDetail. +type Provider struct { + ID string `json:"id"` + + //--------------------------------------------------------------- + // Metadata about the overall provider. + + Owner string `json:"owner"` + Namespace string `json:"namespace"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Source string `json:"source"` + PublishedAt time.Time `json:"published_at"` + Downloads int `json:"downloads"` +} + +// ProviderDetail represents a Provider with full detail. +type ProviderDetail struct { + Provider + + //--------------------------------------------------------------- + // The fields below are only set when requesting this specific + // module. They are available to easily know all available versions + // without multiple API calls. + + Versions []string `json:"versions"` // All versions +} diff --git a/registry/response/provider_list.go b/registry/response/provider_list.go new file mode 100644 index 000000000..1dc7d237f --- /dev/null +++ b/registry/response/provider_list.go @@ -0,0 +1,7 @@ +package response + +// ProviderList is the response structure for a pageable list of providers. +type ProviderList struct { + Meta PaginationMeta `json:"meta"` + Providers []*Provider `json:"providers"` +} diff --git a/registry/response/terraform_provider.go b/registry/response/terraform_provider.go new file mode 100644 index 000000000..4dfb5c51e --- /dev/null +++ b/registry/response/terraform_provider.go @@ -0,0 +1,47 @@ +package response + +// TerraformProvider is the response structure for all required information for +// Terraform to choose a download URL. It must include all versions and all +// platforms for Terraform to perform version and os/arch constraint matching +// locally. +type TerraformProvider struct { + ID string `json:"id"` + Verified bool `json:"verified"` + + Versions []*TerraformProviderVersion `json:"versions"` +} + +// TerraformProviderVersion is the Terraform-specific response structure for a +// provider version. +type TerraformProviderVersion struct { + Version string `json:"version"` + Protocols []string `json:"protocols"` + + Platforms []*TerraformProviderPlatform `json:"platforms"` +} + +// TerraformProviderVersions is the Terraform-specific response structure for an +// array of provider versions +type TerraformProviderVersions struct { + ID string `json:"id"` + Versions []*TerraformProviderVersion `json:"versions"` +} + +// TerraformProviderPlatform is the Terraform-specific response structure for a +// provider platform. +type TerraformProviderPlatform struct { + OS string `json:"os"` + Arch string `json:"arch"` +} + +// TerraformProviderPlatformLocation is the Terraform-specific response +// structure for a provider platform with all details required to perform a +// download. +type TerraformProviderPlatformLocation struct { + OS string `json:"os"` + Arch string `json:"arch"` + Filename string `json:"filename"` + DownloadURL string `json:"download_url"` + ShasumsURL string `json:"shasums_url"` + ShasumsSignatureURL string `json:"shasums_signature_url"` +} diff --git a/registry/test/mock_registry.go b/registry/test/mock_registry.go index bd3d80b7f..e94e6c173 100644 --- a/registry/test/mock_registry.go +++ b/registry/test/mock_registry.go @@ -25,7 +25,8 @@ func Disco(s *httptest.Server) *disco.Disco { services := map[string]interface{}{ // Note that both with and without trailing slashes are supported behaviours // TODO: add specific tests to enumerate both possibilities. - "modules.v1": fmt.Sprintf("%s/v1/modules", s.URL), + "modules.v1": fmt.Sprintf("%s/v1/modules", s.URL), + "providers.v1": fmt.Sprintf("%s/v1/providers", s.URL), } d := disco.NewWithCredentialsSource(credsSrc) @@ -43,6 +44,15 @@ type testMod struct { version string } +// Map of provider names and location of test providers. +// Only one version for now, as we only lookup latest from the registry. +type testProvider struct { + version string + os string + arch string + url string +} + const ( testCred = "test-auth-token" ) @@ -89,6 +99,23 @@ var testMods = map[string][]testMod{ }, } +var testProviders = map[string][]testProvider{ + "terraform-providers/foo": { + { + version: "0.2.3", + url: "https://releases.hashicorp.com/terraform-provider-foo/0.2.3/terraform-provider-foo.zip", + }, + {version: "0.3.0"}, + }, + "terraform-providers/bar": { + { + version: "0.1.1", + url: "https://releases.hashicorp.com/terraform-provider-bar/0.1.1/terraform-provider-bar.zip", + }, + {version: "0.1.2"}, + }, +} + func latestVersion(versions []string) string { var col version.Collection for _, v := range versions { @@ -106,7 +133,7 @@ func latestVersion(versions []string) string { func mockRegHandler() http.Handler { mux := http.NewServeMux() - download := func(w http.ResponseWriter, r *http.Request) { + moduleDownload := func(w http.ResponseWriter, r *http.Request) { p := strings.TrimLeft(r.URL.Path, "/") // handle download request re := regexp.MustCompile(`^([-a-z]+/\w+/\w+).*/download$`) @@ -145,7 +172,7 @@ func mockRegHandler() http.Handler { return } - versions := func(w http.ResponseWriter, r *http.Request) { + moduleVersions := func(w http.ResponseWriter, r *http.Request) { p := strings.TrimLeft(r.URL.Path, "/") re := regexp.MustCompile(`^([-a-z]+/\w+/\w+)/versions$`) matches := re.FindStringSubmatch(p) @@ -197,12 +224,108 @@ func mockRegHandler() http.Handler { mux.Handle("/v1/modules/", http.StripPrefix("/v1/modules/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/download") { - download(w, r) + moduleDownload(w, r) return } if strings.HasSuffix(r.URL.Path, "/versions") { - versions(w, r) + moduleVersions(w, r) + return + } + + http.NotFound(w, r) + })), + ) + + providerDownload := func(w http.ResponseWriter, r *http.Request) { + p := strings.TrimLeft(r.URL.Path, "/") + v := strings.Split(string(p), "/") + + if len(v) != 6 { + w.WriteHeader(http.StatusBadRequest) + return + } + + name := fmt.Sprintf("%s/%s", v[0], v[1]) + + providers, ok := testProviders[name] + if !ok { + http.NotFound(w, r) + return + } + + // for this test / moment we will only return the one provider + loc := response.TerraformProviderPlatformLocation{ + DownloadURL: providers[0].url, + } + + js, err := json.Marshal(loc) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Write(js) + + } + + providerVersions := func(w http.ResponseWriter, r *http.Request) { + p := strings.TrimLeft(r.URL.Path, "/") + re := regexp.MustCompile(`^([-a-z]+/\w+)/versions$`) + matches := re.FindStringSubmatch(p) + + if len(matches) != 2 { + w.WriteHeader(http.StatusBadRequest) + return + } + + // check for auth + if strings.Contains(matches[1], "private/") { + if !strings.Contains(r.Header.Get("Authorization"), testCred) { + http.Error(w, "", http.StatusForbidden) + } + } + + name := fmt.Sprintf("%s", matches[1]) + versions, ok := testProviders[name] + if !ok { + http.NotFound(w, r) + return + } + + // only adding the single requested provider for now + // this is the minimal that any regisry is epected to support + pvs := &response.TerraformProviderVersions{ + ID: name, + } + + for _, v := range versions { + pv := &response.TerraformProviderVersion{ + Version: v.version, + } + pvs.Versions = append(pvs.Versions, pv) + } + + js, err := json.Marshal(pvs) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Write(js) + } + + mux.Handle("/v1/providers/", + http.StripPrefix("/v1/providers/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/download") { + providerDownload(w, r) + return + } + + if strings.HasSuffix(r.URL.Path, "/versions") { + providerVersions(w, r) return } @@ -212,12 +335,12 @@ func mockRegHandler() http.Handler { mux.HandleFunc("/.well-known/terraform.json", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - io.WriteString(w, `{"modules.v1":"http://localhost/v1/modules/"}`) + io.WriteString(w, `{"modules.v1":"http://localhost/v1/modules/", "providers.v1":"http://localhost/v1/providers/"}`) }) return mux } -// NewRegistry return an httptest server that mocks out some registry functionality. +// Registry returns an httptest server that mocks out some registry functionality. func Registry() *httptest.Server { return httptest.NewServer(mockRegHandler()) }