From 248a5e4523b44bc8e56786f324cbd84d92517332 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Tue, 24 Oct 2017 18:00:11 -0400 Subject: [PATCH] copy regsrc and response from registry repo keep these in one place for now --- registry/regsrc/friendly_host.go | 158 +++++++++++++++++++++ registry/regsrc/friendly_host_test.go | 122 ++++++++++++++++ registry/regsrc/module.go | 162 +++++++++++++++++++++ registry/regsrc/module_test.go | 131 +++++++++++++++++ registry/regsrc/regsrc.go | 8 ++ registry/response/module.go | 193 ++++++++++++++++++++++++++ registry/response/module_list.go | 7 + registry/response/module_provider.go | 14 ++ registry/response/module_versions.go | 131 +++++++++++++++++ registry/response/pagination.go | 65 +++++++++ registry/response/pagination_test.go | 122 ++++++++++++++++ registry/response/redirect.go | 6 + 12 files changed, 1119 insertions(+) create mode 100644 registry/regsrc/friendly_host.go create mode 100644 registry/regsrc/friendly_host_test.go create mode 100644 registry/regsrc/module.go create mode 100644 registry/regsrc/module_test.go create mode 100644 registry/regsrc/regsrc.go create mode 100644 registry/response/module.go create mode 100644 registry/response/module_list.go create mode 100644 registry/response/module_provider.go create mode 100644 registry/response/module_versions.go create mode 100644 registry/response/pagination.go create mode 100644 registry/response/pagination_test.go create mode 100644 registry/response/redirect.go diff --git a/registry/regsrc/friendly_host.go b/registry/regsrc/friendly_host.go new file mode 100644 index 000000000..28ca9b0aa --- /dev/null +++ b/registry/regsrc/friendly_host.go @@ -0,0 +1,158 @@ +package regsrc + +import ( + "regexp" + "strings" + + "golang.org/x/net/idna" +) + +var ( + // InvalidHostString is a placeholder returned when a raw host can't be + // converted by IDNA spec. It will never be returned for any host for which + // Valid() is true. + InvalidHostString = "" + + // urlLabelEndSubRe is a sub-expression that matches any character that's + // allowed at the start or end of a URL label according to RFC1123. + urlLabelEndSubRe = "[0-9A-Za-z]" + + // urlLabelEndSubRe is a sub-expression that matches any character that's + // allowed at in a non-start or end of a URL label according to RFC1123. + urlLabelMidSubRe = "[0-9A-Za-z-]" + + // urlLabelUnicodeSubRe is a sub-expression that matches any non-ascii char + // in an IDN (Unicode) display URL. It's not strict - there are only ~15k + // valid Unicode points in IDN RFC (some with conditions). We are just going + // with being liberal with matching and then erroring if we fail to convert + // to punycode later (which validates chars fully). This at least ensures + // ascii chars dissalowed by the RC1123 parts above don't become legal + // again. + urlLabelUnicodeSubRe = "[^[:ascii:]]" + + // hostLabelSubRe is the sub-expression that matches a valid hostname label. + // It does not anchor the start or end so it can be composed into more + // complex RegExps below. Note that for sanity we don't handle disallowing + // raw punycode in this regexp (esp. since re2 doesn't support negative + // lookbehind, but we can capture it's presence here to check later). + hostLabelSubRe = "" + + // Match valid initial char, or unicode char + "(?:" + urlLabelEndSubRe + "|" + urlLabelUnicodeSubRe + ")" + + // Optionally, match 0 to 61 valid URL or Unicode chars, + // followed by one valid end char or unicode char + "(?:" + + "(?:" + urlLabelMidSubRe + "|" + urlLabelUnicodeSubRe + "){0,61}" + + "(?:" + urlLabelEndSubRe + "|" + urlLabelUnicodeSubRe + ")" + + ")?" + + // hostSubRe is the sub-expression that matches a valid host prefix. + // Allows custom port. + hostSubRe = hostLabelSubRe + "(?:\\." + hostLabelSubRe + ")+(?::\\d+)?" + + // hostRe is a regexp that matches a valid host prefix. Additional + // validation of unicode strings is needed for matches. + hostRe = regexp.MustCompile("^" + hostSubRe + "$") +) + +// FriendlyHost describes a registry instance identified in source strings by a +// simple bare hostname like registry.terraform.io. +type FriendlyHost struct { + Raw string +} + +func NewFriendlyHost(host string) *FriendlyHost { + return &FriendlyHost{Raw: host} +} + +// ParseFriendlyHost attempts to parse a valid "friendly host" prefix from the +// given string. If no valid prefix is found, host will be nil and rest will +// contain the full source string. The host prefix must terminate at the end of +// the input or at the first / character. If one or more characters exist after +// the first /, they will be returned as rest (without the / delimiter). +// Hostnames containing punycode WILL be parsed successfully since they may have +// come from an internal normalized source string, however should be considered +// invalid if the string came from a user directly. This must be checked +// explicitly for user-input strings by calling Valid() on the +// returned host. +func ParseFriendlyHost(source string) (host *FriendlyHost, rest string) { + parts := strings.SplitN(source, "/", 2) + + if hostRe.MatchString(parts[0]) { + host = &FriendlyHost{Raw: parts[0]} + if len(parts) == 2 { + rest = parts[1] + } + return + } + + // No match, return whole string as rest along with nil host + rest = source + return +} + +// Valid returns whether the host prefix is considered valid in any case. +// Example of invalid prefixes might include ones that don't conform to the host +// name specifications. Not that IDN prefixes containing punycode are not valid +// input which we expect to always be in user-input or normalised display form. +func (h *FriendlyHost) Valid() bool { + if h.Display() == InvalidHostString { + return false + } + if h.Normalized() == InvalidHostString { + return false + } + if containsPuny(h.Raw) { + return false + } + return true +} + +// Display returns the host formatted for display to the user in CLI or web +// output. +func (h *FriendlyHost) Display() string { + parts := strings.SplitN(h.Raw, ":", 2) + var err error + parts[0], err = idna.Display.ToUnicode(parts[0]) + if err != nil { + return InvalidHostString + } + return strings.Join(parts, ":") +} + +// Normalized returns the host formatted for internal reference or comparison. +func (h *FriendlyHost) Normalized() string { + // For now IDNA does all the normalisation we need including case-folding + // pure ASCII to lower. But breaks if a custom port is included while we + // want to allow that and normalize comparison including it, + parts := strings.SplitN(h.Raw, ":", 2) + var err error + parts[0], err = idna.Lookup.ToASCII(parts[0]) + if err != nil { + return InvalidHostString + } + return strings.Join(parts, ":") +} + +// String returns the host formatted as the user originally typed it assuming it +// was parsed from user input. +func (h *FriendlyHost) String() string { + return h.Raw +} + +// Equal compares the FriendlyHost against another instance taking normalization +// into account. +func (h *FriendlyHost) Equal(other *FriendlyHost) bool { + if other == nil { + return false + } + return h.Normalized() == other.Normalized() +} + +func containsPuny(host string) bool { + for _, lbl := range strings.Split(host, ".") { + if strings.HasPrefix(strings.ToLower(lbl), "xn--") { + return true + } + } + return false +} diff --git a/registry/regsrc/friendly_host_test.go b/registry/regsrc/friendly_host_test.go new file mode 100644 index 000000000..e87774cfe --- /dev/null +++ b/registry/regsrc/friendly_host_test.go @@ -0,0 +1,122 @@ +package regsrc + +import ( + "strings" + "testing" +) + +func TestFriendlyHost(t *testing.T) { + tests := []struct { + name string + source string + wantHost string + wantDisplay string + wantNorm string + wantValid bool + }{ + { + name: "simple ascii", + source: "registry.terraform.io", + wantHost: "registry.terraform.io", + wantDisplay: "registry.terraform.io", + wantNorm: "registry.terraform.io", + wantValid: true, + }, + { + name: "mixed-case ascii", + source: "Registry.TerraForm.io", + wantHost: "Registry.TerraForm.io", + wantDisplay: "registry.terraform.io", // Display case folded + wantNorm: "registry.terraform.io", + wantValid: true, + }, + { + name: "IDN", + source: "ʎɹʇsıƃǝɹ.ɯɹoɟɐɹɹǝʇ.io", + wantHost: "ʎɹʇsıƃǝɹ.ɯɹoɟɐɹɹǝʇ.io", + wantDisplay: "ʎɹʇsıƃǝɹ.ɯɹoɟɐɹɹǝʇ.io", + wantNorm: "xn--s-fka0wmm0zea7g8b.xn--o-8ta85a3b1dwcda1k.io", + wantValid: true, + }, + { + name: "IDN TLD", + source: "zhongwen.中国", + wantHost: "zhongwen.中国", + wantDisplay: "zhongwen.中国", + wantNorm: "zhongwen.xn--fiqs8s", + wantValid: true, + }, + { + name: "IDN Case Folding", + source: "Испытание.com", + wantHost: "Испытание.com", // Raw input retains case + wantDisplay: "испытание.com", // Display form is unicode but case-folded + wantNorm: "xn--80akhbyknj4f.com", + wantValid: true, + }, + { + name: "Punycode is invalid as an input format", + source: "xn--s-fka0wmm0zea7g8b.xn--o-8ta85a3b1dwcda1k.io", + wantHost: "xn--s-fka0wmm0zea7g8b.xn--o-8ta85a3b1dwcda1k.io", + wantDisplay: "ʎɹʇsıƃǝɹ.ɯɹoɟɐɹɹǝʇ.io", + wantNorm: "xn--s-fka0wmm0zea7g8b.xn--o-8ta85a3b1dwcda1k.io", + wantValid: false, + }, + { + name: "non-host prefix is left alone", + source: "foo/bar/baz", + wantHost: "", + wantDisplay: "", + wantNorm: "", + wantValid: false, + }, + } + for _, tt := range tests { + // Matrix each test with prefix and total match variants + for _, sfx := range []string{"", "/", "/foo/bar/baz"} { + t.Run(tt.name+" suffix:"+sfx, func(t *testing.T) { + gotHost, gotRest := ParseFriendlyHost(tt.source + sfx) + + if gotHost == nil { + if tt.wantHost != "" { + t.Fatalf("ParseFriendlyHost() gotHost = nil, want %v", tt.wantHost) + } + // If we return nil host, the whole input string should be in rest + if gotRest != (tt.source + sfx) { + t.Fatalf("ParseFriendlyHost() was nil rest = %s, want %v", gotRest, tt.source+sfx) + } + return + } + + if tt.wantHost == "" { + t.Fatalf("ParseFriendlyHost() gotHost.Raw = %v, want nil", gotHost.Raw) + } + + if v := gotHost.String(); v != tt.wantHost { + t.Fatalf("String() = %v, want %v", v, tt.wantHost) + } + if v := gotHost.Display(); v != tt.wantDisplay { + t.Fatalf("Display() = %v, want %v", v, tt.wantDisplay) + } + if v := gotHost.Normalized(); v != tt.wantNorm { + t.Fatalf("Normalized() = %v, want %v", v, tt.wantNorm) + } + if v := gotHost.Valid(); v != tt.wantValid { + t.Fatalf("Valid() = %v, want %v", v, tt.wantValid) + } + if gotRest != strings.TrimLeft(sfx, "/") { + t.Fatalf("ParseFriendlyHost() rest = %v, want %v", gotRest, strings.TrimLeft(sfx, "/")) + } + + // Also verify that host compares equal with all the variants. + if !gotHost.Equal(&FriendlyHost{Raw: tt.wantDisplay}) { + t.Fatalf("Equal() should be true for %s and %s", tt.wantHost, tt.wantValid) + } + if !gotHost.Equal(&FriendlyHost{Raw: tt.wantNorm}) { + t.Fatalf("Equal() should be true for %s and %s", tt.wantHost, tt.wantNorm) + } + + }) + } + } +} diff --git a/registry/regsrc/module.go b/registry/regsrc/module.go new file mode 100644 index 000000000..e26abcd7f --- /dev/null +++ b/registry/regsrc/module.go @@ -0,0 +1,162 @@ +package regsrc + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +var ( + ErrInvalidModuleSource = errors.New("not a valid registry module source") + + // nameSubRe is the sub-expression that matches a valid module namespace or + // name. It's strictly a super-set of what GitHub allows for user/org and + // repo names respectively, but more restrictive than our original repo-name + // regex which allowed periods but could cause ambiguity with hostname + // prefixes. It does not anchor the start or end so it can be composed into + // more complex RegExps below. Alphanumeric with - and _ allowed in non + // leading or trailing positions. Max length 64 chars. (GitHub username is + // 38 max.) + nameSubRe = "[0-9A-Za-z](?:[0-9A-Za-z-_]{0,62}[0-9A-Za-z])?" + + // providerSubRe is the sub-expression that matches a valid provider. It + // does not anchor the start or end so it can be composed into more complex + // RegExps below. Only lowercase chars and digits are supported in practice. + // Max length 64 chars. + providerSubRe = "[0-9a-z]{1,64}" + + // moduleSourceRe is a regular expression that matches the basic + // namespace/name/provider[//...] format for registry sources. It assumes + // any FriendlyHost prefix has already been removed if present. + moduleSourceRe = regexp.MustCompile( + fmt.Sprintf("^(%s)\\/(%s)\\/(%s)(?:\\/\\/(.*))?$", + nameSubRe, nameSubRe, providerSubRe)) +) + +// Module describes a Terraform Registry Module source. +type Module struct { + // RawHost is the friendly host prefix if one was present. It might be nil + // if the original source had no host prefix which implies + // PublicRegistryHost but is distinct from having an actual pointer to + // PublicRegistryHost since it encodes the fact the original string didn't + // include a host prefix at all which is significant for recovering actual + // input not just normalized form. Most callers should access it with Host() + // which will return public registry host instance if it's nil. + RawHost *FriendlyHost + RawNamespace string + RawName string + RawProvider string + RawSubmodule string +} + +// NewModule construct a new module source from separate parts. Pass empty +// string if host or submodule are not needed. +func NewModule(host, namespace, name, provider, submodule string) *Module { + m := &Module{ + RawNamespace: namespace, + RawName: name, + RawProvider: provider, + RawSubmodule: submodule, + } + if host != "" { + m.RawHost = NewFriendlyHost(host) + } + return m +} + +// ParseModuleSource attempts to parse source as a Terraform registry module +// source. If the string is not found to be in a valid format, +// ErrInvalidModuleSource is returned. Note that this can only be used on +// "input" strings, e.g. either ones supplied by the user or potentially +// normalised but in Display form (unicode). It will fail to parse a source with +// a punycoded domain since this is not permitted input from a user. If you have +// an already normalized string internally, you can compare it without parsing +// by comparing with the normalized version of the subject with the normal +// string equality operator. +func ParseModuleSource(source string) (*Module, error) { + // See if there is a friendly host prefix. + host, rest := ParseFriendlyHost(source) + if host != nil && !host.Valid() { + return nil, ErrInvalidModuleSource + } + + matches := moduleSourceRe.FindStringSubmatch(rest) + if len(matches) < 4 { + return nil, ErrInvalidModuleSource + } + + m := &Module{ + RawHost: host, + RawNamespace: matches[1], + RawName: matches[2], + RawProvider: matches[3], + } + + if len(matches) == 5 { + m.RawSubmodule = matches[4] + } + + return m, nil +} + +// Display returns the source formatted for display to the user in CLI or web +// output. +func (m *Module) Display() string { + return m.formatWithPrefix(m.normalizedHostPrefix(m.Host().Display()), false) +} + +// Normalized returns the source formatted for internal reference or comparison. +func (m *Module) Normalized() string { + return m.formatWithPrefix(m.normalizedHostPrefix(m.Host().Normalized()), false) +} + +// String returns the source formatted as the user originally typed it assuming +// it was parsed from user input. +func (m *Module) String() string { + // Don't normalize public registry hostname - leave it exactly like the user + // input it. + hostPrefix := "" + if m.RawHost != nil { + hostPrefix = m.RawHost.String() + "/" + } + return m.formatWithPrefix(hostPrefix, true) +} + +// Equal compares the module source against another instance taking +// normalization into account. +func (m *Module) Equal(other *Module) bool { + return m.Normalized() == other.Normalized() +} + +// Host returns the FriendlyHost object describing which registry this module is +// in. If the original source string had not host component this will return the +// PublicRegistryHost. +func (m *Module) Host() *FriendlyHost { + if m.RawHost == nil { + return PublicRegistryHost + } + return m.RawHost +} + +func (m *Module) normalizedHostPrefix(host string) string { + if m.Host().Equal(PublicRegistryHost) { + return "" + } + return host + "/" +} + +func (m *Module) formatWithPrefix(hostPrefix string, preserveCase bool) string { + suffix := "" + if m.RawSubmodule != "" { + suffix = "//" + m.RawSubmodule + } + str := fmt.Sprintf("%s%s/%s/%s%s", hostPrefix, m.RawNamespace, m.RawName, + m.RawProvider, suffix) + + // lower case by default + if !preserveCase { + return strings.ToLower(str) + } + return str +} diff --git a/registry/regsrc/module_test.go b/registry/regsrc/module_test.go new file mode 100644 index 000000000..19d9dfa19 --- /dev/null +++ b/registry/regsrc/module_test.go @@ -0,0 +1,131 @@ +package regsrc + +import ( + "testing" +) + +func TestModule(t *testing.T) { + tests := []struct { + name string + source string + wantString string + wantDisplay string + wantNorm string + wantErr bool + }{ + { + name: "public registry", + source: "hashicorp/consul/aws", + wantString: "hashicorp/consul/aws", + wantDisplay: "hashicorp/consul/aws", + wantNorm: "hashicorp/consul/aws", + wantErr: false, + }, + { + name: "public registry, submodule", + source: "hashicorp/consul/aws//foo", + wantString: "hashicorp/consul/aws//foo", + wantDisplay: "hashicorp/consul/aws//foo", + wantNorm: "hashicorp/consul/aws//foo", + wantErr: false, + }, + { + name: "public registry, explicit host", + source: "registry.terraform.io/hashicorp/consul/aws", + wantString: "registry.terraform.io/hashicorp/consul/aws", + wantDisplay: "hashicorp/consul/aws", + wantNorm: "hashicorp/consul/aws", + wantErr: false, + }, + { + name: "public registry, mixed case", + source: "HashiCorp/Consul/aws", + wantString: "HashiCorp/Consul/aws", + wantDisplay: "hashicorp/consul/aws", + wantNorm: "hashicorp/consul/aws", + wantErr: false, + }, + { + name: "private registry, custom port", + source: "Example.com:1234/HashiCorp/Consul/aws", + wantString: "Example.com:1234/HashiCorp/Consul/aws", + wantDisplay: "example.com:1234/hashicorp/consul/aws", + wantNorm: "example.com:1234/hashicorp/consul/aws", + wantErr: false, + }, + { + name: "IDN registry", + source: "Испытание.com/HashiCorp/Consul/aws", + wantString: "Испытание.com/HashiCorp/Consul/aws", + wantDisplay: "испытание.com/hashicorp/consul/aws", + wantNorm: "xn--80akhbyknj4f.com/hashicorp/consul/aws", + wantErr: false, + }, + { + name: "IDN registry, submodule, custom port", + source: "Испытание.com:1234/HashiCorp/Consul/aws//Foo", + wantString: "Испытание.com:1234/HashiCorp/Consul/aws//Foo", + // Note we DO lowercase submodule names. This might causes issues on + // some filesystems (e.g. HFS+) that are case-sensitive where + // //modules/Foo and //modules/foo describe different paths, but + // it's less confusing in general just to not support that. Any user + // with a module with submodules in both cases is already asking for + // portability issues, and terraform can ensure it does + // case-insensitive search for the dir in those cases. + wantDisplay: "испытание.com:1234/hashicorp/consul/aws//foo", + wantNorm: "xn--80akhbyknj4f.com:1234/hashicorp/consul/aws//foo", + wantErr: false, + }, + { + name: "invalid host", + source: "---.com/HashiCorp/Consul/aws", + wantErr: true, + }, + { + name: "invalid format", + source: "foo/var/baz/qux", + wantErr: true, + }, + { + name: "invalid suffix", + source: "foo/var/baz?otherthing", + wantErr: true, + }, + { + name: "valid host, invalid format", + source: "foo.com/var/baz?otherthing", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseModuleSource(tt.source) + + if (err != nil) != tt.wantErr { + t.Fatalf("ParseModuleSource() error = %v, wantErr %v", err, tt.wantErr) + } + + if err != nil { + return + } + + if v := got.String(); v != tt.wantString { + t.Fatalf("String() = %v, want %v", v, tt.wantString) + } + if v := got.Display(); v != tt.wantDisplay { + t.Fatalf("Display() = %v, want %v", v, tt.wantDisplay) + } + if v := got.Normalized(); v != tt.wantNorm { + t.Fatalf("Normalized() = %v, want %v", v, tt.wantNorm) + } + + gotDisplay, err := ParseModuleSource(tt.wantDisplay) + if err != nil { + t.Fatalf("ParseModuleSource(wantDisplay) error = %v", err) + } + if !got.Equal(gotDisplay) { + t.Fatalf("Equal() failed for %s and %s", tt.source, tt.wantDisplay) + } + }) + } +} diff --git a/registry/regsrc/regsrc.go b/registry/regsrc/regsrc.go new file mode 100644 index 000000000..c430bf141 --- /dev/null +++ b/registry/regsrc/regsrc.go @@ -0,0 +1,8 @@ +// Package regsrc provides helpers for working with source strings that identify +// resources within a Terraform registry. +package regsrc + +var ( + // PublicRegistryHost is a FriendlyHost that represents the public registry. + PublicRegistryHost = NewFriendlyHost("registry.terraform.io") +) diff --git a/registry/response/module.go b/registry/response/module.go new file mode 100644 index 000000000..e3d47094b --- /dev/null +++ b/registry/response/module.go @@ -0,0 +1,193 @@ +package response + +import ( + "fmt" + "time" + + "github.com/hashicorp/terraform-registry/api/models" +) + +// Module is the response structure with the data for a single module version. +type Module struct { + ID string `json:"id"` + + //--------------------------------------------------------------- + // Metadata about the overall module. + + Owner string `json:"owner"` + Namespace string `json:"namespace"` + Name string `json:"name"` + Version string `json:"version"` + Provider string `json:"provider"` + Description string `json:"description"` + Source string `json:"source"` + PublishedAt time.Time `json:"published_at"` + Downloads int `json:"downloads"` + Verified bool `json:"verified"` +} + +// ModuleDetail represents a module in full detail. +type ModuleDetail struct { + Module + + //--------------------------------------------------------------- + // Metadata about the overall module. This is only available when + // requesting the specific module (not in list responses). + + // Root is the root module. + Root *ModuleSubmodule `json:"root"` + + // Submodules are the other submodules that are available within + // this module. + Submodules []*ModuleSubmodule `json:"submodules"` + + //--------------------------------------------------------------- + // The fields below are only set when requesting this specific + // module. They are available to easily know all available versions + // and providers without multiple API calls. + + Providers []string `json:"providers"` // All available providers + Versions []string `json:"versions"` // All versions +} + +// ModuleSubmodule is the metadata about a specific submodule within +// a module. This includes the root module as a special case. +type ModuleSubmodule struct { + Path string `json:"path"` + Readme string `json:"readme"` + Empty bool `json:"empty"` + + Inputs []*ModuleInput `json:"inputs"` + Outputs []*ModuleOutput `json:"outputs"` + Dependencies []*ModuleDep `json:"dependencies"` + Resources []*ModuleResource `json:"resources"` +} + +// ModuleInput is an input for a module. +type ModuleInput struct { + Name string `json:"name"` + Description string `json:"description"` + Default string `json:"default"` +} + +// ModuleOutput is an output for a module. +type ModuleOutput struct { + Name string `json:"name"` + Description string `json:"description"` +} + +// ModuleDep is an output for a module. +type ModuleDep struct { + Name string `json:"name"` + Source string `json:"source"` + Version string `json:"version"` +} + +// ModuleProviderDep is the output for a provider dependency +type ModuleProviderDep struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// ModuleResource is an output for a module. +type ModuleResource struct { + Name string `json:"name"` + Type string `json:"type"` +} + +// NewModule creates a Module response object from a model. +func NewModule(mv *models.ModuleVersion) *Module { + m := mv.ModuleProvider.Module + mp := mv.ModuleProvider + + // Build the full module + return &Module{ + ID: fmt.Sprintf( + "%s/%s/%s/%s", + m.Namespace, + m.Name, + mp.Provider, + mv.Version), + + // Base metadata + Owner: m.User.Username, + Namespace: m.Namespace, + Name: m.Name, + Version: mv.Version, + Provider: mv.ModuleProvider.Provider, + Description: mv.Description, + Source: mp.Source, + PublishedAt: mv.PublishedAt, + Downloads: int(mp.Downloads), + Verified: m.Verified, + } +} + +// NewModuleDetail creates a ModuleDetail response object from a model. +func NewModuleDetail(mv *models.ModuleVersion) *ModuleDetail { + m := NewModule(mv) + + // Build the submodule response objects + var submodules []*ModuleSubmodule + var submoduleRoot *ModuleSubmodule + for _, sub := range mv.Submodules { + resp := NewModuleSubmodule(&sub) + + if sub.Root() { + submoduleRoot = resp + } else { + submodules = append(submodules, resp) + } + } + return &ModuleDetail{ + Module: *m, + Root: submoduleRoot, + Submodules: submodules, + } +} + +// NewModuleSubmodule creates a ModuleSubmodule response object from a model. +func NewModuleSubmodule(m *models.ModuleSubmodule) *ModuleSubmodule { + inputs := make([]*ModuleInput, 0, len(m.Variables)) + for _, v := range m.Variables { + inputs = append(inputs, &ModuleInput{ + Name: v.Name, + Description: v.Description.String, + Default: v.Default.String, + }) + } + + outputs := make([]*ModuleOutput, 0, len(m.Outputs)) + for _, v := range m.Outputs { + outputs = append(outputs, &ModuleOutput{ + Name: v.Name, + Description: v.Description.String, + }) + } + + deps := make([]*ModuleDep, 0, len(m.Dependencies)) + for _, v := range m.Dependencies { + deps = append(deps, &ModuleDep{ + Name: v.Name, + Source: v.Source, + }) + } + + resources := make([]*ModuleResource, 0, len(m.Resources)) + for _, v := range m.Resources { + resources = append(resources, &ModuleResource{ + Name: v.Name, + Type: v.Type, + }) + } + + return &ModuleSubmodule{ + Path: m.Path, + Readme: m.Readme, + Empty: m.Empty, + Inputs: inputs, + Outputs: outputs, + Dependencies: deps, + Resources: resources, + } +} diff --git a/registry/response/module_list.go b/registry/response/module_list.go new file mode 100644 index 000000000..978374822 --- /dev/null +++ b/registry/response/module_list.go @@ -0,0 +1,7 @@ +package response + +// ModuleList is the response structure for a pageable list of modules. +type ModuleList struct { + Meta PaginationMeta `json:"meta"` + Modules []*Module `json:"modules"` +} diff --git a/registry/response/module_provider.go b/registry/response/module_provider.go new file mode 100644 index 000000000..e48499dce --- /dev/null +++ b/registry/response/module_provider.go @@ -0,0 +1,14 @@ +package response + +// ModuleProvider represents a single provider for modules. +type ModuleProvider struct { + Name string `json:"name"` + Downloads int `json:"downloads"` + ModuleCount int `json:"module_count"` +} + +// ModuleProviderList is the response structure for a pageable list of ModuleProviders. +type ModuleProviderList struct { + Meta PaginationMeta `json:"meta"` + Providers []*ModuleProvider `json:"providers"` +} diff --git a/registry/response/module_versions.go b/registry/response/module_versions.go new file mode 100644 index 000000000..d5e551e8f --- /dev/null +++ b/registry/response/module_versions.go @@ -0,0 +1,131 @@ +package response + +import ( + "github.com/hashicorp/terraform-registry/api/regsrc" + + "github.com/hashicorp/terraform-registry/api/models" +) + +// ModuleVersions is the response format that contains all metadata about module +// versions needed for terraform CLI to resolve version constraints. See RFC +// TF-042 for details on this format. +type ModuleVersions struct { + Modules []*ModuleProviderVersions `json:"modules"` +} + +// ModuleProviderVersions is the response format for a single module instance, +// containing metadata about all versions and their dependencies. +type ModuleProviderVersions struct { + Source string `json:"source"` + Versions []*ModuleVersion `json:"versions"` +} + +// ModuleVersion is the output metadata for a given version needed by CLI to +// resolve candidate versions to satisfy requirements. +type ModuleVersion struct { + Version string `json:"version"` + Root VersionSubmodule `json:"root"` + Submodules []*VersionSubmodule `json:"submodules"` +} + +// VersionSubmodule is the output metadata for a submodule within a given +// version needed by CLI to resolve candidate versions to satisfy requirements. +// When representing the Root in JSON the path is omitted. +type VersionSubmodule struct { + Path string `json:"path,omitempty"` + Providers []*ModuleProviderDep `json:"providers"` + Dependencies []*ModuleDep `json:"dependencies"` +} + +// NewModuleVersions populates a ModuleVersions response based on a slice of +// ModuleProviders. It is assumed these are fully populated with all versions +// submodules and dependencies etc, required in the response, and in the desired +// order (i.e. the first mp is the specific one requested and any others are +// optionally pre-fetched dependencies.) The host is needed to generate correct +// Source strings for all modules and must be the canonical hostname for the +// registry instance. +func NewModuleVersions(mps []*models.ModuleProvider, + host regsrc.FriendlyHost) *ModuleVersions { + + mods := make([]*ModuleProviderVersions, 0, len(mps)) + for _, mp := range mps { + mods = append(mods, NewModuleProviderVersions(mp, host)) + } + + return &ModuleVersions{ + Modules: mods, + } +} + +// NewModuleProviderVersions constructs the metadata about a specific module +// for the ModuleVersions response. +func NewModuleProviderVersions(mp *models.ModuleProvider, + host regsrc.FriendlyHost) *ModuleProviderVersions { + + src := regsrc.NewModule( + host.String(), + mp.Module.Namespace, + mp.Module.Name, + mp.Provider, + "", + ) + + versions := make([]*ModuleVersion, 0, len(mp.Versions)) + for _, mv := range mp.Versions { + versions = append(versions, NewModuleVersion(&mv)) + } + + return &ModuleProviderVersions{ + Source: src.Display(), + Versions: versions, + } +} + +// NewModuleVersion constructs the metadata about a specific module version +// for the ModuleVersions response. +func NewModuleVersion(mv *models.ModuleVersion) *ModuleVersion { + // Build the submodule response objects + var submodules []*VersionSubmodule + var submoduleRoot VersionSubmodule + for _, sub := range mv.Submodules { + resp := NewVersionSubmodule(&sub) + + if sub.Root() { + submoduleRoot = *resp + } else { + submodules = append(submodules, resp) + } + } + + return &ModuleVersion{ + Version: mv.Version, + Root: submoduleRoot, + Submodules: submodules, + } +} + +// NewVersionSubmodule constructs a representation of a submodule within a +// specific module version for the ModuleVersions response. +func NewVersionSubmodule(m *models.ModuleSubmodule) *VersionSubmodule { + providerDeps := make([]*ModuleProviderDep, 0, len(m.ProviderDependencies)) + for _, v := range m.ProviderDependencies { + providerDeps = append(providerDeps, &ModuleProviderDep{ + Name: v.Provider, + Version: v.VersionConstraints, + }) + } + + deps := make([]*ModuleDep, 0, len(m.Dependencies)) + for _, v := range m.Dependencies { + deps = append(deps, &ModuleDep{ + Name: v.Name, + Source: v.Source, + }) + } + + return &VersionSubmodule{ + Path: m.Path, + Providers: providerDeps, + Dependencies: deps, + } +} diff --git a/registry/response/pagination.go b/registry/response/pagination.go new file mode 100644 index 000000000..75a925490 --- /dev/null +++ b/registry/response/pagination.go @@ -0,0 +1,65 @@ +package response + +import ( + "net/url" + "strconv" +) + +// PaginationMeta is a structure included in responses for pagination. +type PaginationMeta struct { + Limit int `json:"limit"` + CurrentOffset int `json:"current_offset"` + NextOffset *int `json:"next_offset,omitempty"` + PrevOffset *int `json:"prev_offset,omitempty"` + NextURL string `json:"next_url,omitempty"` + PrevURL string `json:"prev_url,omitempty"` +} + +// NewPaginationMeta populates pagination meta data from result parameters +func NewPaginationMeta(offset, limit int, hasMore bool, currentURL string) PaginationMeta { + pm := PaginationMeta{ + Limit: limit, + CurrentOffset: offset, + } + + // Calculate next/prev offsets, leave nil if not valid pages + nextOffset := offset + limit + if hasMore { + pm.NextOffset = &nextOffset + } + + prevOffset := offset - limit + if prevOffset < 0 { + prevOffset = 0 + } + if prevOffset < offset { + pm.PrevOffset = &prevOffset + } + + // If URL format provided, populate URLs. Intentionally swallow URL errors for now, API should + // catch missing URLs if we call with bad URL arg (and we care about them being present). + if currentURL != "" && pm.NextOffset != nil { + pm.NextURL, _ = setQueryParam(currentURL, "offset", *pm.NextOffset, 0) + } + if currentURL != "" && pm.PrevOffset != nil { + pm.PrevURL, _ = setQueryParam(currentURL, "offset", *pm.PrevOffset, 0) + } + + return pm +} + +func setQueryParam(baseURL, key string, val, defaultVal int) (string, error) { + u, err := url.Parse(baseURL) + if err != nil { + return "", err + } + q := u.Query() + if val == defaultVal { + // elide param if it's the default value + q.Del(key) + } else { + q.Set(key, strconv.Itoa(val)) + } + u.RawQuery = q.Encode() + return u.String(), nil +} diff --git a/registry/response/pagination_test.go b/registry/response/pagination_test.go new file mode 100644 index 000000000..be862abbd --- /dev/null +++ b/registry/response/pagination_test.go @@ -0,0 +1,122 @@ +package response + +import ( + "encoding/json" + "testing" +) + +func intPtr(i int) *int { + return &i +} + +func prettyJSON(o interface{}) (string, error) { + bytes, err := json.MarshalIndent(o, "", "\t") + if err != nil { + return "", err + } + return string(bytes), nil +} + +func TestNewPaginationMeta(t *testing.T) { + type args struct { + offset int + limit int + hasMore bool + currentURL string + } + tests := []struct { + name string + args args + wantJSON string + }{ + { + name: "first page", + args: args{0, 10, true, "http://foo.com/v1/bar"}, + wantJSON: `{ + "limit": 10, + "current_offset": 0, + "next_offset": 10, + "next_url": "http://foo.com/v1/bar?offset=10" +}`, + }, + { + name: "second page", + args: args{10, 10, true, "http://foo.com/v1/bar"}, + wantJSON: `{ + "limit": 10, + "current_offset": 10, + "next_offset": 20, + "prev_offset": 0, + "next_url": "http://foo.com/v1/bar?offset=20", + "prev_url": "http://foo.com/v1/bar" +}`, + }, + { + name: "last page", + args: args{40, 10, false, "http://foo.com/v1/bar"}, + wantJSON: `{ + "limit": 10, + "current_offset": 40, + "prev_offset": 30, + "prev_url": "http://foo.com/v1/bar?offset=30" +}`, + }, + { + name: "misaligned start ending exactly on boundary", + args: args{32, 10, false, "http://foo.com/v1/bar"}, + wantJSON: `{ + "limit": 10, + "current_offset": 32, + "prev_offset": 22, + "prev_url": "http://foo.com/v1/bar?offset=22" +}`, + }, + { + name: "misaligned start partially through first page", + args: args{5, 12, true, "http://foo.com/v1/bar"}, + wantJSON: `{ + "limit": 12, + "current_offset": 5, + "next_offset": 17, + "prev_offset": 0, + "next_url": "http://foo.com/v1/bar?offset=17", + "prev_url": "http://foo.com/v1/bar" +}`, + }, + { + name: "no current URL", + args: args{10, 10, true, ""}, + wantJSON: `{ + "limit": 10, + "current_offset": 10, + "next_offset": 20, + "prev_offset": 0 +}`, + }, + { + name: "#58 regression test", + args: args{1, 3, true, ""}, + wantJSON: `{ + "limit": 3, + "current_offset": 1, + "next_offset": 4, + "prev_offset": 0 +}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NewPaginationMeta(tt.args.offset, tt.args.limit, tt.args.hasMore, + tt.args.currentURL) + gotJSON, err := prettyJSON(got) + if err != nil { + t.Fatalf("failed to marshal PaginationMeta to JSON: %s", err) + } + if gotJSON != tt.wantJSON { + // prettyJSON makes debugging easier due to the annoying pointer-to-ints, but it + // also implicitly tests JSON marshalling as we can see if it's omitting fields etc. + t.Fatalf("NewPaginationMeta() =\n%s\n want:\n%s\n", gotJSON, tt.wantJSON) + } + }) + } +} diff --git a/registry/response/redirect.go b/registry/response/redirect.go new file mode 100644 index 000000000..d5eb49ba6 --- /dev/null +++ b/registry/response/redirect.go @@ -0,0 +1,6 @@ +package response + +// Redirect causes the frontend to perform a window redirect. +type Redirect struct { + URL string `json:"url"` +}