From 248a5e4523b44bc8e56786f324cbd84d92517332 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Tue, 24 Oct 2017 18:00:11 -0400 Subject: [PATCH 1/7] copy regsrc and response from registry repo keep these in one place for now --- registry/regsrc/friendly_host.go | 158 +++++++++++++++++++++ registry/regsrc/friendly_host_test.go | 122 ++++++++++++++++ registry/regsrc/module.go | 162 +++++++++++++++++++++ registry/regsrc/module_test.go | 131 +++++++++++++++++ registry/regsrc/regsrc.go | 8 ++ registry/response/module.go | 193 ++++++++++++++++++++++++++ registry/response/module_list.go | 7 + registry/response/module_provider.go | 14 ++ registry/response/module_versions.go | 131 +++++++++++++++++ registry/response/pagination.go | 65 +++++++++ registry/response/pagination_test.go | 122 ++++++++++++++++ registry/response/redirect.go | 6 + 12 files changed, 1119 insertions(+) create mode 100644 registry/regsrc/friendly_host.go create mode 100644 registry/regsrc/friendly_host_test.go create mode 100644 registry/regsrc/module.go create mode 100644 registry/regsrc/module_test.go create mode 100644 registry/regsrc/regsrc.go create mode 100644 registry/response/module.go create mode 100644 registry/response/module_list.go create mode 100644 registry/response/module_provider.go create mode 100644 registry/response/module_versions.go create mode 100644 registry/response/pagination.go create mode 100644 registry/response/pagination_test.go create mode 100644 registry/response/redirect.go diff --git a/registry/regsrc/friendly_host.go b/registry/regsrc/friendly_host.go new file mode 100644 index 000000000..28ca9b0aa --- /dev/null +++ b/registry/regsrc/friendly_host.go @@ -0,0 +1,158 @@ +package regsrc + +import ( + "regexp" + "strings" + + "golang.org/x/net/idna" +) + +var ( + // InvalidHostString is a placeholder returned when a raw host can't be + // converted by IDNA spec. It will never be returned for any host for which + // Valid() is true. + InvalidHostString = "" + + // urlLabelEndSubRe is a sub-expression that matches any character that's + // allowed at the start or end of a URL label according to RFC1123. + urlLabelEndSubRe = "[0-9A-Za-z]" + + // urlLabelEndSubRe is a sub-expression that matches any character that's + // allowed at in a non-start or end of a URL label according to RFC1123. + urlLabelMidSubRe = "[0-9A-Za-z-]" + + // urlLabelUnicodeSubRe is a sub-expression that matches any non-ascii char + // in an IDN (Unicode) display URL. It's not strict - there are only ~15k + // valid Unicode points in IDN RFC (some with conditions). We are just going + // with being liberal with matching and then erroring if we fail to convert + // to punycode later (which validates chars fully). This at least ensures + // ascii chars dissalowed by the RC1123 parts above don't become legal + // again. + urlLabelUnicodeSubRe = "[^[:ascii:]]" + + // hostLabelSubRe is the sub-expression that matches a valid hostname label. + // It does not anchor the start or end so it can be composed into more + // complex RegExps below. Note that for sanity we don't handle disallowing + // raw punycode in this regexp (esp. since re2 doesn't support negative + // lookbehind, but we can capture it's presence here to check later). + hostLabelSubRe = "" + + // Match valid initial char, or unicode char + "(?:" + urlLabelEndSubRe + "|" + urlLabelUnicodeSubRe + ")" + + // Optionally, match 0 to 61 valid URL or Unicode chars, + // followed by one valid end char or unicode char + "(?:" + + "(?:" + urlLabelMidSubRe + "|" + urlLabelUnicodeSubRe + "){0,61}" + + "(?:" + urlLabelEndSubRe + "|" + urlLabelUnicodeSubRe + ")" + + ")?" + + // hostSubRe is the sub-expression that matches a valid host prefix. + // Allows custom port. + hostSubRe = hostLabelSubRe + "(?:\\." + hostLabelSubRe + ")+(?::\\d+)?" + + // hostRe is a regexp that matches a valid host prefix. Additional + // validation of unicode strings is needed for matches. + hostRe = regexp.MustCompile("^" + hostSubRe + "$") +) + +// FriendlyHost describes a registry instance identified in source strings by a +// simple bare hostname like registry.terraform.io. +type FriendlyHost struct { + Raw string +} + +func NewFriendlyHost(host string) *FriendlyHost { + return &FriendlyHost{Raw: host} +} + +// ParseFriendlyHost attempts to parse a valid "friendly host" prefix from the +// given string. If no valid prefix is found, host will be nil and rest will +// contain the full source string. The host prefix must terminate at the end of +// the input or at the first / character. If one or more characters exist after +// the first /, they will be returned as rest (without the / delimiter). +// Hostnames containing punycode WILL be parsed successfully since they may have +// come from an internal normalized source string, however should be considered +// invalid if the string came from a user directly. This must be checked +// explicitly for user-input strings by calling Valid() on the +// returned host. +func ParseFriendlyHost(source string) (host *FriendlyHost, rest string) { + parts := strings.SplitN(source, "/", 2) + + if hostRe.MatchString(parts[0]) { + host = &FriendlyHost{Raw: parts[0]} + if len(parts) == 2 { + rest = parts[1] + } + return + } + + // No match, return whole string as rest along with nil host + rest = source + return +} + +// Valid returns whether the host prefix is considered valid in any case. +// Example of invalid prefixes might include ones that don't conform to the host +// name specifications. Not that IDN prefixes containing punycode are not valid +// input which we expect to always be in user-input or normalised display form. +func (h *FriendlyHost) Valid() bool { + if h.Display() == InvalidHostString { + return false + } + if h.Normalized() == InvalidHostString { + return false + } + if containsPuny(h.Raw) { + return false + } + return true +} + +// Display returns the host formatted for display to the user in CLI or web +// output. +func (h *FriendlyHost) Display() string { + parts := strings.SplitN(h.Raw, ":", 2) + var err error + parts[0], err = idna.Display.ToUnicode(parts[0]) + if err != nil { + return InvalidHostString + } + return strings.Join(parts, ":") +} + +// Normalized returns the host formatted for internal reference or comparison. +func (h *FriendlyHost) Normalized() string { + // For now IDNA does all the normalisation we need including case-folding + // pure ASCII to lower. But breaks if a custom port is included while we + // want to allow that and normalize comparison including it, + parts := strings.SplitN(h.Raw, ":", 2) + var err error + parts[0], err = idna.Lookup.ToASCII(parts[0]) + if err != nil { + return InvalidHostString + } + return strings.Join(parts, ":") +} + +// String returns the host formatted as the user originally typed it assuming it +// was parsed from user input. +func (h *FriendlyHost) String() string { + return h.Raw +} + +// Equal compares the FriendlyHost against another instance taking normalization +// into account. +func (h *FriendlyHost) Equal(other *FriendlyHost) bool { + if other == nil { + return false + } + return h.Normalized() == other.Normalized() +} + +func containsPuny(host string) bool { + for _, lbl := range strings.Split(host, ".") { + if strings.HasPrefix(strings.ToLower(lbl), "xn--") { + return true + } + } + return false +} diff --git a/registry/regsrc/friendly_host_test.go b/registry/regsrc/friendly_host_test.go new file mode 100644 index 000000000..e87774cfe --- /dev/null +++ b/registry/regsrc/friendly_host_test.go @@ -0,0 +1,122 @@ +package regsrc + +import ( + "strings" + "testing" +) + +func TestFriendlyHost(t *testing.T) { + tests := []struct { + name string + source string + wantHost string + wantDisplay string + wantNorm string + wantValid bool + }{ + { + name: "simple ascii", + source: "registry.terraform.io", + wantHost: "registry.terraform.io", + wantDisplay: "registry.terraform.io", + wantNorm: "registry.terraform.io", + wantValid: true, + }, + { + name: "mixed-case ascii", + source: "Registry.TerraForm.io", + wantHost: "Registry.TerraForm.io", + wantDisplay: "registry.terraform.io", // Display case folded + wantNorm: "registry.terraform.io", + wantValid: true, + }, + { + name: "IDN", + source: "ʎɹʇsıƃǝɹ.ɯɹoɟɐɹɹǝʇ.io", + wantHost: "ʎɹʇsıƃǝɹ.ɯɹoɟɐɹɹǝʇ.io", + wantDisplay: "ʎɹʇsıƃǝɹ.ɯɹoɟɐɹɹǝʇ.io", + wantNorm: "xn--s-fka0wmm0zea7g8b.xn--o-8ta85a3b1dwcda1k.io", + wantValid: true, + }, + { + name: "IDN TLD", + source: "zhongwen.中国", + wantHost: "zhongwen.中国", + wantDisplay: "zhongwen.中国", + wantNorm: "zhongwen.xn--fiqs8s", + wantValid: true, + }, + { + name: "IDN Case Folding", + source: "Испытание.com", + wantHost: "Испытание.com", // Raw input retains case + wantDisplay: "испытание.com", // Display form is unicode but case-folded + wantNorm: "xn--80akhbyknj4f.com", + wantValid: true, + }, + { + name: "Punycode is invalid as an input format", + source: "xn--s-fka0wmm0zea7g8b.xn--o-8ta85a3b1dwcda1k.io", + wantHost: "xn--s-fka0wmm0zea7g8b.xn--o-8ta85a3b1dwcda1k.io", + wantDisplay: "ʎɹʇsıƃǝɹ.ɯɹoɟɐɹɹǝʇ.io", + wantNorm: "xn--s-fka0wmm0zea7g8b.xn--o-8ta85a3b1dwcda1k.io", + wantValid: false, + }, + { + name: "non-host prefix is left alone", + source: "foo/bar/baz", + wantHost: "", + wantDisplay: "", + wantNorm: "", + wantValid: false, + }, + } + for _, tt := range tests { + // Matrix each test with prefix and total match variants + for _, sfx := range []string{"", "/", "/foo/bar/baz"} { + t.Run(tt.name+" suffix:"+sfx, func(t *testing.T) { + gotHost, gotRest := ParseFriendlyHost(tt.source + sfx) + + if gotHost == nil { + if tt.wantHost != "" { + t.Fatalf("ParseFriendlyHost() gotHost = nil, want %v", tt.wantHost) + } + // If we return nil host, the whole input string should be in rest + if gotRest != (tt.source + sfx) { + t.Fatalf("ParseFriendlyHost() was nil rest = %s, want %v", gotRest, tt.source+sfx) + } + return + } + + if tt.wantHost == "" { + t.Fatalf("ParseFriendlyHost() gotHost.Raw = %v, want nil", gotHost.Raw) + } + + if v := gotHost.String(); v != tt.wantHost { + t.Fatalf("String() = %v, want %v", v, tt.wantHost) + } + if v := gotHost.Display(); v != tt.wantDisplay { + t.Fatalf("Display() = %v, want %v", v, tt.wantDisplay) + } + if v := gotHost.Normalized(); v != tt.wantNorm { + t.Fatalf("Normalized() = %v, want %v", v, tt.wantNorm) + } + if v := gotHost.Valid(); v != tt.wantValid { + t.Fatalf("Valid() = %v, want %v", v, tt.wantValid) + } + if gotRest != strings.TrimLeft(sfx, "/") { + t.Fatalf("ParseFriendlyHost() rest = %v, want %v", gotRest, strings.TrimLeft(sfx, "/")) + } + + // Also verify that host compares equal with all the variants. + if !gotHost.Equal(&FriendlyHost{Raw: tt.wantDisplay}) { + t.Fatalf("Equal() should be true for %s and %s", tt.wantHost, tt.wantValid) + } + if !gotHost.Equal(&FriendlyHost{Raw: tt.wantNorm}) { + t.Fatalf("Equal() should be true for %s and %s", tt.wantHost, tt.wantNorm) + } + + }) + } + } +} diff --git a/registry/regsrc/module.go b/registry/regsrc/module.go new file mode 100644 index 000000000..e26abcd7f --- /dev/null +++ b/registry/regsrc/module.go @@ -0,0 +1,162 @@ +package regsrc + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +var ( + ErrInvalidModuleSource = errors.New("not a valid registry module source") + + // nameSubRe is the sub-expression that matches a valid module namespace or + // name. It's strictly a super-set of what GitHub allows for user/org and + // repo names respectively, but more restrictive than our original repo-name + // regex which allowed periods but could cause ambiguity with hostname + // prefixes. It does not anchor the start or end so it can be composed into + // more complex RegExps below. Alphanumeric with - and _ allowed in non + // leading or trailing positions. Max length 64 chars. (GitHub username is + // 38 max.) + nameSubRe = "[0-9A-Za-z](?:[0-9A-Za-z-_]{0,62}[0-9A-Za-z])?" + + // providerSubRe is the sub-expression that matches a valid provider. It + // does not anchor the start or end so it can be composed into more complex + // RegExps below. Only lowercase chars and digits are supported in practice. + // Max length 64 chars. + providerSubRe = "[0-9a-z]{1,64}" + + // moduleSourceRe is a regular expression that matches the basic + // namespace/name/provider[//...] format for registry sources. It assumes + // any FriendlyHost prefix has already been removed if present. + moduleSourceRe = regexp.MustCompile( + fmt.Sprintf("^(%s)\\/(%s)\\/(%s)(?:\\/\\/(.*))?$", + nameSubRe, nameSubRe, providerSubRe)) +) + +// Module describes a Terraform Registry Module source. +type Module struct { + // RawHost is the friendly host prefix if one was present. It might be nil + // if the original source had no host prefix which implies + // PublicRegistryHost but is distinct from having an actual pointer to + // PublicRegistryHost since it encodes the fact the original string didn't + // include a host prefix at all which is significant for recovering actual + // input not just normalized form. Most callers should access it with Host() + // which will return public registry host instance if it's nil. + RawHost *FriendlyHost + RawNamespace string + RawName string + RawProvider string + RawSubmodule string +} + +// NewModule construct a new module source from separate parts. Pass empty +// string if host or submodule are not needed. +func NewModule(host, namespace, name, provider, submodule string) *Module { + m := &Module{ + RawNamespace: namespace, + RawName: name, + RawProvider: provider, + RawSubmodule: submodule, + } + if host != "" { + m.RawHost = NewFriendlyHost(host) + } + return m +} + +// ParseModuleSource attempts to parse source as a Terraform registry module +// source. If the string is not found to be in a valid format, +// ErrInvalidModuleSource is returned. Note that this can only be used on +// "input" strings, e.g. either ones supplied by the user or potentially +// normalised but in Display form (unicode). It will fail to parse a source with +// a punycoded domain since this is not permitted input from a user. If you have +// an already normalized string internally, you can compare it without parsing +// by comparing with the normalized version of the subject with the normal +// string equality operator. +func ParseModuleSource(source string) (*Module, error) { + // See if there is a friendly host prefix. + host, rest := ParseFriendlyHost(source) + if host != nil && !host.Valid() { + return nil, ErrInvalidModuleSource + } + + matches := moduleSourceRe.FindStringSubmatch(rest) + if len(matches) < 4 { + return nil, ErrInvalidModuleSource + } + + m := &Module{ + RawHost: host, + RawNamespace: matches[1], + RawName: matches[2], + RawProvider: matches[3], + } + + if len(matches) == 5 { + m.RawSubmodule = matches[4] + } + + return m, nil +} + +// Display returns the source formatted for display to the user in CLI or web +// output. +func (m *Module) Display() string { + return m.formatWithPrefix(m.normalizedHostPrefix(m.Host().Display()), false) +} + +// Normalized returns the source formatted for internal reference or comparison. +func (m *Module) Normalized() string { + return m.formatWithPrefix(m.normalizedHostPrefix(m.Host().Normalized()), false) +} + +// String returns the source formatted as the user originally typed it assuming +// it was parsed from user input. +func (m *Module) String() string { + // Don't normalize public registry hostname - leave it exactly like the user + // input it. + hostPrefix := "" + if m.RawHost != nil { + hostPrefix = m.RawHost.String() + "/" + } + return m.formatWithPrefix(hostPrefix, true) +} + +// Equal compares the module source against another instance taking +// normalization into account. +func (m *Module) Equal(other *Module) bool { + return m.Normalized() == other.Normalized() +} + +// Host returns the FriendlyHost object describing which registry this module is +// in. If the original source string had not host component this will return the +// PublicRegistryHost. +func (m *Module) Host() *FriendlyHost { + if m.RawHost == nil { + return PublicRegistryHost + } + return m.RawHost +} + +func (m *Module) normalizedHostPrefix(host string) string { + if m.Host().Equal(PublicRegistryHost) { + return "" + } + return host + "/" +} + +func (m *Module) formatWithPrefix(hostPrefix string, preserveCase bool) string { + suffix := "" + if m.RawSubmodule != "" { + suffix = "//" + m.RawSubmodule + } + str := fmt.Sprintf("%s%s/%s/%s%s", hostPrefix, m.RawNamespace, m.RawName, + m.RawProvider, suffix) + + // lower case by default + if !preserveCase { + return strings.ToLower(str) + } + return str +} diff --git a/registry/regsrc/module_test.go b/registry/regsrc/module_test.go new file mode 100644 index 000000000..19d9dfa19 --- /dev/null +++ b/registry/regsrc/module_test.go @@ -0,0 +1,131 @@ +package regsrc + +import ( + "testing" +) + +func TestModule(t *testing.T) { + tests := []struct { + name string + source string + wantString string + wantDisplay string + wantNorm string + wantErr bool + }{ + { + name: "public registry", + source: "hashicorp/consul/aws", + wantString: "hashicorp/consul/aws", + wantDisplay: "hashicorp/consul/aws", + wantNorm: "hashicorp/consul/aws", + wantErr: false, + }, + { + name: "public registry, submodule", + source: "hashicorp/consul/aws//foo", + wantString: "hashicorp/consul/aws//foo", + wantDisplay: "hashicorp/consul/aws//foo", + wantNorm: "hashicorp/consul/aws//foo", + wantErr: false, + }, + { + name: "public registry, explicit host", + source: "registry.terraform.io/hashicorp/consul/aws", + wantString: "registry.terraform.io/hashicorp/consul/aws", + wantDisplay: "hashicorp/consul/aws", + wantNorm: "hashicorp/consul/aws", + wantErr: false, + }, + { + name: "public registry, mixed case", + source: "HashiCorp/Consul/aws", + wantString: "HashiCorp/Consul/aws", + wantDisplay: "hashicorp/consul/aws", + wantNorm: "hashicorp/consul/aws", + wantErr: false, + }, + { + name: "private registry, custom port", + source: "Example.com:1234/HashiCorp/Consul/aws", + wantString: "Example.com:1234/HashiCorp/Consul/aws", + wantDisplay: "example.com:1234/hashicorp/consul/aws", + wantNorm: "example.com:1234/hashicorp/consul/aws", + wantErr: false, + }, + { + name: "IDN registry", + source: "Испытание.com/HashiCorp/Consul/aws", + wantString: "Испытание.com/HashiCorp/Consul/aws", + wantDisplay: "испытание.com/hashicorp/consul/aws", + wantNorm: "xn--80akhbyknj4f.com/hashicorp/consul/aws", + wantErr: false, + }, + { + name: "IDN registry, submodule, custom port", + source: "Испытание.com:1234/HashiCorp/Consul/aws//Foo", + wantString: "Испытание.com:1234/HashiCorp/Consul/aws//Foo", + // Note we DO lowercase submodule names. This might causes issues on + // some filesystems (e.g. HFS+) that are case-sensitive where + // //modules/Foo and //modules/foo describe different paths, but + // it's less confusing in general just to not support that. Any user + // with a module with submodules in both cases is already asking for + // portability issues, and terraform can ensure it does + // case-insensitive search for the dir in those cases. + wantDisplay: "испытание.com:1234/hashicorp/consul/aws//foo", + wantNorm: "xn--80akhbyknj4f.com:1234/hashicorp/consul/aws//foo", + wantErr: false, + }, + { + name: "invalid host", + source: "---.com/HashiCorp/Consul/aws", + wantErr: true, + }, + { + name: "invalid format", + source: "foo/var/baz/qux", + wantErr: true, + }, + { + name: "invalid suffix", + source: "foo/var/baz?otherthing", + wantErr: true, + }, + { + name: "valid host, invalid format", + source: "foo.com/var/baz?otherthing", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseModuleSource(tt.source) + + if (err != nil) != tt.wantErr { + t.Fatalf("ParseModuleSource() error = %v, wantErr %v", err, tt.wantErr) + } + + if err != nil { + return + } + + if v := got.String(); v != tt.wantString { + t.Fatalf("String() = %v, want %v", v, tt.wantString) + } + if v := got.Display(); v != tt.wantDisplay { + t.Fatalf("Display() = %v, want %v", v, tt.wantDisplay) + } + if v := got.Normalized(); v != tt.wantNorm { + t.Fatalf("Normalized() = %v, want %v", v, tt.wantNorm) + } + + gotDisplay, err := ParseModuleSource(tt.wantDisplay) + if err != nil { + t.Fatalf("ParseModuleSource(wantDisplay) error = %v", err) + } + if !got.Equal(gotDisplay) { + t.Fatalf("Equal() failed for %s and %s", tt.source, tt.wantDisplay) + } + }) + } +} diff --git a/registry/regsrc/regsrc.go b/registry/regsrc/regsrc.go new file mode 100644 index 000000000..c430bf141 --- /dev/null +++ b/registry/regsrc/regsrc.go @@ -0,0 +1,8 @@ +// Package regsrc provides helpers for working with source strings that identify +// resources within a Terraform registry. +package regsrc + +var ( + // PublicRegistryHost is a FriendlyHost that represents the public registry. + PublicRegistryHost = NewFriendlyHost("registry.terraform.io") +) diff --git a/registry/response/module.go b/registry/response/module.go new file mode 100644 index 000000000..e3d47094b --- /dev/null +++ b/registry/response/module.go @@ -0,0 +1,193 @@ +package response + +import ( + "fmt" + "time" + + "github.com/hashicorp/terraform-registry/api/models" +) + +// Module is the response structure with the data for a single module version. +type Module struct { + ID string `json:"id"` + + //--------------------------------------------------------------- + // Metadata about the overall module. + + Owner string `json:"owner"` + Namespace string `json:"namespace"` + Name string `json:"name"` + Version string `json:"version"` + Provider string `json:"provider"` + Description string `json:"description"` + Source string `json:"source"` + PublishedAt time.Time `json:"published_at"` + Downloads int `json:"downloads"` + Verified bool `json:"verified"` +} + +// ModuleDetail represents a module in full detail. +type ModuleDetail struct { + Module + + //--------------------------------------------------------------- + // Metadata about the overall module. This is only available when + // requesting the specific module (not in list responses). + + // Root is the root module. + Root *ModuleSubmodule `json:"root"` + + // Submodules are the other submodules that are available within + // this module. + Submodules []*ModuleSubmodule `json:"submodules"` + + //--------------------------------------------------------------- + // The fields below are only set when requesting this specific + // module. They are available to easily know all available versions + // and providers without multiple API calls. + + Providers []string `json:"providers"` // All available providers + Versions []string `json:"versions"` // All versions +} + +// ModuleSubmodule is the metadata about a specific submodule within +// a module. This includes the root module as a special case. +type ModuleSubmodule struct { + Path string `json:"path"` + Readme string `json:"readme"` + Empty bool `json:"empty"` + + Inputs []*ModuleInput `json:"inputs"` + Outputs []*ModuleOutput `json:"outputs"` + Dependencies []*ModuleDep `json:"dependencies"` + Resources []*ModuleResource `json:"resources"` +} + +// ModuleInput is an input for a module. +type ModuleInput struct { + Name string `json:"name"` + Description string `json:"description"` + Default string `json:"default"` +} + +// ModuleOutput is an output for a module. +type ModuleOutput struct { + Name string `json:"name"` + Description string `json:"description"` +} + +// ModuleDep is an output for a module. +type ModuleDep struct { + Name string `json:"name"` + Source string `json:"source"` + Version string `json:"version"` +} + +// ModuleProviderDep is the output for a provider dependency +type ModuleProviderDep struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// ModuleResource is an output for a module. +type ModuleResource struct { + Name string `json:"name"` + Type string `json:"type"` +} + +// NewModule creates a Module response object from a model. +func NewModule(mv *models.ModuleVersion) *Module { + m := mv.ModuleProvider.Module + mp := mv.ModuleProvider + + // Build the full module + return &Module{ + ID: fmt.Sprintf( + "%s/%s/%s/%s", + m.Namespace, + m.Name, + mp.Provider, + mv.Version), + + // Base metadata + Owner: m.User.Username, + Namespace: m.Namespace, + Name: m.Name, + Version: mv.Version, + Provider: mv.ModuleProvider.Provider, + Description: mv.Description, + Source: mp.Source, + PublishedAt: mv.PublishedAt, + Downloads: int(mp.Downloads), + Verified: m.Verified, + } +} + +// NewModuleDetail creates a ModuleDetail response object from a model. +func NewModuleDetail(mv *models.ModuleVersion) *ModuleDetail { + m := NewModule(mv) + + // Build the submodule response objects + var submodules []*ModuleSubmodule + var submoduleRoot *ModuleSubmodule + for _, sub := range mv.Submodules { + resp := NewModuleSubmodule(&sub) + + if sub.Root() { + submoduleRoot = resp + } else { + submodules = append(submodules, resp) + } + } + return &ModuleDetail{ + Module: *m, + Root: submoduleRoot, + Submodules: submodules, + } +} + +// NewModuleSubmodule creates a ModuleSubmodule response object from a model. +func NewModuleSubmodule(m *models.ModuleSubmodule) *ModuleSubmodule { + inputs := make([]*ModuleInput, 0, len(m.Variables)) + for _, v := range m.Variables { + inputs = append(inputs, &ModuleInput{ + Name: v.Name, + Description: v.Description.String, + Default: v.Default.String, + }) + } + + outputs := make([]*ModuleOutput, 0, len(m.Outputs)) + for _, v := range m.Outputs { + outputs = append(outputs, &ModuleOutput{ + Name: v.Name, + Description: v.Description.String, + }) + } + + deps := make([]*ModuleDep, 0, len(m.Dependencies)) + for _, v := range m.Dependencies { + deps = append(deps, &ModuleDep{ + Name: v.Name, + Source: v.Source, + }) + } + + resources := make([]*ModuleResource, 0, len(m.Resources)) + for _, v := range m.Resources { + resources = append(resources, &ModuleResource{ + Name: v.Name, + Type: v.Type, + }) + } + + return &ModuleSubmodule{ + Path: m.Path, + Readme: m.Readme, + Empty: m.Empty, + Inputs: inputs, + Outputs: outputs, + Dependencies: deps, + Resources: resources, + } +} diff --git a/registry/response/module_list.go b/registry/response/module_list.go new file mode 100644 index 000000000..978374822 --- /dev/null +++ b/registry/response/module_list.go @@ -0,0 +1,7 @@ +package response + +// ModuleList is the response structure for a pageable list of modules. +type ModuleList struct { + Meta PaginationMeta `json:"meta"` + Modules []*Module `json:"modules"` +} diff --git a/registry/response/module_provider.go b/registry/response/module_provider.go new file mode 100644 index 000000000..e48499dce --- /dev/null +++ b/registry/response/module_provider.go @@ -0,0 +1,14 @@ +package response + +// ModuleProvider represents a single provider for modules. +type ModuleProvider struct { + Name string `json:"name"` + Downloads int `json:"downloads"` + ModuleCount int `json:"module_count"` +} + +// ModuleProviderList is the response structure for a pageable list of ModuleProviders. +type ModuleProviderList struct { + Meta PaginationMeta `json:"meta"` + Providers []*ModuleProvider `json:"providers"` +} diff --git a/registry/response/module_versions.go b/registry/response/module_versions.go new file mode 100644 index 000000000..d5e551e8f --- /dev/null +++ b/registry/response/module_versions.go @@ -0,0 +1,131 @@ +package response + +import ( + "github.com/hashicorp/terraform-registry/api/regsrc" + + "github.com/hashicorp/terraform-registry/api/models" +) + +// ModuleVersions is the response format that contains all metadata about module +// versions needed for terraform CLI to resolve version constraints. See RFC +// TF-042 for details on this format. +type ModuleVersions struct { + Modules []*ModuleProviderVersions `json:"modules"` +} + +// ModuleProviderVersions is the response format for a single module instance, +// containing metadata about all versions and their dependencies. +type ModuleProviderVersions struct { + Source string `json:"source"` + Versions []*ModuleVersion `json:"versions"` +} + +// ModuleVersion is the output metadata for a given version needed by CLI to +// resolve candidate versions to satisfy requirements. +type ModuleVersion struct { + Version string `json:"version"` + Root VersionSubmodule `json:"root"` + Submodules []*VersionSubmodule `json:"submodules"` +} + +// VersionSubmodule is the output metadata for a submodule within a given +// version needed by CLI to resolve candidate versions to satisfy requirements. +// When representing the Root in JSON the path is omitted. +type VersionSubmodule struct { + Path string `json:"path,omitempty"` + Providers []*ModuleProviderDep `json:"providers"` + Dependencies []*ModuleDep `json:"dependencies"` +} + +// NewModuleVersions populates a ModuleVersions response based on a slice of +// ModuleProviders. It is assumed these are fully populated with all versions +// submodules and dependencies etc, required in the response, and in the desired +// order (i.e. the first mp is the specific one requested and any others are +// optionally pre-fetched dependencies.) The host is needed to generate correct +// Source strings for all modules and must be the canonical hostname for the +// registry instance. +func NewModuleVersions(mps []*models.ModuleProvider, + host regsrc.FriendlyHost) *ModuleVersions { + + mods := make([]*ModuleProviderVersions, 0, len(mps)) + for _, mp := range mps { + mods = append(mods, NewModuleProviderVersions(mp, host)) + } + + return &ModuleVersions{ + Modules: mods, + } +} + +// NewModuleProviderVersions constructs the metadata about a specific module +// for the ModuleVersions response. +func NewModuleProviderVersions(mp *models.ModuleProvider, + host regsrc.FriendlyHost) *ModuleProviderVersions { + + src := regsrc.NewModule( + host.String(), + mp.Module.Namespace, + mp.Module.Name, + mp.Provider, + "", + ) + + versions := make([]*ModuleVersion, 0, len(mp.Versions)) + for _, mv := range mp.Versions { + versions = append(versions, NewModuleVersion(&mv)) + } + + return &ModuleProviderVersions{ + Source: src.Display(), + Versions: versions, + } +} + +// NewModuleVersion constructs the metadata about a specific module version +// for the ModuleVersions response. +func NewModuleVersion(mv *models.ModuleVersion) *ModuleVersion { + // Build the submodule response objects + var submodules []*VersionSubmodule + var submoduleRoot VersionSubmodule + for _, sub := range mv.Submodules { + resp := NewVersionSubmodule(&sub) + + if sub.Root() { + submoduleRoot = *resp + } else { + submodules = append(submodules, resp) + } + } + + return &ModuleVersion{ + Version: mv.Version, + Root: submoduleRoot, + Submodules: submodules, + } +} + +// NewVersionSubmodule constructs a representation of a submodule within a +// specific module version for the ModuleVersions response. +func NewVersionSubmodule(m *models.ModuleSubmodule) *VersionSubmodule { + providerDeps := make([]*ModuleProviderDep, 0, len(m.ProviderDependencies)) + for _, v := range m.ProviderDependencies { + providerDeps = append(providerDeps, &ModuleProviderDep{ + Name: v.Provider, + Version: v.VersionConstraints, + }) + } + + deps := make([]*ModuleDep, 0, len(m.Dependencies)) + for _, v := range m.Dependencies { + deps = append(deps, &ModuleDep{ + Name: v.Name, + Source: v.Source, + }) + } + + return &VersionSubmodule{ + Path: m.Path, + Providers: providerDeps, + Dependencies: deps, + } +} diff --git a/registry/response/pagination.go b/registry/response/pagination.go new file mode 100644 index 000000000..75a925490 --- /dev/null +++ b/registry/response/pagination.go @@ -0,0 +1,65 @@ +package response + +import ( + "net/url" + "strconv" +) + +// PaginationMeta is a structure included in responses for pagination. +type PaginationMeta struct { + Limit int `json:"limit"` + CurrentOffset int `json:"current_offset"` + NextOffset *int `json:"next_offset,omitempty"` + PrevOffset *int `json:"prev_offset,omitempty"` + NextURL string `json:"next_url,omitempty"` + PrevURL string `json:"prev_url,omitempty"` +} + +// NewPaginationMeta populates pagination meta data from result parameters +func NewPaginationMeta(offset, limit int, hasMore bool, currentURL string) PaginationMeta { + pm := PaginationMeta{ + Limit: limit, + CurrentOffset: offset, + } + + // Calculate next/prev offsets, leave nil if not valid pages + nextOffset := offset + limit + if hasMore { + pm.NextOffset = &nextOffset + } + + prevOffset := offset - limit + if prevOffset < 0 { + prevOffset = 0 + } + if prevOffset < offset { + pm.PrevOffset = &prevOffset + } + + // If URL format provided, populate URLs. Intentionally swallow URL errors for now, API should + // catch missing URLs if we call with bad URL arg (and we care about them being present). + if currentURL != "" && pm.NextOffset != nil { + pm.NextURL, _ = setQueryParam(currentURL, "offset", *pm.NextOffset, 0) + } + if currentURL != "" && pm.PrevOffset != nil { + pm.PrevURL, _ = setQueryParam(currentURL, "offset", *pm.PrevOffset, 0) + } + + return pm +} + +func setQueryParam(baseURL, key string, val, defaultVal int) (string, error) { + u, err := url.Parse(baseURL) + if err != nil { + return "", err + } + q := u.Query() + if val == defaultVal { + // elide param if it's the default value + q.Del(key) + } else { + q.Set(key, strconv.Itoa(val)) + } + u.RawQuery = q.Encode() + return u.String(), nil +} diff --git a/registry/response/pagination_test.go b/registry/response/pagination_test.go new file mode 100644 index 000000000..be862abbd --- /dev/null +++ b/registry/response/pagination_test.go @@ -0,0 +1,122 @@ +package response + +import ( + "encoding/json" + "testing" +) + +func intPtr(i int) *int { + return &i +} + +func prettyJSON(o interface{}) (string, error) { + bytes, err := json.MarshalIndent(o, "", "\t") + if err != nil { + return "", err + } + return string(bytes), nil +} + +func TestNewPaginationMeta(t *testing.T) { + type args struct { + offset int + limit int + hasMore bool + currentURL string + } + tests := []struct { + name string + args args + wantJSON string + }{ + { + name: "first page", + args: args{0, 10, true, "http://foo.com/v1/bar"}, + wantJSON: `{ + "limit": 10, + "current_offset": 0, + "next_offset": 10, + "next_url": "http://foo.com/v1/bar?offset=10" +}`, + }, + { + name: "second page", + args: args{10, 10, true, "http://foo.com/v1/bar"}, + wantJSON: `{ + "limit": 10, + "current_offset": 10, + "next_offset": 20, + "prev_offset": 0, + "next_url": "http://foo.com/v1/bar?offset=20", + "prev_url": "http://foo.com/v1/bar" +}`, + }, + { + name: "last page", + args: args{40, 10, false, "http://foo.com/v1/bar"}, + wantJSON: `{ + "limit": 10, + "current_offset": 40, + "prev_offset": 30, + "prev_url": "http://foo.com/v1/bar?offset=30" +}`, + }, + { + name: "misaligned start ending exactly on boundary", + args: args{32, 10, false, "http://foo.com/v1/bar"}, + wantJSON: `{ + "limit": 10, + "current_offset": 32, + "prev_offset": 22, + "prev_url": "http://foo.com/v1/bar?offset=22" +}`, + }, + { + name: "misaligned start partially through first page", + args: args{5, 12, true, "http://foo.com/v1/bar"}, + wantJSON: `{ + "limit": 12, + "current_offset": 5, + "next_offset": 17, + "prev_offset": 0, + "next_url": "http://foo.com/v1/bar?offset=17", + "prev_url": "http://foo.com/v1/bar" +}`, + }, + { + name: "no current URL", + args: args{10, 10, true, ""}, + wantJSON: `{ + "limit": 10, + "current_offset": 10, + "next_offset": 20, + "prev_offset": 0 +}`, + }, + { + name: "#58 regression test", + args: args{1, 3, true, ""}, + wantJSON: `{ + "limit": 3, + "current_offset": 1, + "next_offset": 4, + "prev_offset": 0 +}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NewPaginationMeta(tt.args.offset, tt.args.limit, tt.args.hasMore, + tt.args.currentURL) + gotJSON, err := prettyJSON(got) + if err != nil { + t.Fatalf("failed to marshal PaginationMeta to JSON: %s", err) + } + if gotJSON != tt.wantJSON { + // prettyJSON makes debugging easier due to the annoying pointer-to-ints, but it + // also implicitly tests JSON marshalling as we can see if it's omitting fields etc. + t.Fatalf("NewPaginationMeta() =\n%s\n want:\n%s\n", gotJSON, tt.wantJSON) + } + }) + } +} diff --git a/registry/response/redirect.go b/registry/response/redirect.go new file mode 100644 index 000000000..d5eb49ba6 --- /dev/null +++ b/registry/response/redirect.go @@ -0,0 +1,6 @@ +package response + +// Redirect causes the frontend to perform a window redirect. +type Redirect struct { + URL string `json:"url"` +} From 27e578e7fb54ba572ae79a212c148c392448aa09 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Tue, 24 Oct 2017 18:56:20 -0400 Subject: [PATCH 2/7] convert FriendlyHost to use svchost for validation Removed some of the test cases that we don't allow in the svchost package. Will check back if those are needed in the registry and work around them as necessary. --- registry/regsrc/friendly_host.go | 17 +++++------------ registry/regsrc/friendly_host_test.go | 22 ++++++++++++++++------ 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/registry/regsrc/friendly_host.go b/registry/regsrc/friendly_host.go index 28ca9b0aa..da93cdfd2 100644 --- a/registry/regsrc/friendly_host.go +++ b/registry/regsrc/friendly_host.go @@ -4,7 +4,7 @@ import ( "regexp" "strings" - "golang.org/x/net/idna" + "github.com/hashicorp/terraform/svchost" ) var ( @@ -110,27 +110,20 @@ func (h *FriendlyHost) Valid() bool { // Display returns the host formatted for display to the user in CLI or web // output. func (h *FriendlyHost) Display() string { - parts := strings.SplitN(h.Raw, ":", 2) - var err error - parts[0], err = idna.Display.ToUnicode(parts[0]) + hostname, err := svchost.ForComparison(h.Raw) if err != nil { return InvalidHostString } - return strings.Join(parts, ":") + return hostname.ForDisplay() } // Normalized returns the host formatted for internal reference or comparison. func (h *FriendlyHost) Normalized() string { - // For now IDNA does all the normalisation we need including case-folding - // pure ASCII to lower. But breaks if a custom port is included while we - // want to allow that and normalize comparison including it, - parts := strings.SplitN(h.Raw, ":", 2) - var err error - parts[0], err = idna.Lookup.ToASCII(parts[0]) + hostname, err := svchost.ForComparison(h.Raw) if err != nil { return InvalidHostString } - return strings.Join(parts, ":") + return hostname.String() } // String returns the host formatted as the user originally typed it assuming it diff --git a/registry/regsrc/friendly_host_test.go b/registry/regsrc/friendly_host_test.go index e87774cfe..5ccac5b0d 100644 --- a/registry/regsrc/friendly_host_test.go +++ b/registry/regsrc/friendly_host_test.go @@ -95,15 +95,21 @@ func TestFriendlyHost(t *testing.T) { if v := gotHost.String(); v != tt.wantHost { t.Fatalf("String() = %v, want %v", v, tt.wantHost) } + if v := gotHost.Valid(); v != tt.wantValid { + t.Fatalf("Valid() = %v, want %v", v, tt.wantValid) + } + + // FIXME: should we allow punycode as input + if !tt.wantValid { + return + } + if v := gotHost.Display(); v != tt.wantDisplay { t.Fatalf("Display() = %v, want %v", v, tt.wantDisplay) } if v := gotHost.Normalized(); v != tt.wantNorm { t.Fatalf("Normalized() = %v, want %v", v, tt.wantNorm) } - if v := gotHost.Valid(); v != tt.wantValid { - t.Fatalf("Valid() = %v, want %v", v, tt.wantValid) - } if gotRest != strings.TrimLeft(sfx, "/") { t.Fatalf("ParseFriendlyHost() rest = %v, want %v", gotRest, strings.TrimLeft(sfx, "/")) } @@ -112,9 +118,13 @@ func TestFriendlyHost(t *testing.T) { if !gotHost.Equal(&FriendlyHost{Raw: tt.wantDisplay}) { t.Fatalf("Equal() should be true for %s and %s", tt.wantHost, tt.wantValid) } - if !gotHost.Equal(&FriendlyHost{Raw: tt.wantNorm}) { - t.Fatalf("Equal() should be true for %s and %s", tt.wantHost, tt.wantNorm) - } + + // FIXME: Do we need to accept normalized input? + //if !gotHost.Equal(&FriendlyHost{Raw: tt.wantNorm}) { + // fmt.Println(gotHost.Normalized(), tt.wantNorm) + // fmt.Println(" ", (&FriendlyHost{Raw: tt.wantNorm}).Normalized()) + // t.Fatalf("Equal() should be true for %s and %s", tt.wantHost, tt.wantNorm) + //} }) } From 666164c36938f6c2a4c6d5e9698125618639f7ae Mon Sep 17 00:00:00 2001 From: James Bardin Date: Tue, 24 Oct 2017 19:27:36 -0400 Subject: [PATCH 3/7] disallow github and bitbucket hosts These hosts have special usage in module source strings, and can't be valid registry hosts. --- registry/regsrc/module.go | 13 +++++++++++-- registry/regsrc/module_test.go | 10 ++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/registry/regsrc/module.go b/registry/regsrc/module.go index e26abcd7f..05a7bfa32 100644 --- a/registry/regsrc/module.go +++ b/registry/regsrc/module.go @@ -32,6 +32,13 @@ var ( moduleSourceRe = regexp.MustCompile( fmt.Sprintf("^(%s)\\/(%s)\\/(%s)(?:\\/\\/(.*))?$", nameSubRe, nameSubRe, providerSubRe)) + + // disallowed is a set of hostnames that have special usage in modules and + // can't be registry hosts + disallowed = map[string]bool{ + "github.com": true, + "bitbucket.org": true, + } ) // Module describes a Terraform Registry Module source. @@ -77,8 +84,10 @@ func NewModule(host, namespace, name, provider, submodule string) *Module { func ParseModuleSource(source string) (*Module, error) { // See if there is a friendly host prefix. host, rest := ParseFriendlyHost(source) - if host != nil && !host.Valid() { - return nil, ErrInvalidModuleSource + if host != nil { + if !host.Valid() || disallowed[host.Display()] { + return nil, ErrInvalidModuleSource + } } matches := moduleSourceRe.FindStringSubmatch(rest) diff --git a/registry/regsrc/module_test.go b/registry/regsrc/module_test.go index 19d9dfa19..bae502b0d 100644 --- a/registry/regsrc/module_test.go +++ b/registry/regsrc/module_test.go @@ -96,6 +96,16 @@ func TestModule(t *testing.T) { source: "foo.com/var/baz?otherthing", wantErr: true, }, + { + name: "disallow github", + source: "github.com/HashiCorp/Consul/aws", + wantErr: true, + }, + { + name: "disallow bitbucket", + source: "bitbucket.org/HashiCorp/Consul/aws", + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 4f76d3a556b26a67c4e5fe735bd8a3f9f3c60dde Mon Sep 17 00:00:00 2001 From: James Bardin Date: Wed, 25 Oct 2017 15:27:29 -0400 Subject: [PATCH 4/7] remove models dependency from the regresp That isn't needed by terraform, and can be left in a registry package. --- registry/response/module.go | 100 ------------------------------------ 1 file changed, 100 deletions(-) diff --git a/registry/response/module.go b/registry/response/module.go index e3d47094b..3bd2b3df2 100644 --- a/registry/response/module.go +++ b/registry/response/module.go @@ -1,10 +1,7 @@ package response import ( - "fmt" "time" - - "github.com/hashicorp/terraform-registry/api/models" ) // Module is the response structure with the data for a single module version. @@ -94,100 +91,3 @@ type ModuleResource struct { Name string `json:"name"` Type string `json:"type"` } - -// NewModule creates a Module response object from a model. -func NewModule(mv *models.ModuleVersion) *Module { - m := mv.ModuleProvider.Module - mp := mv.ModuleProvider - - // Build the full module - return &Module{ - ID: fmt.Sprintf( - "%s/%s/%s/%s", - m.Namespace, - m.Name, - mp.Provider, - mv.Version), - - // Base metadata - Owner: m.User.Username, - Namespace: m.Namespace, - Name: m.Name, - Version: mv.Version, - Provider: mv.ModuleProvider.Provider, - Description: mv.Description, - Source: mp.Source, - PublishedAt: mv.PublishedAt, - Downloads: int(mp.Downloads), - Verified: m.Verified, - } -} - -// NewModuleDetail creates a ModuleDetail response object from a model. -func NewModuleDetail(mv *models.ModuleVersion) *ModuleDetail { - m := NewModule(mv) - - // Build the submodule response objects - var submodules []*ModuleSubmodule - var submoduleRoot *ModuleSubmodule - for _, sub := range mv.Submodules { - resp := NewModuleSubmodule(&sub) - - if sub.Root() { - submoduleRoot = resp - } else { - submodules = append(submodules, resp) - } - } - return &ModuleDetail{ - Module: *m, - Root: submoduleRoot, - Submodules: submodules, - } -} - -// NewModuleSubmodule creates a ModuleSubmodule response object from a model. -func NewModuleSubmodule(m *models.ModuleSubmodule) *ModuleSubmodule { - inputs := make([]*ModuleInput, 0, len(m.Variables)) - for _, v := range m.Variables { - inputs = append(inputs, &ModuleInput{ - Name: v.Name, - Description: v.Description.String, - Default: v.Default.String, - }) - } - - outputs := make([]*ModuleOutput, 0, len(m.Outputs)) - for _, v := range m.Outputs { - outputs = append(outputs, &ModuleOutput{ - Name: v.Name, - Description: v.Description.String, - }) - } - - deps := make([]*ModuleDep, 0, len(m.Dependencies)) - for _, v := range m.Dependencies { - deps = append(deps, &ModuleDep{ - Name: v.Name, - Source: v.Source, - }) - } - - resources := make([]*ModuleResource, 0, len(m.Resources)) - for _, v := range m.Resources { - resources = append(resources, &ModuleResource{ - Name: v.Name, - Type: v.Type, - }) - } - - return &ModuleSubmodule{ - Path: m.Path, - Readme: m.Readme, - Empty: m.Empty, - Inputs: inputs, - Outputs: outputs, - Dependencies: deps, - Resources: resources, - } -} From a88146083945d9496fc1f47c0a73a95c238e6f58 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Wed, 25 Oct 2017 15:49:35 -0400 Subject: [PATCH 5/7] fix vet error in registry package --- registry/regsrc/friendly_host_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/registry/regsrc/friendly_host_test.go b/registry/regsrc/friendly_host_test.go index 5ccac5b0d..740395bf6 100644 --- a/registry/regsrc/friendly_host_test.go +++ b/registry/regsrc/friendly_host_test.go @@ -116,7 +116,7 @@ func TestFriendlyHost(t *testing.T) { // Also verify that host compares equal with all the variants. if !gotHost.Equal(&FriendlyHost{Raw: tt.wantDisplay}) { - t.Fatalf("Equal() should be true for %s and %s", tt.wantHost, tt.wantValid) + t.Fatalf("Equal() should be true for %s and %t", tt.wantHost, tt.wantValid) } // FIXME: Do we need to accept normalized input? From 54c5aab7ef4b4655bbbb0e756f06d6cd519e5891 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Wed, 25 Oct 2017 15:57:51 -0400 Subject: [PATCH 6/7] missed another models dep in the registry --- registry/response/module_versions.go | 99 ---------------------------- 1 file changed, 99 deletions(-) diff --git a/registry/response/module_versions.go b/registry/response/module_versions.go index d5e551e8f..f69e9750c 100644 --- a/registry/response/module_versions.go +++ b/registry/response/module_versions.go @@ -1,11 +1,5 @@ package response -import ( - "github.com/hashicorp/terraform-registry/api/regsrc" - - "github.com/hashicorp/terraform-registry/api/models" -) - // ModuleVersions is the response format that contains all metadata about module // versions needed for terraform CLI to resolve version constraints. See RFC // TF-042 for details on this format. @@ -36,96 +30,3 @@ type VersionSubmodule struct { Providers []*ModuleProviderDep `json:"providers"` Dependencies []*ModuleDep `json:"dependencies"` } - -// NewModuleVersions populates a ModuleVersions response based on a slice of -// ModuleProviders. It is assumed these are fully populated with all versions -// submodules and dependencies etc, required in the response, and in the desired -// order (i.e. the first mp is the specific one requested and any others are -// optionally pre-fetched dependencies.) The host is needed to generate correct -// Source strings for all modules and must be the canonical hostname for the -// registry instance. -func NewModuleVersions(mps []*models.ModuleProvider, - host regsrc.FriendlyHost) *ModuleVersions { - - mods := make([]*ModuleProviderVersions, 0, len(mps)) - for _, mp := range mps { - mods = append(mods, NewModuleProviderVersions(mp, host)) - } - - return &ModuleVersions{ - Modules: mods, - } -} - -// NewModuleProviderVersions constructs the metadata about a specific module -// for the ModuleVersions response. -func NewModuleProviderVersions(mp *models.ModuleProvider, - host regsrc.FriendlyHost) *ModuleProviderVersions { - - src := regsrc.NewModule( - host.String(), - mp.Module.Namespace, - mp.Module.Name, - mp.Provider, - "", - ) - - versions := make([]*ModuleVersion, 0, len(mp.Versions)) - for _, mv := range mp.Versions { - versions = append(versions, NewModuleVersion(&mv)) - } - - return &ModuleProviderVersions{ - Source: src.Display(), - Versions: versions, - } -} - -// NewModuleVersion constructs the metadata about a specific module version -// for the ModuleVersions response. -func NewModuleVersion(mv *models.ModuleVersion) *ModuleVersion { - // Build the submodule response objects - var submodules []*VersionSubmodule - var submoduleRoot VersionSubmodule - for _, sub := range mv.Submodules { - resp := NewVersionSubmodule(&sub) - - if sub.Root() { - submoduleRoot = *resp - } else { - submodules = append(submodules, resp) - } - } - - return &ModuleVersion{ - Version: mv.Version, - Root: submoduleRoot, - Submodules: submodules, - } -} - -// NewVersionSubmodule constructs a representation of a submodule within a -// specific module version for the ModuleVersions response. -func NewVersionSubmodule(m *models.ModuleSubmodule) *VersionSubmodule { - providerDeps := make([]*ModuleProviderDep, 0, len(m.ProviderDependencies)) - for _, v := range m.ProviderDependencies { - providerDeps = append(providerDeps, &ModuleProviderDep{ - Name: v.Provider, - Version: v.VersionConstraints, - }) - } - - deps := make([]*ModuleDep, 0, len(m.Dependencies)) - for _, v := range m.Dependencies { - deps = append(deps, &ModuleDep{ - Name: v.Name, - Source: v.Source, - }) - } - - return &VersionSubmodule{ - Path: m.Path, - Providers: providerDeps, - Dependencies: deps, - } -} From 0a673fa5ef5082a69ce8f4a64928010b5a21c3d4 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Wed, 25 Oct 2017 17:10:15 -0400 Subject: [PATCH 7/7] use svchost.IsValid for FriendlyHost.Valid --- registry/regsrc/friendly_host.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/registry/regsrc/friendly_host.go b/registry/regsrc/friendly_host.go index da93cdfd2..648e2a193 100644 --- a/registry/regsrc/friendly_host.go +++ b/registry/regsrc/friendly_host.go @@ -95,16 +95,7 @@ func ParseFriendlyHost(source string) (host *FriendlyHost, rest string) { // name specifications. Not that IDN prefixes containing punycode are not valid // input which we expect to always be in user-input or normalised display form. func (h *FriendlyHost) Valid() bool { - if h.Display() == InvalidHostString { - return false - } - if h.Normalized() == InvalidHostString { - return false - } - if containsPuny(h.Raw) { - return false - } - return true + return svchost.IsValid(h.Raw) } // Display returns the host formatted for display to the user in CLI or web