Add registry detector

Add a getter.Detector for detecting registry modules and looking up
the download location of the latest version. This is essentially a
temporary API until constraint solving is supported by the registry, as
then we'll have to supply the full set of known contraints to the
registry at once for resolution and we will fetch specific versions of
modules.
This commit is contained in:
James Bardin 2017-08-31 09:10:55 -04:00
parent 67bdadf5c6
commit a83ff57aea
3 changed files with 244 additions and 1 deletions

View File

@ -1,10 +1,17 @@
package module package module
import ( import (
"fmt"
"io/ioutil" "io/ioutil"
"log"
"net/http"
"os" "os"
"regexp"
"strings"
"github.com/hashicorp/go-getter" "github.com/hashicorp/go-getter"
cleanhttp "github.com/hashicorp/go-cleanhttp"
) )
// GetMode is an enum that describes how modules are loaded. // GetMode is an enum that describes how modules are loaded.
@ -69,3 +76,96 @@ func getStorage(s getter.Storage, key string, src string, mode GetMode) (string,
// Get the directory where the module is. // Get the directory where the module is.
return s.Dir(key) return s.Dir(key)
} }
const (
registryAPI = "https://registry.terraform.io/v1/modules/"
xTerraformGet = "X-Terraform-Get"
)
var detectors = []getter.Detector{
new(getter.GitHubDetector),
new(getter.BitBucketDetector),
new(getter.S3Detector),
new(registryDetector),
new(getter.FileDetector),
}
// these prefixes can't be registry IDs
// "http", "./", "/", "getter::"
var skipRegistry = regexp.MustCompile(`^(http|\./|/|[A-Za-z0-9]+::)`).MatchString
// registryDetector implements getter.Detector to detect Terraform Registry modules.
// If a path looks like a registry module identifier, attempt to locate it in
// the registry. If it's not found, pass it on in case it can be found by
// other means.
type registryDetector struct {
// override the default registry URL
api string
client *http.Client
}
func (d registryDetector) Detect(src, _ string) (string, bool, error) {
// the namespace can't start with "http", a relative or absolute path, or
// contain a go-getter "forced getter"
if skipRegistry(src) {
return "", false, nil
}
// there are 3 parts to a registry ID
if len(strings.Split(src, "/")) != 3 {
return "", false, nil
}
return d.lookupModule(src)
}
// Lookup the module in the registry.
// Since existing module sources may match a registry ID format, we only log
// registry errors and continue discovery.
func (d registryDetector) lookupModule(src string) (string, bool, error) {
if d.api == "" {
d.api = registryAPI
}
if d.client == nil {
d.client = cleanhttp.DefaultClient()
}
// src is already partially validated in Detect. We know it's a path, and
// if it can be parsed as a URL we will hand it off to the registry to
// determine if it's truly valid.
resp, err := d.client.Get(fmt.Sprintf("%s/%s/download", d.api, src))
if err != nil {
log.Println("[WARN] error looking up module %q: %s", src, err)
return "", false, nil
}
defer resp.Body.Close()
// there should be no body, but save it for logging
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
fmt.Println("[WARN] error reading response body from registry: %s", err)
return "", false, nil
}
switch resp.StatusCode {
case http.StatusOK, http.StatusNoContent:
// OK
case http.StatusNotFound:
log.Printf("[INFO] module %q not found in registry", src)
return "", false, nil
default:
// anything else is an error:
log.Printf("[WARN] error getting download location for %q: %s resp:%s", src, resp.Status, body)
return "", false, nil
}
// the download location is in the X-Terraform-Get header
location := resp.Header.Get(xTerraformGet)
if location == "" {
return "", false, fmt.Errorf("failed to get download URL for %q: %s resp:%s", src, resp.Status, body)
}
return location, true, nil
}

143
config/module/get_test.go Normal file
View File

@ -0,0 +1,143 @@
package module
import (
"fmt"
"net/http"
"net/http/httptest"
"regexp"
"sort"
"strings"
"testing"
version "github.com/hashicorp/go-version"
)
// map of module names and version for test module.
// only one version for now, as we only lookup latest from the registry
var testMods = map[string]string{
"registry/foo/bar": "0.2.3",
"registry/foo/baz": "1.10.0",
}
func latestVersion(versions []string) string {
var col version.Collection
for _, v := range versions {
ver, err := version.NewVersion(v)
if err != nil {
panic(err)
}
col = append(col, ver)
}
sort.Sort(col)
return col[len(col)-1].String()
}
// Just enough like a registry to exercise our code.
// Returns the location of the latest version
func mockRegistry() *httptest.Server {
mux := http.NewServeMux()
server := httptest.NewServer(mux)
mux.Handle("/v1/modules/",
http.StripPrefix("/v1/modules/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
p := strings.TrimLeft(r.URL.Path, "/")
// handle download request
download := regexp.MustCompile(`^(\w+/\w+/\w+)/download$`)
// download lookup
matches := download.FindStringSubmatch(p)
if len(matches) != 2 {
w.WriteHeader(http.StatusBadRequest)
return
}
version, ok := testMods[matches[1]]
if !ok {
w.WriteHeader(http.StatusNotFound)
return
}
location := fmt.Sprintf("%s/download/%s/%s", server.URL, matches[1], version)
w.Header().Set(xTerraformGet, location)
w.WriteHeader(http.StatusNoContent)
// no body
return
})),
)
return server
}
func TestDetectRegistry(t *testing.T) {
server := mockRegistry()
defer server.Close()
detector := registryDetector{
api: server.URL + "/v1/modules/",
client: server.Client(),
}
for _, tc := range []struct {
module string
location string
found bool
err bool
}{
{
module: "registry/foo/bar",
location: "download/registry/foo/bar/0.2.3",
found: true,
},
{
module: "registry/foo/baz",
location: "download/registry/foo/baz/1.10.0",
found: true,
},
// this should not be found, but not stop detection
{
module: "registry/foo/notfound",
found: false,
},
// a full url should not be detected
{
module: "http://example.com/registry/foo/notfound",
found: false,
},
// paths should not be detected
{
module: "./local/foo/notfound",
found: false,
},
{
module: "/local/foo/notfound",
found: false,
},
// wrong number of parts can't be regisry IDs
{
module: "something/registry/foo/notfound",
found: false,
},
} {
t.Run(tc.module, func(t *testing.T) {
loc, ok, err := detector.Detect(tc.module, "")
if (err == nil) == tc.err {
t.Fatalf("expected error? %t; got error :%v", tc.err, err)
}
if ok != tc.found {
t.Fatalf("expected OK == %t", tc.found)
}
loc = strings.TrimPrefix(loc, server.URL+"/")
if strings.TrimPrefix(loc, server.URL) != tc.location {
t.Fatalf("expected location: %q, got %q", tc.location, loc)
}
})
}
}

View File

@ -180,7 +180,7 @@ func (t *Tree) Load(s getter.Storage, mode GetMode) error {
// Split out the subdir if we have one // Split out the subdir if we have one
source, subDir := getter.SourceDirSubdir(m.Source) source, subDir := getter.SourceDirSubdir(m.Source)
source, err := getter.Detect(source, t.config.Dir, getter.Detectors) source, err := getter.Detect(source, t.config.Dir, detectors)
if err != nil { if err != nil {
return fmt.Errorf("module %s: %s", m.Name, err) return fmt.Errorf("module %s: %s", m.Name, err)
} }