Merge branch 'jbardin/module-registry' into v0.10.6-release
This commit is contained in:
James Bardin
commit
74955ffd4a
|
|
@ -1,10 +1,18 @@
|
|||
package module
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/hashicorp/go-getter"
|
||||
|
||||
cleanhttp "github.com/hashicorp/go-cleanhttp"
|
||||
)
|
||||
|
||||
// GetMode is an enum that describes how modules are loaded.
|
||||
|
|
@ -69,3 +77,131 @@ func getStorage(s getter.Storage, key string, src string, mode GetMode) (string,
|
|||
// Get the directory where the module is.
|
||||
return s.Dir(key)
|
||||
}
|
||||
|
||||
const (
|
||||
registryAPI = "https://registry.terraform.io/v1/modules/"
|
||||
xTerraformGet = "X-Terraform-Get"
|
||||
)
|
||||
|
||||
var detectors = []getter.Detector{
|
||||
new(getter.GitHubDetector),
|
||||
new(getter.BitBucketDetector),
|
||||
new(getter.S3Detector),
|
||||
new(localDetector),
|
||||
new(registryDetector),
|
||||
}
|
||||
|
||||
// these prefixes can't be registry IDs
|
||||
// "http", "./", "/", "getter::"
|
||||
var skipRegistry = regexp.MustCompile(`^(http|\./|/|[A-Za-z0-9]+::)`).MatchString
|
||||
|
||||
// registryDetector implements getter.Detector to detect Terraform Registry modules.
|
||||
// If a path looks like a registry module identifier, attempt to locate it in
|
||||
// the registry. If it's not found, pass it on in case it can be found by
|
||||
// other means.
|
||||
type registryDetector struct {
|
||||
// override the default registry URL
|
||||
api string
|
||||
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (d registryDetector) Detect(src, _ string) (string, bool, error) {
|
||||
// the namespace can't start with "http", a relative or absolute path, or
|
||||
// contain a go-getter "forced getter"
|
||||
if skipRegistry(src) {
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// there are 3 parts to a registry ID
|
||||
if len(strings.Split(src, "/")) != 3 {
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
return d.lookupModule(src)
|
||||
}
|
||||
|
||||
// Lookup the module in the registry.
|
||||
func (d registryDetector) lookupModule(src string) (string, bool, error) {
|
||||
if d.api == "" {
|
||||
d.api = registryAPI
|
||||
}
|
||||
|
||||
if d.client == nil {
|
||||
d.client = cleanhttp.DefaultClient()
|
||||
}
|
||||
|
||||
// src is already partially validated in Detect. We know it's a path, and
|
||||
// if it can be parsed as a URL we will hand it off to the registry to
|
||||
// determine if it's truly valid.
|
||||
resp, err := d.client.Get(fmt.Sprintf("%s/%s/download", d.api, src))
|
||||
if err != nil {
|
||||
log.Printf("[WARN] error looking up module %q: %s", src, err)
|
||||
return "", false, nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// there should be no body, but save it for logging
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
fmt.Printf("[WARN] error reading response body from registry: %s", err)
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK, http.StatusNoContent:
|
||||
// OK
|
||||
case http.StatusNotFound:
|
||||
log.Printf("[INFO] module %q not found in registry", src)
|
||||
return "", false, nil
|
||||
default:
|
||||
// anything else is an error:
|
||||
log.Printf("[WARN] error getting download location for %q: %s resp:%s", src, resp.Status, body)
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// the download location is in the X-Terraform-Get header
|
||||
location := resp.Header.Get(xTerraformGet)
|
||||
if location == "" {
|
||||
return "", false, fmt.Errorf("failed to get download URL for %q: %s resp:%s", src, resp.Status, body)
|
||||
}
|
||||
|
||||
return location, true, nil
|
||||
}
|
||||
|
||||
// localDetector wraps the default getter.FileDetector and checks if the module
|
||||
// exists in the local filesystem. The default FileDetector only converts paths
|
||||
// into file URLs, and returns found. We want to first check for a local module
|
||||
// before passing it off to the registryDetector so we don't inadvertently
|
||||
// replace a local module with a registry module of the same name.
|
||||
type localDetector struct{}
|
||||
|
||||
func (d localDetector) Detect(src, wd string) (string, bool, error) {
|
||||
localSrc, ok, err := new(getter.FileDetector).Detect(src, wd)
|
||||
if err != nil {
|
||||
return src, ok, err
|
||||
}
|
||||
|
||||
if !ok {
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
u, err := url.Parse(localSrc)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
|
||||
_, err = os.Stat(u.Path)
|
||||
|
||||
// just continue detection if it doesn't exist
|
||||
if os.IsNotExist(err) {
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// return any other errors
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
|
||||
return localSrc, true, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,274 @@
|
|||
package module
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
getter "github.com/hashicorp/go-getter"
|
||||
version "github.com/hashicorp/go-version"
|
||||
)
|
||||
|
||||
// map of module names and version for test module.
|
||||
// only one version for now, as we only lookup latest from the registry
|
||||
var testMods = map[string]string{
|
||||
"registry/foo/bar": "0.2.3",
|
||||
"registry/foo/baz": "1.10.0",
|
||||
}
|
||||
|
||||
func latestVersion(versions []string) string {
|
||||
var col version.Collection
|
||||
for _, v := range versions {
|
||||
ver, err := version.NewVersion(v)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
col = append(col, ver)
|
||||
}
|
||||
|
||||
sort.Sort(col)
|
||||
return col[len(col)-1].String()
|
||||
}
|
||||
|
||||
// Just enough like a registry to exercise our code.
|
||||
// Returns the location of the latest version
|
||||
func mockRegistry() *httptest.Server {
|
||||
mux := http.NewServeMux()
|
||||
server := httptest.NewServer(mux)
|
||||
|
||||
mux.Handle("/v1/modules/",
|
||||
http.StripPrefix("/v1/modules/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
p := strings.TrimLeft(r.URL.Path, "/")
|
||||
// handle download request
|
||||
download := regexp.MustCompile(`^(\w+/\w+/\w+)/download$`)
|
||||
|
||||
// download lookup
|
||||
matches := download.FindStringSubmatch(p)
|
||||
if len(matches) != 2 {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
version, ok := testMods[matches[1]]
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
location := fmt.Sprintf("%s/download/%s/%s", server.URL, matches[1], version)
|
||||
w.Header().Set(xTerraformGet, location)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
// no body
|
||||
return
|
||||
})),
|
||||
)
|
||||
|
||||
return server
|
||||
}
|
||||
|
||||
func TestDetectRegistry(t *testing.T) {
|
||||
server := mockRegistry()
|
||||
defer server.Close()
|
||||
|
||||
detector := registryDetector{
|
||||
api: server.URL + "/v1/modules/",
|
||||
client: server.Client(),
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
source string
|
||||
location string
|
||||
found bool
|
||||
err bool
|
||||
}{
|
||||
{
|
||||
source: "registry/foo/bar",
|
||||
location: "download/registry/foo/bar/0.2.3",
|
||||
found: true,
|
||||
},
|
||||
{
|
||||
source: "registry/foo/baz",
|
||||
location: "download/registry/foo/baz/1.10.0",
|
||||
found: true,
|
||||
},
|
||||
// this should not be found, but not stop detection
|
||||
{
|
||||
source: "registry/foo/notfound",
|
||||
found: false,
|
||||
},
|
||||
|
||||
// a full url should not be detected
|
||||
{
|
||||
source: "http://example.com/registry/foo/notfound",
|
||||
found: false,
|
||||
},
|
||||
|
||||
// paths should not be detected
|
||||
{
|
||||
source: "./local/foo/notfound",
|
||||
found: false,
|
||||
},
|
||||
{
|
||||
source: "/local/foo/notfound",
|
||||
found: false,
|
||||
},
|
||||
|
||||
// wrong number of parts can't be regisry IDs
|
||||
{
|
||||
source: "something/registry/foo/notfound",
|
||||
found: false,
|
||||
},
|
||||
} {
|
||||
|
||||
t.Run(tc.source, func(t *testing.T) {
|
||||
loc, ok, err := detector.Detect(tc.source, "")
|
||||
if (err == nil) == tc.err {
|
||||
t.Fatalf("expected error? %t; got error :%v", tc.err, err)
|
||||
}
|
||||
|
||||
if ok != tc.found {
|
||||
t.Fatalf("expected OK == %t", tc.found)
|
||||
}
|
||||
|
||||
loc = strings.TrimPrefix(loc, server.URL+"/")
|
||||
if strings.TrimPrefix(loc, server.URL) != tc.location {
|
||||
t.Fatalf("expected location: %q, got %q", tc.location, loc)
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// check that the full set of detectors works as expected
|
||||
func TestDetectors(t *testing.T) {
|
||||
server := mockRegistry()
|
||||
defer server.Close()
|
||||
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
regDetector := ®istryDetector{
|
||||
api: server.URL + "/v1/modules/",
|
||||
client: server.Client(),
|
||||
}
|
||||
|
||||
detectors := []getter.Detector{
|
||||
new(getter.GitHubDetector),
|
||||
new(getter.BitBucketDetector),
|
||||
new(getter.S3Detector),
|
||||
new(localDetector),
|
||||
regDetector,
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
source string
|
||||
location string
|
||||
fixture string
|
||||
err bool
|
||||
}{
|
||||
{
|
||||
source: "registry/foo/bar",
|
||||
location: "download/registry/foo/bar/0.2.3",
|
||||
},
|
||||
// this should not be found, but not stop detection
|
||||
{
|
||||
source: "registry/foo/notfound",
|
||||
err: true,
|
||||
},
|
||||
// a full url should be unchanged
|
||||
{
|
||||
source: "http://example.com/registry/foo/notfound?" +
|
||||
"checksum=sha256:f19056b80a426d797ff9e470da069c171a6c6befa83e2da7f6c706207742acab",
|
||||
location: "http://example.com/registry/foo/notfound?" +
|
||||
"checksum=sha256:f19056b80a426d797ff9e470da069c171a6c6befa83e2da7f6c706207742acab",
|
||||
},
|
||||
|
||||
// forced getters will return untouched
|
||||
{
|
||||
source: "git::http://example.com/registry/foo/notfound?param=value",
|
||||
location: "git::http://example.com/registry/foo/notfound?param=value",
|
||||
},
|
||||
|
||||
// local paths should be detected as such, even if they're match
|
||||
// registry modules.
|
||||
{
|
||||
source: "./registry/foo/bar",
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
source: "/registry/foo/bar",
|
||||
err: true,
|
||||
},
|
||||
|
||||
// wrong number of parts can't be regisry IDs
|
||||
{
|
||||
source: "something/registry/foo/notfound",
|
||||
err: true,
|
||||
},
|
||||
|
||||
// make sure a local module that looks like a registry id takes precedence
|
||||
{
|
||||
source: "namespace/identifier/provider",
|
||||
fixture: "discover-subdirs",
|
||||
// this should be found locally
|
||||
location: "file://" + filepath.Join(wd, fixtureDir, "discover-subdirs/namespace/identifier/provider"),
|
||||
},
|
||||
} {
|
||||
|
||||
t.Run(tc.source, func(t *testing.T) {
|
||||
dir := wd
|
||||
if tc.fixture != "" {
|
||||
dir = filepath.Join(wd, fixtureDir, tc.fixture)
|
||||
if err := os.Chdir(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.Chdir(wd)
|
||||
}
|
||||
|
||||
loc, err := getter.Detect(tc.source, dir, detectors)
|
||||
if (err == nil) == tc.err {
|
||||
t.Fatalf("expected error? %t; got error :%v", tc.err, err)
|
||||
}
|
||||
|
||||
loc = strings.TrimPrefix(loc, server.URL+"/")
|
||||
if strings.TrimPrefix(loc, server.URL) != tc.location {
|
||||
t.Fatalf("expected location: %q, got %q", tc.location, loc)
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccRegistryDiscover(t *testing.T) {
|
||||
if os.Getenv("TF_ACC") == "" {
|
||||
t.Skip("skipping ACC test")
|
||||
}
|
||||
|
||||
// simply check that we get a valid github URL for this from the registry
|
||||
loc, err := getter.Detect("hashicorp/consul/aws", "./", detectors)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
u, err := url.Parse(loc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(u.Host, "github.com") {
|
||||
t.Fatalf("expected host 'github.com', got: %q", u.Host)
|
||||
}
|
||||
|
||||
if !strings.Contains(u.String(), "consul") {
|
||||
t.Fatalf("url doesn't contain 'consul': %s", u.String())
|
||||
}
|
||||
}
|
||||
|
|
@ -13,6 +13,7 @@ import (
|
|||
const fixtureDir = "./test-fixtures"
|
||||
|
||||
func tempDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir, err := ioutil.TempDir("", "tf")
|
||||
if err != nil {
|
||||
t.Fatalf("err: %s", err)
|
||||
|
|
@ -25,6 +26,7 @@ func tempDir(t *testing.T) string {
|
|||
}
|
||||
|
||||
func testConfig(t *testing.T, n string) *config.Config {
|
||||
t.Helper()
|
||||
c, err := config.LoadDir(filepath.Join(fixtureDir, n))
|
||||
if err != nil {
|
||||
t.Fatalf("err: %s", err)
|
||||
|
|
@ -34,5 +36,6 @@ func testConfig(t *testing.T, n string) *config.Config {
|
|||
}
|
||||
|
||||
func testStorage(t *testing.T) getter.Storage {
|
||||
t.Helper()
|
||||
return &getter.FolderStorage{StorageDir: tempDir(t)}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
module "provider" {
|
||||
source = "namespace/identifier/provider"
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
output "local" {
|
||||
value = "test"
|
||||
}
|
||||
|
|
@ -4,7 +4,6 @@ import (
|
|||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
|
|
@ -177,23 +176,14 @@ func (t *Tree) Load(s getter.Storage, mode GetMode) error {
|
|||
copy(path, t.path)
|
||||
path = append(path, m.Name)
|
||||
|
||||
// Split out the subdir if we have one
|
||||
source, subDir := getter.SourceDirSubdir(m.Source)
|
||||
|
||||
source, err := getter.Detect(source, t.config.Dir, getter.Detectors)
|
||||
source, err := getter.Detect(m.Source, t.config.Dir, detectors)
|
||||
if err != nil {
|
||||
return fmt.Errorf("module %s: %s", m.Name, err)
|
||||
}
|
||||
|
||||
// Check if the detector introduced something new.
|
||||
source, subDir2 := getter.SourceDirSubdir(source)
|
||||
if subDir2 != "" {
|
||||
subDir = filepath.Join(subDir2, subDir)
|
||||
}
|
||||
|
||||
// Get the directory where this module is so we can load it
|
||||
key := strings.Join(path, ".")
|
||||
key = fmt.Sprintf("root.%s-%s", key, m.Source)
|
||||
key = fmt.Sprintf("module.%s-%s", key, m.Source)
|
||||
|
||||
dir, ok, err := getStorage(s, key, source, mode)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -203,12 +193,6 @@ func (t *Tree) Load(s getter.Storage, mode GetMode) error {
|
|||
"module %s: not found, may need to be downloaded using 'terraform get'", m.Name)
|
||||
}
|
||||
|
||||
// If we have a subdirectory, then merge that in
|
||||
if subDir != "" {
|
||||
dir = filepath.Join(dir, subDir)
|
||||
}
|
||||
|
||||
// Load the configurations.Dir(source)
|
||||
children[m.Name], err = NewTreeModule(m.Name, dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
|
|
|
|||
Loading…
Reference in New Issue