Merge branch 'jbardin/module-registry' into v0.10.6-release
This commit is contained in:
commit
74955ffd4a
|
@ -1,10 +1,18 @@
|
||||||
package module
|
package module
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/hashicorp/go-getter"
|
"github.com/hashicorp/go-getter"
|
||||||
|
|
||||||
|
cleanhttp "github.com/hashicorp/go-cleanhttp"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GetMode is an enum that describes how modules are loaded.
|
// GetMode is an enum that describes how modules are loaded.
|
||||||
|
@ -69,3 +77,131 @@ func getStorage(s getter.Storage, key string, src string, mode GetMode) (string,
|
||||||
// Get the directory where the module is.
|
// Get the directory where the module is.
|
||||||
return s.Dir(key)
|
return s.Dir(key)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
registryAPI = "https://registry.terraform.io/v1/modules/"
|
||||||
|
xTerraformGet = "X-Terraform-Get"
|
||||||
|
)
|
||||||
|
|
||||||
|
var detectors = []getter.Detector{
|
||||||
|
new(getter.GitHubDetector),
|
||||||
|
new(getter.BitBucketDetector),
|
||||||
|
new(getter.S3Detector),
|
||||||
|
new(localDetector),
|
||||||
|
new(registryDetector),
|
||||||
|
}
|
||||||
|
|
||||||
|
// these prefixes can't be registry IDs
|
||||||
|
// "http", "./", "/", "getter::"
|
||||||
|
var skipRegistry = regexp.MustCompile(`^(http|\./|/|[A-Za-z0-9]+::)`).MatchString
|
||||||
|
|
||||||
|
// registryDetector implements getter.Detector to detect Terraform Registry modules.
|
||||||
|
// If a path looks like a registry module identifier, attempt to locate it in
|
||||||
|
// the registry. If it's not found, pass it on in case it can be found by
|
||||||
|
// other means.
|
||||||
|
type registryDetector struct {
|
||||||
|
// override the default registry URL
|
||||||
|
api string
|
||||||
|
|
||||||
|
client *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d registryDetector) Detect(src, _ string) (string, bool, error) {
|
||||||
|
// the namespace can't start with "http", a relative or absolute path, or
|
||||||
|
// contain a go-getter "forced getter"
|
||||||
|
if skipRegistry(src) {
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// there are 3 parts to a registry ID
|
||||||
|
if len(strings.Split(src, "/")) != 3 {
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return d.lookupModule(src)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Lookup the module in the registry.
|
||||||
|
func (d registryDetector) lookupModule(src string) (string, bool, error) {
|
||||||
|
if d.api == "" {
|
||||||
|
d.api = registryAPI
|
||||||
|
}
|
||||||
|
|
||||||
|
if d.client == nil {
|
||||||
|
d.client = cleanhttp.DefaultClient()
|
||||||
|
}
|
||||||
|
|
||||||
|
// src is already partially validated in Detect. We know it's a path, and
|
||||||
|
// if it can be parsed as a URL we will hand it off to the registry to
|
||||||
|
// determine if it's truly valid.
|
||||||
|
resp, err := d.client.Get(fmt.Sprintf("%s/%s/download", d.api, src))
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[WARN] error looking up module %q: %s", src, err)
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
// there should be no body, but save it for logging
|
||||||
|
body, err := ioutil.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("[WARN] error reading response body from registry: %s", err)
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
switch resp.StatusCode {
|
||||||
|
case http.StatusOK, http.StatusNoContent:
|
||||||
|
// OK
|
||||||
|
case http.StatusNotFound:
|
||||||
|
log.Printf("[INFO] module %q not found in registry", src)
|
||||||
|
return "", false, nil
|
||||||
|
default:
|
||||||
|
// anything else is an error:
|
||||||
|
log.Printf("[WARN] error getting download location for %q: %s resp:%s", src, resp.Status, body)
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// the download location is in the X-Terraform-Get header
|
||||||
|
location := resp.Header.Get(xTerraformGet)
|
||||||
|
if location == "" {
|
||||||
|
return "", false, fmt.Errorf("failed to get download URL for %q: %s resp:%s", src, resp.Status, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
return location, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// localDetector wraps the default getter.FileDetector and checks if the module
|
||||||
|
// exists in the local filesystem. The default FileDetector only converts paths
|
||||||
|
// into file URLs, and returns found. We want to first check for a local module
|
||||||
|
// before passing it off to the registryDetector so we don't inadvertently
|
||||||
|
// replace a local module with a registry module of the same name.
|
||||||
|
type localDetector struct{}
|
||||||
|
|
||||||
|
func (d localDetector) Detect(src, wd string) (string, bool, error) {
|
||||||
|
localSrc, ok, err := new(getter.FileDetector).Detect(src, wd)
|
||||||
|
if err != nil {
|
||||||
|
return src, ok, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !ok {
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
u, err := url.Parse(localSrc)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = os.Stat(u.Path)
|
||||||
|
|
||||||
|
// just continue detection if it doesn't exist
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// return any other errors
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return localSrc, true, nil
|
||||||
|
}
|
||||||
|
|
|
@ -0,0 +1,274 @@
|
||||||
|
package module
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
getter "github.com/hashicorp/go-getter"
|
||||||
|
version "github.com/hashicorp/go-version"
|
||||||
|
)
|
||||||
|
|
||||||
|
// map of module names and version for test module.
|
||||||
|
// only one version for now, as we only lookup latest from the registry
|
||||||
|
var testMods = map[string]string{
|
||||||
|
"registry/foo/bar": "0.2.3",
|
||||||
|
"registry/foo/baz": "1.10.0",
|
||||||
|
}
|
||||||
|
|
||||||
|
func latestVersion(versions []string) string {
|
||||||
|
var col version.Collection
|
||||||
|
for _, v := range versions {
|
||||||
|
ver, err := version.NewVersion(v)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
col = append(col, ver)
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Sort(col)
|
||||||
|
return col[len(col)-1].String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Just enough like a registry to exercise our code.
|
||||||
|
// Returns the location of the latest version
|
||||||
|
func mockRegistry() *httptest.Server {
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
server := httptest.NewServer(mux)
|
||||||
|
|
||||||
|
mux.Handle("/v1/modules/",
|
||||||
|
http.StripPrefix("/v1/modules/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
p := strings.TrimLeft(r.URL.Path, "/")
|
||||||
|
// handle download request
|
||||||
|
download := regexp.MustCompile(`^(\w+/\w+/\w+)/download$`)
|
||||||
|
|
||||||
|
// download lookup
|
||||||
|
matches := download.FindStringSubmatch(p)
|
||||||
|
if len(matches) != 2 {
|
||||||
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
version, ok := testMods[matches[1]]
|
||||||
|
if !ok {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
location := fmt.Sprintf("%s/download/%s/%s", server.URL, matches[1], version)
|
||||||
|
w.Header().Set(xTerraformGet, location)
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
// no body
|
||||||
|
return
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
|
||||||
|
return server
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectRegistry(t *testing.T) {
|
||||||
|
server := mockRegistry()
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
detector := registryDetector{
|
||||||
|
api: server.URL + "/v1/modules/",
|
||||||
|
client: server.Client(),
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
source string
|
||||||
|
location string
|
||||||
|
found bool
|
||||||
|
err bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
source: "registry/foo/bar",
|
||||||
|
location: "download/registry/foo/bar/0.2.3",
|
||||||
|
found: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
source: "registry/foo/baz",
|
||||||
|
location: "download/registry/foo/baz/1.10.0",
|
||||||
|
found: true,
|
||||||
|
},
|
||||||
|
// this should not be found, but not stop detection
|
||||||
|
{
|
||||||
|
source: "registry/foo/notfound",
|
||||||
|
found: false,
|
||||||
|
},
|
||||||
|
|
||||||
|
// a full url should not be detected
|
||||||
|
{
|
||||||
|
source: "http://example.com/registry/foo/notfound",
|
||||||
|
found: false,
|
||||||
|
},
|
||||||
|
|
||||||
|
// paths should not be detected
|
||||||
|
{
|
||||||
|
source: "./local/foo/notfound",
|
||||||
|
found: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
source: "/local/foo/notfound",
|
||||||
|
found: false,
|
||||||
|
},
|
||||||
|
|
||||||
|
// wrong number of parts can't be regisry IDs
|
||||||
|
{
|
||||||
|
source: "something/registry/foo/notfound",
|
||||||
|
found: false,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
|
||||||
|
t.Run(tc.source, func(t *testing.T) {
|
||||||
|
loc, ok, err := detector.Detect(tc.source, "")
|
||||||
|
if (err == nil) == tc.err {
|
||||||
|
t.Fatalf("expected error? %t; got error :%v", tc.err, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if ok != tc.found {
|
||||||
|
t.Fatalf("expected OK == %t", tc.found)
|
||||||
|
}
|
||||||
|
|
||||||
|
loc = strings.TrimPrefix(loc, server.URL+"/")
|
||||||
|
if strings.TrimPrefix(loc, server.URL) != tc.location {
|
||||||
|
t.Fatalf("expected location: %q, got %q", tc.location, loc)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// check that the full set of detectors works as expected
|
||||||
|
func TestDetectors(t *testing.T) {
|
||||||
|
server := mockRegistry()
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
wd, err := os.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
regDetector := ®istryDetector{
|
||||||
|
api: server.URL + "/v1/modules/",
|
||||||
|
client: server.Client(),
|
||||||
|
}
|
||||||
|
|
||||||
|
detectors := []getter.Detector{
|
||||||
|
new(getter.GitHubDetector),
|
||||||
|
new(getter.BitBucketDetector),
|
||||||
|
new(getter.S3Detector),
|
||||||
|
new(localDetector),
|
||||||
|
regDetector,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
source string
|
||||||
|
location string
|
||||||
|
fixture string
|
||||||
|
err bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
source: "registry/foo/bar",
|
||||||
|
location: "download/registry/foo/bar/0.2.3",
|
||||||
|
},
|
||||||
|
// this should not be found, but not stop detection
|
||||||
|
{
|
||||||
|
source: "registry/foo/notfound",
|
||||||
|
err: true,
|
||||||
|
},
|
||||||
|
// a full url should be unchanged
|
||||||
|
{
|
||||||
|
source: "http://example.com/registry/foo/notfound?" +
|
||||||
|
"checksum=sha256:f19056b80a426d797ff9e470da069c171a6c6befa83e2da7f6c706207742acab",
|
||||||
|
location: "http://example.com/registry/foo/notfound?" +
|
||||||
|
"checksum=sha256:f19056b80a426d797ff9e470da069c171a6c6befa83e2da7f6c706207742acab",
|
||||||
|
},
|
||||||
|
|
||||||
|
// forced getters will return untouched
|
||||||
|
{
|
||||||
|
source: "git::http://example.com/registry/foo/notfound?param=value",
|
||||||
|
location: "git::http://example.com/registry/foo/notfound?param=value",
|
||||||
|
},
|
||||||
|
|
||||||
|
// local paths should be detected as such, even if they're match
|
||||||
|
// registry modules.
|
||||||
|
{
|
||||||
|
source: "./registry/foo/bar",
|
||||||
|
err: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
source: "/registry/foo/bar",
|
||||||
|
err: true,
|
||||||
|
},
|
||||||
|
|
||||||
|
// wrong number of parts can't be regisry IDs
|
||||||
|
{
|
||||||
|
source: "something/registry/foo/notfound",
|
||||||
|
err: true,
|
||||||
|
},
|
||||||
|
|
||||||
|
// make sure a local module that looks like a registry id takes precedence
|
||||||
|
{
|
||||||
|
source: "namespace/identifier/provider",
|
||||||
|
fixture: "discover-subdirs",
|
||||||
|
// this should be found locally
|
||||||
|
location: "file://" + filepath.Join(wd, fixtureDir, "discover-subdirs/namespace/identifier/provider"),
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
|
||||||
|
t.Run(tc.source, func(t *testing.T) {
|
||||||
|
dir := wd
|
||||||
|
if tc.fixture != "" {
|
||||||
|
dir = filepath.Join(wd, fixtureDir, tc.fixture)
|
||||||
|
if err := os.Chdir(dir); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer os.Chdir(wd)
|
||||||
|
}
|
||||||
|
|
||||||
|
loc, err := getter.Detect(tc.source, dir, detectors)
|
||||||
|
if (err == nil) == tc.err {
|
||||||
|
t.Fatalf("expected error? %t; got error :%v", tc.err, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
loc = strings.TrimPrefix(loc, server.URL+"/")
|
||||||
|
if strings.TrimPrefix(loc, server.URL) != tc.location {
|
||||||
|
t.Fatalf("expected location: %q, got %q", tc.location, loc)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccRegistryDiscover(t *testing.T) {
|
||||||
|
if os.Getenv("TF_ACC") == "" {
|
||||||
|
t.Skip("skipping ACC test")
|
||||||
|
}
|
||||||
|
|
||||||
|
// simply check that we get a valid github URL for this from the registry
|
||||||
|
loc, err := getter.Detect("hashicorp/consul/aws", "./", detectors)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
u, err := url.Parse(loc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.HasSuffix(u.Host, "github.com") {
|
||||||
|
t.Fatalf("expected host 'github.com', got: %q", u.Host)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(u.String(), "consul") {
|
||||||
|
t.Fatalf("url doesn't contain 'consul': %s", u.String())
|
||||||
|
}
|
||||||
|
}
|
|
@ -13,6 +13,7 @@ import (
|
||||||
const fixtureDir = "./test-fixtures"
|
const fixtureDir = "./test-fixtures"
|
||||||
|
|
||||||
func tempDir(t *testing.T) string {
|
func tempDir(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
dir, err := ioutil.TempDir("", "tf")
|
dir, err := ioutil.TempDir("", "tf")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("err: %s", err)
|
t.Fatalf("err: %s", err)
|
||||||
|
@ -25,6 +26,7 @@ func tempDir(t *testing.T) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func testConfig(t *testing.T, n string) *config.Config {
|
func testConfig(t *testing.T, n string) *config.Config {
|
||||||
|
t.Helper()
|
||||||
c, err := config.LoadDir(filepath.Join(fixtureDir, n))
|
c, err := config.LoadDir(filepath.Join(fixtureDir, n))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("err: %s", err)
|
t.Fatalf("err: %s", err)
|
||||||
|
@ -34,5 +36,6 @@ func testConfig(t *testing.T, n string) *config.Config {
|
||||||
}
|
}
|
||||||
|
|
||||||
func testStorage(t *testing.T) getter.Storage {
|
func testStorage(t *testing.T) getter.Storage {
|
||||||
|
t.Helper()
|
||||||
return &getter.FolderStorage{StorageDir: tempDir(t)}
|
return &getter.FolderStorage{StorageDir: tempDir(t)}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,3 @@
|
||||||
|
module "provider" {
|
||||||
|
source = "namespace/identifier/provider"
|
||||||
|
}
|
|
@ -0,0 +1,3 @@
|
||||||
|
output "local" {
|
||||||
|
value = "test"
|
||||||
|
}
|
|
@ -4,7 +4,6 @@ import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"bytes"
|
"bytes"
|
||||||
"fmt"
|
"fmt"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
@ -177,23 +176,14 @@ func (t *Tree) Load(s getter.Storage, mode GetMode) error {
|
||||||
copy(path, t.path)
|
copy(path, t.path)
|
||||||
path = append(path, m.Name)
|
path = append(path, m.Name)
|
||||||
|
|
||||||
// Split out the subdir if we have one
|
source, err := getter.Detect(m.Source, t.config.Dir, detectors)
|
||||||
source, subDir := getter.SourceDirSubdir(m.Source)
|
|
||||||
|
|
||||||
source, err := getter.Detect(source, t.config.Dir, getter.Detectors)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("module %s: %s", m.Name, err)
|
return fmt.Errorf("module %s: %s", m.Name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if the detector introduced something new.
|
|
||||||
source, subDir2 := getter.SourceDirSubdir(source)
|
|
||||||
if subDir2 != "" {
|
|
||||||
subDir = filepath.Join(subDir2, subDir)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the directory where this module is so we can load it
|
// Get the directory where this module is so we can load it
|
||||||
key := strings.Join(path, ".")
|
key := strings.Join(path, ".")
|
||||||
key = fmt.Sprintf("root.%s-%s", key, m.Source)
|
key = fmt.Sprintf("module.%s-%s", key, m.Source)
|
||||||
|
|
||||||
dir, ok, err := getStorage(s, key, source, mode)
|
dir, ok, err := getStorage(s, key, source, mode)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -203,12 +193,6 @@ func (t *Tree) Load(s getter.Storage, mode GetMode) error {
|
||||||
"module %s: not found, may need to be downloaded using 'terraform get'", m.Name)
|
"module %s: not found, may need to be downloaded using 'terraform get'", m.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we have a subdirectory, then merge that in
|
|
||||||
if subDir != "" {
|
|
||||||
dir = filepath.Join(dir, subDir)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Load the configurations.Dir(source)
|
|
||||||
children[m.Name], err = NewTreeModule(m.Name, dir)
|
children[m.Name], err = NewTreeModule(m.Name, dir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
|
|
Loading…
Reference in New Issue