diff --git a/config/module/get.go b/config/module/get.go index 96b4a63c3..ba9d29259 100644 --- a/config/module/get.go +++ b/config/module/get.go @@ -1,10 +1,18 @@ package module import ( + "fmt" "io/ioutil" + "log" + "net/http" + "net/url" "os" + "regexp" + "strings" "github.com/hashicorp/go-getter" + + cleanhttp "github.com/hashicorp/go-cleanhttp" ) // GetMode is an enum that describes how modules are loaded. @@ -69,3 +77,131 @@ func getStorage(s getter.Storage, key string, src string, mode GetMode) (string, // Get the directory where the module is. return s.Dir(key) } + +const ( + registryAPI = "https://registry.terraform.io/v1/modules/" + xTerraformGet = "X-Terraform-Get" +) + +var detectors = []getter.Detector{ + new(getter.GitHubDetector), + new(getter.BitBucketDetector), + new(getter.S3Detector), + new(localDetector), + new(registryDetector), +} + +// these prefixes can't be registry IDs +// "http", "./", "/", "getter::" +var skipRegistry = regexp.MustCompile(`^(http|\./|/|[A-Za-z0-9]+::)`).MatchString + +// registryDetector implements getter.Detector to detect Terraform Registry modules. +// If a path looks like a registry module identifier, attempt to locate it in +// the registry. If it's not found, pass it on in case it can be found by +// other means. +type registryDetector struct { + // override the default registry URL + api string + + client *http.Client +} + +func (d registryDetector) Detect(src, _ string) (string, bool, error) { + // the namespace can't start with "http", a relative or absolute path, or + // contain a go-getter "forced getter" + if skipRegistry(src) { + return "", false, nil + } + + // there are 3 parts to a registry ID + if len(strings.Split(src, "/")) != 3 { + return "", false, nil + } + + return d.lookupModule(src) +} + +// Lookup the module in the registry. +func (d registryDetector) lookupModule(src string) (string, bool, error) { + if d.api == "" { + d.api = registryAPI + } + + if d.client == nil { + d.client = cleanhttp.DefaultClient() + } + + // src is already partially validated in Detect. We know it's a path, and + // if it can be parsed as a URL we will hand it off to the registry to + // determine if it's truly valid. + resp, err := d.client.Get(fmt.Sprintf("%s/%s/download", d.api, src)) + if err != nil { + log.Printf("[WARN] error looking up module %q: %s", src, err) + return "", false, nil + } + defer resp.Body.Close() + + // there should be no body, but save it for logging + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Printf("[WARN] error reading response body from registry: %s", err) + return "", false, nil + } + + switch resp.StatusCode { + case http.StatusOK, http.StatusNoContent: + // OK + case http.StatusNotFound: + log.Printf("[INFO] module %q not found in registry", src) + return "", false, nil + default: + // anything else is an error: + log.Printf("[WARN] error getting download location for %q: %s resp:%s", src, resp.Status, body) + return "", false, nil + } + + // the download location is in the X-Terraform-Get header + location := resp.Header.Get(xTerraformGet) + if location == "" { + return "", false, fmt.Errorf("failed to get download URL for %q: %s resp:%s", src, resp.Status, body) + } + + return location, true, nil +} + +// localDetector wraps the default getter.FileDetector and checks if the module +// exists in the local filesystem. The default FileDetector only converts paths +// into file URLs, and returns found. We want to first check for a local module +// before passing it off to the registryDetector so we don't inadvertently +// replace a local module with a registry module of the same name. +type localDetector struct{} + +func (d localDetector) Detect(src, wd string) (string, bool, error) { + localSrc, ok, err := new(getter.FileDetector).Detect(src, wd) + if err != nil { + return src, ok, err + } + + if !ok { + return "", false, nil + } + + u, err := url.Parse(localSrc) + if err != nil { + return "", false, err + } + + _, err = os.Stat(u.Path) + + // just continue detection if it doesn't exist + if os.IsNotExist(err) { + return "", false, nil + } + + // return any other errors + if err != nil { + return "", false, err + } + + return localSrc, true, nil +} diff --git a/config/module/get_test.go b/config/module/get_test.go new file mode 100644 index 000000000..a24ea4a81 --- /dev/null +++ b/config/module/get_test.go @@ -0,0 +1,274 @@ +package module + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" + + getter "github.com/hashicorp/go-getter" + version "github.com/hashicorp/go-version" +) + +// map of module names and version for test module. +// only one version for now, as we only lookup latest from the registry +var testMods = map[string]string{ + "registry/foo/bar": "0.2.3", + "registry/foo/baz": "1.10.0", +} + +func latestVersion(versions []string) string { + var col version.Collection + for _, v := range versions { + ver, err := version.NewVersion(v) + if err != nil { + panic(err) + } + col = append(col, ver) + } + + sort.Sort(col) + return col[len(col)-1].String() +} + +// Just enough like a registry to exercise our code. +// Returns the location of the latest version +func mockRegistry() *httptest.Server { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + + mux.Handle("/v1/modules/", + http.StripPrefix("/v1/modules/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + p := strings.TrimLeft(r.URL.Path, "/") + // handle download request + download := regexp.MustCompile(`^(\w+/\w+/\w+)/download$`) + + // download lookup + matches := download.FindStringSubmatch(p) + if len(matches) != 2 { + w.WriteHeader(http.StatusBadRequest) + return + } + + version, ok := testMods[matches[1]] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + + location := fmt.Sprintf("%s/download/%s/%s", server.URL, matches[1], version) + w.Header().Set(xTerraformGet, location) + w.WriteHeader(http.StatusNoContent) + // no body + return + })), + ) + + return server +} + +func TestDetectRegistry(t *testing.T) { + server := mockRegistry() + defer server.Close() + + detector := registryDetector{ + api: server.URL + "/v1/modules/", + client: server.Client(), + } + + for _, tc := range []struct { + source string + location string + found bool + err bool + }{ + { + source: "registry/foo/bar", + location: "download/registry/foo/bar/0.2.3", + found: true, + }, + { + source: "registry/foo/baz", + location: "download/registry/foo/baz/1.10.0", + found: true, + }, + // this should not be found, but not stop detection + { + source: "registry/foo/notfound", + found: false, + }, + + // a full url should not be detected + { + source: "http://example.com/registry/foo/notfound", + found: false, + }, + + // paths should not be detected + { + source: "./local/foo/notfound", + found: false, + }, + { + source: "/local/foo/notfound", + found: false, + }, + + // wrong number of parts can't be regisry IDs + { + source: "something/registry/foo/notfound", + found: false, + }, + } { + + t.Run(tc.source, func(t *testing.T) { + loc, ok, err := detector.Detect(tc.source, "") + if (err == nil) == tc.err { + t.Fatalf("expected error? %t; got error :%v", tc.err, err) + } + + if ok != tc.found { + t.Fatalf("expected OK == %t", tc.found) + } + + loc = strings.TrimPrefix(loc, server.URL+"/") + if strings.TrimPrefix(loc, server.URL) != tc.location { + t.Fatalf("expected location: %q, got %q", tc.location, loc) + } + }) + + } +} + +// check that the full set of detectors works as expected +func TestDetectors(t *testing.T) { + server := mockRegistry() + defer server.Close() + + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + regDetector := ®istryDetector{ + api: server.URL + "/v1/modules/", + client: server.Client(), + } + + detectors := []getter.Detector{ + new(getter.GitHubDetector), + new(getter.BitBucketDetector), + new(getter.S3Detector), + new(localDetector), + regDetector, + } + + for _, tc := range []struct { + source string + location string + fixture string + err bool + }{ + { + source: "registry/foo/bar", + location: "download/registry/foo/bar/0.2.3", + }, + // this should not be found, but not stop detection + { + source: "registry/foo/notfound", + err: true, + }, + // a full url should be unchanged + { + source: "http://example.com/registry/foo/notfound?" + + "checksum=sha256:f19056b80a426d797ff9e470da069c171a6c6befa83e2da7f6c706207742acab", + location: "http://example.com/registry/foo/notfound?" + + "checksum=sha256:f19056b80a426d797ff9e470da069c171a6c6befa83e2da7f6c706207742acab", + }, + + // forced getters will return untouched + { + source: "git::http://example.com/registry/foo/notfound?param=value", + location: "git::http://example.com/registry/foo/notfound?param=value", + }, + + // local paths should be detected as such, even if they're match + // registry modules. + { + source: "./registry/foo/bar", + err: true, + }, + { + source: "/registry/foo/bar", + err: true, + }, + + // wrong number of parts can't be regisry IDs + { + source: "something/registry/foo/notfound", + err: true, + }, + + // make sure a local module that looks like a registry id takes precedence + { + source: "namespace/identifier/provider", + fixture: "discover-subdirs", + // this should be found locally + location: "file://" + filepath.Join(wd, fixtureDir, "discover-subdirs/namespace/identifier/provider"), + }, + } { + + t.Run(tc.source, func(t *testing.T) { + dir := wd + if tc.fixture != "" { + dir = filepath.Join(wd, fixtureDir, tc.fixture) + if err := os.Chdir(dir); err != nil { + t.Fatal(err) + } + defer os.Chdir(wd) + } + + loc, err := getter.Detect(tc.source, dir, detectors) + if (err == nil) == tc.err { + t.Fatalf("expected error? %t; got error :%v", tc.err, err) + } + + loc = strings.TrimPrefix(loc, server.URL+"/") + if strings.TrimPrefix(loc, server.URL) != tc.location { + t.Fatalf("expected location: %q, got %q", tc.location, loc) + } + }) + + } +} + +func TestAccRegistryDiscover(t *testing.T) { + if os.Getenv("TF_ACC") == "" { + t.Skip("skipping ACC test") + } + + // simply check that we get a valid github URL for this from the registry + loc, err := getter.Detect("hashicorp/consul/aws", "./", detectors) + if err != nil { + t.Fatal(err) + } + + u, err := url.Parse(loc) + if err != nil { + t.Fatal(err) + } + + if !strings.HasSuffix(u.Host, "github.com") { + t.Fatalf("expected host 'github.com', got: %q", u.Host) + } + + if !strings.Contains(u.String(), "consul") { + t.Fatalf("url doesn't contain 'consul': %s", u.String()) + } +} diff --git a/config/module/module_test.go b/config/module/module_test.go index 89fee6ec5..99f5edad7 100644 --- a/config/module/module_test.go +++ b/config/module/module_test.go @@ -13,6 +13,7 @@ import ( const fixtureDir = "./test-fixtures" func tempDir(t *testing.T) string { + t.Helper() dir, err := ioutil.TempDir("", "tf") if err != nil { t.Fatalf("err: %s", err) @@ -25,6 +26,7 @@ func tempDir(t *testing.T) string { } func testConfig(t *testing.T, n string) *config.Config { + t.Helper() c, err := config.LoadDir(filepath.Join(fixtureDir, n)) if err != nil { t.Fatalf("err: %s", err) @@ -34,5 +36,6 @@ func testConfig(t *testing.T, n string) *config.Config { } func testStorage(t *testing.T) getter.Storage { + t.Helper() return &getter.FolderStorage{StorageDir: tempDir(t)} } diff --git a/config/module/test-fixtures/discover-subdirs/main.tf b/config/module/test-fixtures/discover-subdirs/main.tf new file mode 100644 index 000000000..7ece92721 --- /dev/null +++ b/config/module/test-fixtures/discover-subdirs/main.tf @@ -0,0 +1,3 @@ +module "provider" { + source = "namespace/identifier/provider" +} diff --git a/config/module/test-fixtures/discover-subdirs/namespace/identifier/provider/main.tf b/config/module/test-fixtures/discover-subdirs/namespace/identifier/provider/main.tf new file mode 100644 index 000000000..907b0c7e4 --- /dev/null +++ b/config/module/test-fixtures/discover-subdirs/namespace/identifier/provider/main.tf @@ -0,0 +1,3 @@ +output "local" { + value = "test" +} diff --git a/config/module/tree.go b/config/module/tree.go index 4b0b153f7..a4398016a 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "fmt" - "path/filepath" "strings" "sync" @@ -177,23 +176,14 @@ func (t *Tree) Load(s getter.Storage, mode GetMode) error { copy(path, t.path) path = append(path, m.Name) - // Split out the subdir if we have one - source, subDir := getter.SourceDirSubdir(m.Source) - - source, err := getter.Detect(source, t.config.Dir, getter.Detectors) + source, err := getter.Detect(m.Source, t.config.Dir, detectors) if err != nil { return fmt.Errorf("module %s: %s", m.Name, err) } - - // Check if the detector introduced something new. - source, subDir2 := getter.SourceDirSubdir(source) - if subDir2 != "" { - subDir = filepath.Join(subDir2, subDir) - } - // Get the directory where this module is so we can load it key := strings.Join(path, ".") - key = fmt.Sprintf("root.%s-%s", key, m.Source) + key = fmt.Sprintf("module.%s-%s", key, m.Source) + dir, ok, err := getStorage(s, key, source, mode) if err != nil { return err @@ -203,12 +193,6 @@ func (t *Tree) Load(s getter.Storage, mode GetMode) error { "module %s: not found, may need to be downloaded using 'terraform get'", m.Name) } - // If we have a subdirectory, then merge that in - if subDir != "" { - dir = filepath.Join(dir, subDir) - } - - // Load the configurations.Dir(source) children[m.Name], err = NewTreeModule(m.Name, dir) if err != nil { return fmt.Errorf(