diff --git a/go.mod b/go.mod index d68dfd7cb..194b60ca7 100644 --- a/go.mod +++ b/go.mod @@ -36,12 +36,11 @@ require ( github.com/hashicorp/go-azure-helpers v0.18.0 github.com/hashicorp/go-checkpoint v0.5.0 github.com/hashicorp/go-cleanhttp v0.5.2 - github.com/hashicorp/go-getter v1.5.9 + github.com/hashicorp/go-getter v1.5.10 github.com/hashicorp/go-hclog v0.15.0 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-plugin v1.4.3 github.com/hashicorp/go-retryablehttp v0.7.0 - github.com/hashicorp/go-safetemp v1.0.0 github.com/hashicorp/go-tfe v0.21.0 github.com/hashicorp/go-uuid v1.0.2 github.com/hashicorp/go-version v1.3.0 @@ -145,6 +144,7 @@ require ( github.com/hashicorp/go-immutable-radix v1.0.0 // indirect github.com/hashicorp/go-msgpack v0.5.4 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-slug v0.7.0 // indirect github.com/hashicorp/golang-lru v0.5.1 // indirect github.com/hashicorp/jsonapi v0.0.0-20210826224640-ee7dae0fb22d // indirect diff --git a/go.sum b/go.sum index e7d439d84..8ef3722b7 100644 --- a/go.sum +++ b/go.sum @@ -375,8 +375,8 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320/go.mod h1:EiZBMaudVLy8fmjf9Npq1dq9RalhveqZG5w/yz3mHWs= github.com/hashicorp/go-getter v1.5.3/go.mod h1:BrrV/1clo8cCYu6mxvboYg+KutTiFnXjMEgDD8+i7ZI= -github.com/hashicorp/go-getter v1.5.9 h1:b7ahZW50iQiUek/at3CvZhPK1/jiV6CtKcsJiR6E4R0= -github.com/hashicorp/go-getter v1.5.9/go.mod h1:BrrV/1clo8cCYu6mxvboYg+KutTiFnXjMEgDD8+i7ZI= +github.com/hashicorp/go-getter v1.5.10 h1:EN9YigTlv5Ola0IuleFzQGuaYPPHHtWusP/5AypWEMs= +github.com/hashicorp/go-getter v1.5.10/go.mod h1:9i48BP6wpWweI/0/+FBjqLrp9S8XtwUGjiu0QkWHEaY= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= diff --git a/internal/getmodules/getter.go b/internal/getmodules/getter.go index ba4c8d89c..95f334762 100644 --- a/internal/getmodules/getter.go +++ b/internal/getmodules/getter.go @@ -73,7 +73,7 @@ var goGetterDecompressors = map[string]getter.Decompressor{ var goGetterGetters = map[string]getter.Getter{ "file": new(getter.FileGetter), "gcs": new(getter.GCSGetter), - "git": new(gitGetter), + "git": new(getter.GitGetter), "hg": new(getter.HgGetter), "s3": new(getter.S3Getter), "http": getterHTTPGetter, diff --git a/internal/getmodules/git_getter.go b/internal/getmodules/git_getter.go deleted file mode 100644 index 1b811b8fb..000000000 --- a/internal/getmodules/git_getter.go +++ /dev/null @@ -1,416 +0,0 @@ -package getmodules - -import ( - "bytes" - "context" - "encoding/base64" - "fmt" - "io/ioutil" - "net/url" - "os" - "os/exec" - "path/filepath" - "regexp" - "runtime" - "strconv" - "strings" - "syscall" - - getter "github.com/hashicorp/go-getter" - urlhelper "github.com/hashicorp/go-getter/helper/url" - safetemp "github.com/hashicorp/go-safetemp" - version "github.com/hashicorp/go-version" -) - -// getter is our base getter; it regroups -// fields all getters have in common. -type getterCommon struct { - client *getter.Client -} - -func (g *getterCommon) SetClient(c *getter.Client) { g.client = c } - -// Context tries to returns the Contex from the getter's -// client. otherwise context.Background() is returned. -func (g *getterCommon) Context() context.Context { - if g == nil || g.client == nil { - return context.Background() - } - return g.client.Ctx -} - -// gitGetter is a temporary fork of getter.GitGetter to allow us to tactically -// fix https://github.com/hashicorp/terraform/issues/30119 only within -// Terraform. -// -// This should be only a brief workaround to help us decouple work on the -// Terraform CLI v1.1.1 release so that we can get it done without having to -// coordinate with every other go-getter caller first. However, this fork -// should be healed promptly after v1.1.1 by upstreaming something like this -// fix into upstream go-getter, so that other go-getter callers can also -// benefit from it. -type gitGetter struct { - getterCommon -} - -var defaultBranchRegexp = regexp.MustCompile(`\s->\sorigin/(.*)`) -var lsRemoteSymRefRegexp = regexp.MustCompile(`ref: refs/heads/([^\s]+).*`) - -func (g *gitGetter) ClientMode(_ *url.URL) (getter.ClientMode, error) { - return getter.ClientModeDir, nil -} - -func (g *gitGetter) Get(dst string, u *url.URL) error { - ctx := g.Context() - if _, err := exec.LookPath("git"); err != nil { - return fmt.Errorf("git must be available and on the PATH") - } - - // The port number must be parseable as an integer. If not, the user - // was probably trying to use a scp-style address, in which case the - // ssh:// prefix must be removed to indicate that. - // - // This is not necessary in versions of Go which have patched - // CVE-2019-14809 (e.g. Go 1.12.8+) - if portStr := u.Port(); portStr != "" { - if _, err := strconv.ParseUint(portStr, 10, 16); err != nil { - return fmt.Errorf("invalid port number %q; if using the \"scp-like\" git address scheme where a colon introduces the path instead, remove the ssh:// portion and use just the git:: prefix", portStr) - } - } - - // Extract some query parameters we use - var ref, sshKey string - depth := 0 // 0 means "not set" - q := u.Query() - if len(q) > 0 { - ref = q.Get("ref") - q.Del("ref") - - sshKey = q.Get("sshkey") - q.Del("sshkey") - - if n, err := strconv.Atoi(q.Get("depth")); err == nil { - depth = n - } - q.Del("depth") - - // Copy the URL - var newU url.URL = *u - u = &newU - u.RawQuery = q.Encode() - } - - var sshKeyFile string - if sshKey != "" { - // Check that the git version is sufficiently new. - if err := checkGitVersion("2.3"); err != nil { - return fmt.Errorf("Error using ssh key: %v", err) - } - - // We have an SSH key - decode it. - raw, err := base64.StdEncoding.DecodeString(sshKey) - if err != nil { - return err - } - - // Create a temp file for the key and ensure it is removed. - fh, err := ioutil.TempFile("", "go-getter") - if err != nil { - return err - } - sshKeyFile = fh.Name() - defer os.Remove(sshKeyFile) - - // Set the permissions prior to writing the key material. - if err := os.Chmod(sshKeyFile, 0600); err != nil { - return err - } - - // Write the raw key into the temp file. - _, err = fh.Write(raw) - fh.Close() - if err != nil { - return err - } - } - - // Clone or update the repository - _, err := os.Stat(dst) - if err != nil && !os.IsNotExist(err) { - return err - } - if err == nil { - err = g.update(ctx, dst, sshKeyFile, ref, depth) - } else { - err = g.clone(ctx, dst, sshKeyFile, u, ref, depth) - } - if err != nil { - return err - } - - // Next: check out the proper tag/branch if it is specified, and checkout - if ref != "" { - if err := g.checkout(dst, ref); err != nil { - return err - } - } - - // Lastly, download any/all submodules. - return g.fetchSubmodules(ctx, dst, sshKeyFile, depth) -} - -// GetFile for Git doesn't support updating at this time. It will download -// the file every time. -func (g *gitGetter) GetFile(dst string, u *url.URL) error { - td, tdcloser, err := safetemp.Dir("", "getter") - if err != nil { - return err - } - defer tdcloser.Close() - - // Get the filename, and strip the filename from the URL so we can - // just get the repository directly. - filename := filepath.Base(u.Path) - u.Path = filepath.Dir(u.Path) - - // Get the full repository - if err := g.Get(td, u); err != nil { - return err - } - - // Copy the single file - u, err = urlhelper.Parse(fmtFileURL(filepath.Join(td, filename))) - if err != nil { - return err - } - - fg := &getter.FileGetter{Copy: true} - return fg.GetFile(dst, u) -} - -func (g *gitGetter) checkout(dst string, ref string) error { - cmd := exec.Command("git", "checkout", ref) - cmd.Dir = dst - return getRunCommand(cmd) -} - -// gitCommitIDRegex is a pattern intended to match strings that seem -// "likely to be" git commit IDs, rather than named refs. This cannot be -// an exact decision because it's valid to name a branch or tag after a series -// of hexadecimal digits too. -// -// We require at least 7 digits here because that's the smallest size git -// itself will typically generate, and so it'll reduce the risk of false -// positives on short branch names that happen to also be "hex words". -var gitCommitIDRegex = regexp.MustCompile("^[0-9a-fA-F]{7,40}$") - -func (g *gitGetter) clone(ctx context.Context, dst, sshKeyFile string, u *url.URL, ref string, depth int) error { - args := []string{"clone"} - - autoBranch := false - if ref == "" { - ref = findRemoteDefaultBranch(u) - autoBranch = true - } - if depth > 0 { - args = append(args, "--depth", strconv.Itoa(depth)) - args = append(args, "--branch", ref) - } - args = append(args, u.String(), dst) - - cmd := exec.CommandContext(ctx, "git", args...) - setupGitEnv(cmd, sshKeyFile) - err := getRunCommand(cmd) - if err != nil { - if depth > 0 && !autoBranch { - // If we're creating a shallow clone then the given ref must be - // a named ref (branch or tag) rather than a commit directly. - // We can't accurately recognize the resulting error here without - // hard-coding assumptions about git's human-readable output, but - // we can at least try a heuristic. - if gitCommitIDRegex.MatchString(ref) { - return fmt.Errorf("%w (note that setting 'depth' requires 'ref' to be a branch or tag name)", err) - } - } - return err - } - - if depth < 1 && !autoBranch { - // If we didn't add --depth and --branch above then we will now be - // on the remote repository's default branch, rather than the selected - // ref, so we'll need to fix that before we return. - return g.checkout(dst, ref) - } - return nil -} - -func (g *gitGetter) update(ctx context.Context, dst, sshKeyFile, ref string, depth int) error { - // Determine if we're a branch. If we're NOT a branch, then we just - // switch to master prior to checking out - cmd := exec.CommandContext(ctx, "git", "show-ref", "-q", "--verify", "refs/heads/"+ref) - cmd.Dir = dst - - if getRunCommand(cmd) != nil { - // Not a branch, switch to default branch. This will also catch - // non-existent branches, in which case we want to switch to default - // and then checkout the proper branch later. - ref = findDefaultBranch(dst) - } - - // We have to be on a branch to pull - if err := g.checkout(dst, ref); err != nil { - return err - } - - if depth > 0 { - cmd = exec.Command("git", "pull", "--depth", strconv.Itoa(depth), "--ff-only") - } else { - cmd = exec.Command("git", "pull", "--ff-only") - } - - cmd.Dir = dst - setupGitEnv(cmd, sshKeyFile) - return getRunCommand(cmd) -} - -// fetchSubmodules downloads any configured submodules recursively. -func (g *gitGetter) fetchSubmodules(ctx context.Context, dst, sshKeyFile string, depth int) error { - args := []string{"submodule", "update", "--init", "--recursive"} - if depth > 0 { - args = append(args, "--depth", strconv.Itoa(depth)) - } - cmd := exec.CommandContext(ctx, "git", args...) - cmd.Dir = dst - setupGitEnv(cmd, sshKeyFile) - return getRunCommand(cmd) -} - -// findDefaultBranch checks the repo's origin remote for its default branch -// (generally "master"). "master" is returned if an origin default branch -// can't be determined. -func findDefaultBranch(dst string) string { - var stdoutbuf bytes.Buffer - cmd := exec.Command("git", "branch", "-r", "--points-at", "refs/remotes/origin/HEAD") - cmd.Dir = dst - cmd.Stdout = &stdoutbuf - err := cmd.Run() - matches := defaultBranchRegexp.FindStringSubmatch(stdoutbuf.String()) - if err != nil || matches == nil { - return "master" - } - return matches[len(matches)-1] -} - -// findRemoteDefaultBranch checks the remote repo's HEAD symref to return the remote repo's -// default branch. "master" is returned if no HEAD symref exists. -func findRemoteDefaultBranch(u *url.URL) string { - var stdoutbuf bytes.Buffer - cmd := exec.Command("git", "ls-remote", "--symref", u.String(), "HEAD") - cmd.Stdout = &stdoutbuf - err := cmd.Run() - matches := lsRemoteSymRefRegexp.FindStringSubmatch(stdoutbuf.String()) - if err != nil || matches == nil { - return "master" - } - return matches[len(matches)-1] -} - -// setupGitEnv sets up the environment for the given command. This is used to -// pass configuration data to git and ssh and enables advanced cloning methods. -func setupGitEnv(cmd *exec.Cmd, sshKeyFile string) { - const gitSSHCommand = "GIT_SSH_COMMAND=" - var sshCmd []string - - // If we have an existing GIT_SSH_COMMAND, we need to append our options. - // We will also remove our old entry to make sure the behavior is the same - // with versions of Go < 1.9. - env := os.Environ() - for i, v := range env { - if strings.HasPrefix(v, gitSSHCommand) && len(v) > len(gitSSHCommand) { - sshCmd = []string{v} - - env[i], env[len(env)-1] = env[len(env)-1], env[i] - env = env[:len(env)-1] - break - } - } - - if len(sshCmd) == 0 { - sshCmd = []string{gitSSHCommand + "ssh"} - } - - if sshKeyFile != "" { - // We have an SSH key temp file configured, tell ssh about this. - if runtime.GOOS == "windows" { - sshKeyFile = strings.Replace(sshKeyFile, `\`, `/`, -1) - } - sshCmd = append(sshCmd, "-i", sshKeyFile) - } - - env = append(env, strings.Join(sshCmd, " ")) - cmd.Env = env -} - -// checkGitVersion is used to check the version of git installed on the system -// against a known minimum version. Returns an error if the installed version -// is older than the given minimum. -func checkGitVersion(min string) error { - want, err := version.NewVersion(min) - if err != nil { - return err - } - - out, err := exec.Command("git", "version").Output() - if err != nil { - return err - } - - fields := strings.Fields(string(out)) - if len(fields) < 3 { - return fmt.Errorf("Unexpected 'git version' output: %q", string(out)) - } - v := fields[2] - if runtime.GOOS == "windows" && strings.Contains(v, ".windows.") { - // on windows, git version will return for example: - // git version 2.20.1.windows.1 - // Which does not follow the semantic versionning specs - // https://semver.org. We remove that part in order for - // go-version to not error. - v = v[:strings.Index(v, ".windows.")] - } - - have, err := version.NewVersion(v) - if err != nil { - return err - } - - if have.LessThan(want) { - return fmt.Errorf("Required git version = %s, have %s", want, have) - } - - return nil -} - -// getRunCommand is a helper that will run a command and capture the output -// in the case an error happens. -func getRunCommand(cmd *exec.Cmd) error { - var buf bytes.Buffer - cmd.Stdout = &buf - cmd.Stderr = &buf - err := cmd.Run() - if err == nil { - return nil - } - if exiterr, ok := err.(*exec.ExitError); ok { - // The program has exited with an exit code != 0 - if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { - return fmt.Errorf( - "%s exited with %d: %s", - cmd.Path, - status.ExitStatus(), - buf.String()) - } - } - - return fmt.Errorf("error running %s: %s", cmd.Path, buf.String()) -} diff --git a/internal/getmodules/git_getter_test.go b/internal/getmodules/git_getter_test.go deleted file mode 100644 index 5893c9c8e..000000000 --- a/internal/getmodules/git_getter_test.go +++ /dev/null @@ -1,827 +0,0 @@ -package getmodules - -import ( - "bytes" - "encoding/base64" - "io/ioutil" - "net/url" - "os" - "os/exec" - "path/filepath" - "reflect" - "runtime" - "strings" - "testing" - - getter "github.com/hashicorp/go-getter" - urlhelper "github.com/hashicorp/go-getter/helper/url" -) - -var testHasGit bool - -func init() { - if _, err := exec.LookPath("git"); err == nil { - testHasGit = true - } -} - -func TestGitGetter_impl(t *testing.T) { - var _ getter.Getter = new(gitGetter) -} - -func TestGitGetter(t *testing.T) { - if !testHasGit { - t.Skip("git not found, skipping") - } - - g := new(gitGetter) - dst := tempDir(t) - - repo := testGitRepo(t, "basic") - repo.commitFile("foo.txt", "hello") - - // With a dir that doesn't exist - if err := g.Get(dst, repo.url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "foo.txt") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestGitGetter_branch(t *testing.T) { - if !testHasGit { - t.Skip("git not found, skipping") - } - - g := new(gitGetter) - dst := tempDir(t) - - repo := testGitRepo(t, "branch") - repo.git("checkout", "-b", "test-branch") - repo.commitFile("branch.txt", "branch") - - q := repo.url.Query() - q.Add("ref", "test-branch") - repo.url.RawQuery = q.Encode() - - if err := g.Get(dst, repo.url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "branch.txt") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } - - // Get again should work - if err := g.Get(dst, repo.url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath = filepath.Join(dst, "branch.txt") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestGitGetter_commitID(t *testing.T) { - if !testHasGit { - t.Skip("git not found, skipping") - } - - g := new(gitGetter) - dst := tempDir(t) - - // We're going to create different content on the main branch vs. - // another branch here, so that below we can recognize if we - // correctly cloned the commit actually requested (from the - // "other branch"), not the one at HEAD. - repo := testGitRepo(t, "commit_id") - repo.git("checkout", "-b", "main-branch") - repo.commitFile("wrong.txt", "Nope") - repo.git("checkout", "-b", "other-branch") - repo.commitFile("hello.txt", "Yep") - commitID, err := repo.latestCommit() - if err != nil { - t.Fatal(err) - } - // Return to the main branch so that HEAD of this repository - // will be that, rather than "test-branch". - repo.git("checkout", "main-branch") - - q := repo.url.Query() - q.Add("ref", commitID) - repo.url.RawQuery = q.Encode() - - t.Logf("Getting %s", repo.url) - if err := g.Get(dst, repo.url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "hello.txt") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } - - // Get again should work - if err := g.Get(dst, repo.url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath = filepath.Join(dst, "hello.txt") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestGitGetter_remoteWithoutMaster(t *testing.T) { - if !testHasGit { - t.Log("git not found, skipping") - t.Skip() - } - - g := new(gitGetter) - dst := tempDir(t) - - repo := testGitRepo(t, "branch") - repo.git("checkout", "-b", "test-branch") - repo.commitFile("branch.txt", "branch") - - q := repo.url.Query() - repo.url.RawQuery = q.Encode() - - if err := g.Get(dst, repo.url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "branch.txt") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } - - // Get again should work - if err := g.Get(dst, repo.url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath = filepath.Join(dst, "branch.txt") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestGitGetter_shallowClone(t *testing.T) { - if !testHasGit { - t.Log("git not found, skipping") - t.Skip() - } - - g := new(gitGetter) - dst := tempDir(t) - - repo := testGitRepo(t, "upstream") - repo.commitFile("upstream.txt", "0") - repo.commitFile("upstream.txt", "1") - - // Specifiy a clone depth of 1 - q := repo.url.Query() - q.Add("depth", "1") - repo.url.RawQuery = q.Encode() - - if err := g.Get(dst, repo.url); err != nil { - t.Fatalf("err: %s", err) - } - - // Assert rev-list count is '1' - cmd := exec.Command("git", "rev-list", "HEAD", "--count") - cmd.Dir = dst - b, err := cmd.Output() - if err != nil { - t.Fatalf("err: %s", err) - } - - out := strings.TrimSpace(string(b)) - if out != "1" { - t.Fatalf("expected rev-list count to be '1' but got %v", out) - } -} - -func TestGitGetter_shallowCloneWithTag(t *testing.T) { - if !testHasGit { - t.Log("git not found, skipping") - t.Skip() - } - - g := new(gitGetter) - dst := tempDir(t) - - repo := testGitRepo(t, "upstream") - repo.commitFile("v1.0.txt", "0") - repo.git("tag", "v1.0") - repo.commitFile("v1.1.txt", "1") - - // Specifiy a clone depth of 1 with a tag - q := repo.url.Query() - q.Add("ref", "v1.0") - q.Add("depth", "1") - repo.url.RawQuery = q.Encode() - - if err := g.Get(dst, repo.url); err != nil { - t.Fatalf("err: %s", err) - } - - // Assert rev-list count is '1' - cmd := exec.Command("git", "rev-list", "HEAD", "--count") - cmd.Dir = dst - b, err := cmd.Output() - if err != nil { - t.Fatalf("err: %s", err) - } - - out := strings.TrimSpace(string(b)) - if out != "1" { - t.Fatalf("expected rev-list count to be '1' but got %v", out) - } - - // Verify the v1.0 file exists - mainPath := filepath.Join(dst, "v1.0.txt") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the v1.1 file does not exists - mainPath = filepath.Join(dst, "v1.1.txt") - if _, err := os.Stat(mainPath); err == nil { - t.Fatalf("expected v1.1 file to not exist") - } -} - -func TestGitGetter_shallowCloneWithCommitID(t *testing.T) { - if !testHasGit { - t.Log("git not found, skipping") - t.Skip() - } - - g := new(gitGetter) - dst := tempDir(t) - - repo := testGitRepo(t, "upstream") - repo.commitFile("v1.0.txt", "0") - repo.git("tag", "v1.0") - repo.commitFile("v1.1.txt", "1") - - commitID, err := repo.latestCommit() - if err != nil { - t.Fatal(err) - } - - // Specify a clone depth of 1 with a naked commit ID - // This is intentionally invalid: shallow clone always requires a named ref. - q := repo.url.Query() - q.Add("ref", commitID[:8]) - q.Add("depth", "1") - repo.url.RawQuery = q.Encode() - - t.Logf("Getting %s", repo.url) - err = g.Get(dst, repo.url) - if err == nil { - t.Fatalf("success; want error") - } - // We use a heuristic to generate an extra hint in the error message if - // it looks like the user was trying to combine ref=COMMIT with depth. - if got, want := err.Error(), "(note that setting 'depth' requires 'ref' to be a branch or tag name)"; !strings.Contains(got, want) { - t.Errorf("missing error message hint\ngot: %s\nwant substring: %s", got, want) - } -} - -func TestGitGetter_branchUpdate(t *testing.T) { - if !testHasGit { - t.Skip("git not found, skipping") - } - - g := new(gitGetter) - dst := tempDir(t) - - // First setup the state with a fresh branch - repo := testGitRepo(t, "branch-update") - repo.git("checkout", "-b", "test-branch") - repo.commitFile("branch.txt", "branch") - - // Get the "test-branch" branch - q := repo.url.Query() - q.Add("ref", "test-branch") - repo.url.RawQuery = q.Encode() - if err := g.Get(dst, repo.url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "branch.txt") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } - - // Commit an update to the branch - repo.commitFile("branch-update.txt", "branch-update") - - // Get again should work - if err := g.Get(dst, repo.url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath = filepath.Join(dst, "branch-update.txt") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestGitGetter_tag(t *testing.T) { - if !testHasGit { - t.Skip("git not found, skipping") - } - - g := new(gitGetter) - dst := tempDir(t) - - repo := testGitRepo(t, "tag") - repo.commitFile("tag.txt", "tag") - repo.git("tag", "v1.0") - - q := repo.url.Query() - q.Add("ref", "v1.0") - repo.url.RawQuery = q.Encode() - - if err := g.Get(dst, repo.url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "tag.txt") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } - - // Get again should work - if err := g.Get(dst, repo.url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath = filepath.Join(dst, "tag.txt") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestGitGetter_GetFile(t *testing.T) { - if !testHasGit { - t.Skip("git not found, skipping") - } - - g := new(gitGetter) - dst := tempTestFile(t) - defer os.RemoveAll(filepath.Dir(dst)) - - repo := testGitRepo(t, "file") - repo.commitFile("file.txt", "hello") - - // Download the file - repo.url.Path = filepath.Join(repo.url.Path, "file.txt") - if err := g.GetFile(dst, repo.url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - if _, err := os.Stat(dst); err != nil { - t.Fatalf("err: %s", err) - } - assertContents(t, dst, "hello") -} - -func TestGitGetter_gitVersion(t *testing.T) { - if !testHasGit { - t.Skip("git not found, skipping") - } - if runtime.GOOS == "windows" { - t.Skip("skipping on windows since the test requires sh") - } - dir, err := ioutil.TempDir("", "go-getter") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - script := filepath.Join(dir, "git") - err = ioutil.WriteFile( - script, - []byte("#!/bin/sh\necho \"git version 2.0 (Some Metadata Here)\n\""), - 0700) - if err != nil { - t.Fatal(err) - } - - defer func(v string) { - os.Setenv("PATH", v) - }(os.Getenv("PATH")) - - os.Setenv("PATH", dir) - - // Asking for a higher version throws an error - if err := checkGitVersion("2.3"); err == nil { - t.Fatal("expect git version error") - } - - // Passes when version is satisfied - if err := checkGitVersion("1.9"); err != nil { - t.Fatal(err) - } -} - -func TestGitGetter_sshKey(t *testing.T) { - if !testHasGit { - t.Skip("git not found, skipping") - } - - g := new(gitGetter) - dst := tempDir(t) - - encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken)) - - // avoid getting locked by a github authenticity validation prompt - os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes") - defer os.Setenv("GIT_SSH_COMMAND", "") - - u, err := urlhelper.Parse("ssh://git@github.com/hashicorp/test-private-repo" + - "?sshkey=" + encodedKey) - if err != nil { - t.Fatal(err) - } - - if err := g.Get(dst, u); err != nil { - t.Fatalf("err: %s", err) - } - - readmePath := filepath.Join(dst, "README.md") - if _, err := os.Stat(readmePath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestGitGetter_sshSCPStyle(t *testing.T) { - if !testHasGit { - t.Skip("git not found, skipping") - } - - g := new(gitGetter) - dst := tempDir(t) - - encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken)) - - // avoid getting locked by a github authenticity validation prompt - os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes") - defer os.Setenv("GIT_SSH_COMMAND", "") - - // This test exercises the combination of the git detector and the - // git getter, to make sure that together they make scp-style URLs work. - client := &getter.Client{ - Src: "git@github.com:hashicorp/test-private-repo?sshkey=" + encodedKey, - Dst: dst, - Pwd: ".", - - Mode: getter.ClientModeDir, - - Detectors: []getter.Detector{ - new(getter.GitDetector), - }, - Getters: map[string]getter.Getter{ - "git": g, - }, - } - - if err := client.Get(); err != nil { - t.Fatalf("client.Get failed: %s", err) - } - - readmePath := filepath.Join(dst, "README.md") - if _, err := os.Stat(readmePath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestGitGetter_sshExplicitPort(t *testing.T) { - if !testHasGit { - t.Skip("git not found, skipping") - } - - g := new(gitGetter) - dst := tempDir(t) - - encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken)) - - // avoid getting locked by a github authenticity validation prompt - os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes") - defer os.Setenv("GIT_SSH_COMMAND", "") - - // This test exercises the combination of the git detector and the - // git getter, to make sure that together they make scp-style URLs work. - client := &getter.Client{ - Src: "git::ssh://git@github.com:22/hashicorp/test-private-repo?sshkey=" + encodedKey, - Dst: dst, - Pwd: ".", - - Mode: getter.ClientModeDir, - - Detectors: []getter.Detector{ - new(getter.GitDetector), - }, - Getters: map[string]getter.Getter{ - "git": g, - }, - } - - if err := client.Get(); err != nil { - t.Fatalf("client.Get failed: %s", err) - } - - readmePath := filepath.Join(dst, "README.md") - if _, err := os.Stat(readmePath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestGitGetter_sshSCPStyleInvalidScheme(t *testing.T) { - if !testHasGit { - t.Skip("git not found, skipping") - } - - g := new(gitGetter) - dst := tempDir(t) - - encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken)) - - // avoid getting locked by a github authenticity validation prompt - os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes") - defer os.Setenv("GIT_SSH_COMMAND", "") - - // This test exercises the combination of the git detector and the - // git getter, to make sure that together they make scp-style URLs work. - client := &getter.Client{ - Src: "git::ssh://git@github.com:hashicorp/test-private-repo?sshkey=" + encodedKey, - Dst: dst, - Pwd: ".", - - Mode: getter.ClientModeDir, - - Detectors: []getter.Detector{ - new(getter.GitDetector), - }, - Getters: map[string]getter.Getter{ - "git": g, - }, - } - - err := client.Get() - if err == nil { - t.Fatalf("get succeeded; want error") - } - - got := err.Error() - want1, want2 := `invalid source string`, `invalid port number "hashicorp"` - if !(strings.Contains(got, want1) || strings.Contains(got, want2)) { - t.Fatalf("wrong error\ngot: %s\nwant: %q or %q", got, want1, want2) - } -} - -func TestGitGetter_submodule(t *testing.T) { - if !testHasGit { - t.Skip("git not found, skipping") - } - - g := new(gitGetter) - dst := tempDir(t) - - relpath := func(basepath, targpath string) string { - relpath, err := filepath.Rel(basepath, targpath) - if err != nil { - t.Fatal(err) - } - return strings.Replace(relpath, `\`, `/`, -1) - // on windows git still prefers relatives paths - // containing `/` for submodules - } - - // Set up the grandchild - gc := testGitRepo(t, "grandchild") - gc.commitFile("grandchild.txt", "grandchild") - - // Set up the child - c := testGitRepo(t, "child") - c.commitFile("child.txt", "child") - c.git("submodule", "add", "-f", relpath(c.dir, gc.dir)) - c.git("commit", "-m", "Add grandchild submodule") - - // Set up the parent - p := testGitRepo(t, "parent") - p.commitFile("parent.txt", "parent") - p.git("submodule", "add", "-f", relpath(p.dir, c.dir)) - p.git("commit", "-m", "Add child submodule") - - // Clone the root repository - if err := g.Get(dst, p.url); err != nil { - t.Fatalf("err: %s", err) - } - - // Check that the files exist - for _, path := range []string{ - filepath.Join(dst, "parent.txt"), - filepath.Join(dst, "child", "child.txt"), - filepath.Join(dst, "child", "grandchild", "grandchild.txt"), - } { - if _, err := os.Stat(path); err != nil { - t.Fatalf("err: %s", err) - } - } -} - -func TestGitGetter_setupGitEnv_sshKey(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("skipping on windows since the test requires sh") - } - - cmd := exec.Command("/bin/sh", "-c", "echo $GIT_SSH_COMMAND") - setupGitEnv(cmd, "/tmp/foo.pem") - out, err := cmd.Output() - if err != nil { - t.Fatal(err) - } - - actual := strings.TrimSpace(string(out)) - if actual != "ssh -i /tmp/foo.pem" { - t.Fatalf("unexpected GIT_SSH_COMMAND: %q", actual) - } -} - -func TestGitGetter_setupGitEnvWithExisting_sshKey(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skipf("skipping on windows since the test requires sh") - return - } - - // start with an existing ssh command configuration - os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes") - defer os.Setenv("GIT_SSH_COMMAND", "") - - cmd := exec.Command("/bin/sh", "-c", "echo $GIT_SSH_COMMAND") - setupGitEnv(cmd, "/tmp/foo.pem") - out, err := cmd.Output() - if err != nil { - t.Fatal(err) - } - - actual := strings.TrimSpace(string(out)) - if actual != "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i /tmp/foo.pem" { - t.Fatalf("unexpected GIT_SSH_COMMAND: %q", actual) - } -} - -// gitRepo is a helper struct which controls a single temp git repo. -type gitRepo struct { - t *testing.T - url *url.URL - dir string -} - -// testGitRepo creates a new test git repository. -func testGitRepo(t *testing.T, name string) *gitRepo { - t.Helper() - dir, err := ioutil.TempDir("", "go-getter") - if err != nil { - t.Fatal(err) - } - dir = filepath.Join(dir, name) - if err := os.Mkdir(dir, 0700); err != nil { - t.Fatal(err) - } - - r := &gitRepo{ - t: t, - dir: dir, - } - - url, err := urlhelper.Parse("file://" + r.dir) - if err != nil { - t.Fatal(err) - } - r.url = url - - t.Logf("initializing git repo in %s", dir) - r.git("init") - r.git("config", "user.name", "go-getter") - r.git("config", "user.email", "go-getter@hashicorp.com") - - return r -} - -// git runs a git command against the repo. -func (r *gitRepo) git(args ...string) { - cmd := exec.Command("git", args...) - cmd.Dir = r.dir - bfr := bytes.NewBuffer(nil) - cmd.Stderr = bfr - if err := cmd.Run(); err != nil { - r.t.Fatal(err, bfr.String()) - } -} - -// commitFile writes and commits a text file to the repo. -func (r *gitRepo) commitFile(file, content string) { - path := filepath.Join(r.dir, file) - if err := ioutil.WriteFile(path, []byte(content), 0600); err != nil { - r.t.Fatal(err) - } - r.git("add", file) - r.git("commit", "-m", "Adding "+file) -} - -// latestCommit returns the full commit id of the latest commit on the current -// branch. -func (r *gitRepo) latestCommit() (string, error) { - cmd := exec.Command("git", "rev-parse", "HEAD") - cmd.Dir = r.dir - rawOut, err := cmd.Output() - if err != nil { - return "", err - } - rawOut = bytes.TrimSpace(rawOut) - return string(rawOut), nil -} - -// This is a read-only deploy key for an empty test repository. -// Note: This is split over multiple lines to avoid being disabled by key -// scanners automatically. -var testGitToken = `-----BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEA9cHsxCl3Jjgu9DHpwvmfFOl1XEdY+ShHDR/cMnzJ5ddk5/oV -Wy6EWatvyHZfRSZMwzv4PtKeUPm6iXjqWp4xdWU9khlPzozyj+U9Fq70TRVUW9E5 -T1XdQVwJE421yffr4VMMwu60wBqjI1epapH2i2inYvw9Zl9X2MXq0+jTvFvDerbT -mDtfStDPljenELAIZtWVETSvbI46gALwbxbM2292ZUIL4D6jRz0aZMmyy/twYv8r -9WGJLwmYzU518Ie7zqKW/mCTdTrV0WRiDj0MeRaPgrGY9amuHE4r9iG/cJkwpKAO -Ccz0Hs6i89u9vZnTqZU9V7weJqRAQcMjXXR6yQIDAQABAoIBAQDBzICKnGxiTlHw -rd+6qqChnAy5jWYDbZjCJ8q8YZ3RS08+g/8NXZxvHftTqM0uOaq1FviHig3gq15H -hHvCpBc6jXDFYoKFzq6FfO/0kFkE5HoWweIgxwRow0xBCDJAJ+ryUEyy+Ay/pQHb -IAjwilRS0V+WdnVw4mTjBAhPvb4jPOo97Yfy3PYUyx2F3newkqXOZy+zx3G/ANoa -ncypfMGyy76sfCWKqw4J1gVkVQLwbB6gQkXUFGYwY9sRrxbG93kQw76Flc/E/s52 -62j4v1IM0fq0t/St+Y/+s6Lkw` + `aqt3ft1nsqWcRaVDdqvMfkzgJGXlw0bGzJG5MEQ -AIBq3dHRAoGBAP8OeG/DKG2Z1VmSfzuz1pas1fbZ+F7venOBrjez3sKlb3Pyl2aH -mt2wjaTUi5v10VrHgYtOEdqyhQeUSYydWXIBKNMag0NLLrfFUKZK+57wrHWFdFjn -VgpsdkLSNTOZpC8gA5OaJ+36IcOPfGqyyP9wuuRoaYnVT1KEzqLa9FEFAoGBAPaq -pglwhil2rxjJE4zq0afQLNpAfi7Xqcrepij+xvJIcIj7nawxXuPxqRFxONE/h3yX -zkybO8wLdbHX9Iw/wc1j50Uf1Z5gHdLf7/hQJoWKpz1RnkWRy6CYON8v1tpVp0tb -OAajR/kZnzebq2mfa7pyy5zDCX++2kp/dcFwHf31AoGAE8oupBVTZLWj7TBFuP8q -LkS40U92Sv9v09iDCQVmylmFvUxcXPM2m+7f/qMTNgWrucxzC7kB/6MMWVszHbrz -vrnCTibnemgx9sZTjKOSxHFOIEw7i85fSa3Cu0qOIDPSnmlwfZpfcMKQrhjLAYhf -uhooFiLX1X78iZ2OXup4PHUCgYEAsmBrm83sp1V1gAYBBlnVbXakyNv0pCk/Vz61 -iFXeRt1NzDGxLxGw3kQnED8BaIh5kQcyn8Fud7sdzJMv/LAqlT4Ww60mzNYTGyjo -H3jOsqm3ESfRvduWFreeAQBWbiOczGjV1i8D4EbAFfWT+tjXjchwKBf+6Yt5zn/o -Bw/uEHUCgYAFs+JPOR25oRyBs7ujrMo/OY1z/eXTVVgZxY+tYGe1FJqDeFyR7ytK -+JBB1MuDwQKGm2wSIXdCzTNoIx2B9zTseiPTwT8G7vqNFhXoIaTBp4P2xIQb45mJ -7GkTsMBHwpSMOXgX9Weq3v5xOJ2WxVtjENmd6qzxcYCO5lP15O17hA== ------END RSA PRIVATE KEY-----` - -func assertContents(t *testing.T, path string, contents string) { - data, err := ioutil.ReadFile(path) - if err != nil { - t.Fatalf("err: %s", err) - } - - if !reflect.DeepEqual(data, []byte(contents)) { - t.Fatalf("bad. expected:\n\n%s\n\nGot:\n\n%s", contents, string(data)) - } -} - -func tempDir(t *testing.T) string { - dir, err := ioutil.TempDir("", "tf") - if err != nil { - t.Fatalf("err: %s", err) - } - if err := os.RemoveAll(dir); err != nil { - t.Fatalf("err: %s", err) - } - - return dir -} - -func tempTestFile(t *testing.T) string { - dir := tempDir(t) - return filepath.Join(dir, "foo") -} diff --git a/website/docs/language/modules/sources.mdx b/website/docs/language/modules/sources.mdx index 1af189b94..d4aa0d35b 100644 --- a/website/docs/language/modules/sources.mdx +++ b/website/docs/language/modules/sources.mdx @@ -239,20 +239,44 @@ only SSH key authentication is supported, and By default, Terraform will clone and use the default branch (referenced by `HEAD`) in the selected repository. You can override this using the `ref` argument. The value of the `ref` argument can be any reference that would be accepted -by the `git checkout` command, such as branch, SHA-1 hash (short or full), or tag names. The [Git documentation](https://git-scm.com/book/en/v2/Git-Tools-Revision-Selection#_single_revisions) contains a complete list. +by the `git checkout` command, such as branch, SHA-1 hash (short or full), or tag names. +For a full list of the possible values, see +[Git Tools - Revision Selection](https://git-scm.com/book/en/v2/Git-Tools-Revision-Selection#_single_revisions) +in [the Git Book](https://git-scm.com/book/en/v2). ```hcl -# referencing a specific release +# select a specific tag module "vpc" { source = "git::https://example.com/vpc.git?ref=v1.2.0" } -# referencing a specific commit SHA-1 hash +# directly select a commit using its SHA-1 hash module "storage" { source = "git::https://example.com/storage.git?ref=51d462976d84fdea54b47d80dcabbf680badcdb8" } ``` +### Shallow Clone + +For larger repositories you may prefer to make only a shallow clone in order +to reduce the time taken to retrieve the remote repository. + +The `depth` URL argument corresponds to +[the `--depth` argument to `git clone`](https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---depthltdepthgt), +telling Git to create a shallow clone with the history truncated to only +the specified number of commits. + +However, because shallow clone requires different Git protocol behavior, +setting the `depth` argument makes Terraform pass your [`ref` argument](#selecting-a-revision), +if any, to +[the `--branch` argument to `git clone`](https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---branchltnamegt) +instead. That means it must specify a named branch or tag known to the remote +repository, and that raw commit IDs are not acceptable. + +Because Terraform only uses the most recent selected commit to find the source +code of your specified module, it is not typically useful to set `depth` +to any value other than `1`. + ### "scp-like" address syntax When using Git over SSH, we recommend using the `ssh://`-prefixed URL form @@ -421,10 +445,12 @@ module "consul" { } ``` -The module installer uses Google Cloud SDK to authenticate with GCS. To set credentials you can: +The module installer uses Google Cloud SDK to authenticate with GCS. You can +use any of the following methods to set Google Cloud Platform credentials: -* Enter the path of your service account key file in the GOOGLE_APPLICATION_CREDENTIALS environment variable, or; -* If you're running Terraform from a GCE instance, default credentials are automatically available. See [Creating and Enabling Service Accounts](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances) for Instances for more details +* Set the `GOOGLE_OAUTH_ACCESS_TOKEN` environment variable to a raw Google Cloud Platform OAuth access token. +* Enter the path of your service account key file in the `GOOGLE_APPLICATION_CREDENTIALS` environment variable. +* If you're running Terraform from a GCE instance, default credentials are automatically available. See [Creating and Enabling Service Accounts](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances) for Instances for more details. * On your computer, you can make your Google identity available by running `gcloud auth application-default login`. ## Modules in Package Sub-directories