From e94fcc011d56436b73313b9c1e8cfd3f7d3e2d14 Mon Sep 17 00:00:00 2001 From: clint shryock Date: Thu, 26 Jan 2017 14:33:22 -0600 Subject: [PATCH] update go-getter --- .../github.com/hashicorp/go-getter/README.md | 6 + .../github.com/hashicorp/go-getter/client.go | 15 ++- vendor/github.com/hashicorp/go-getter/get.go | 4 + .../hashicorp/go-getter/get_file.go | 24 ++++ .../hashicorp/go-getter/get_file_unix.go | 2 +- .../github.com/hashicorp/go-getter/get_git.go | 123 ++++++++++++++++-- .../github.com/hashicorp/go-getter/get_hg.go | 4 + .../hashicorp/go-getter/get_http.go | 7 + .../hashicorp/go-getter/get_mock.go | 7 + .../github.com/hashicorp/go-getter/get_s3.go | 39 ++++++ vendor/vendor.json | 6 +- 11 files changed, 219 insertions(+), 18 deletions(-) diff --git a/vendor/github.com/hashicorp/go-getter/README.md b/vendor/github.com/hashicorp/go-getter/README.md index 30aed323f..4a0b6a625 100644 --- a/vendor/github.com/hashicorp/go-getter/README.md +++ b/vendor/github.com/hashicorp/go-getter/README.md @@ -210,6 +210,12 @@ None a commit SHA, a branch name, etc. If it is a named ref such as a branch name, go-getter will update it to the latest on each get. + * `sshkey` - An SSH private key to use during clones. The provided key must + be a base64-encoded string. For example, to generate a suitable `sshkey` + from a private key file on disk, you would run `base64 -w0 `. + + **Note**: Git 2.3+ is required to use this feature. + ### Mercurial (`hg`) * `rev` - The Mercurial revision to checkout. diff --git a/vendor/github.com/hashicorp/go-getter/client.go b/vendor/github.com/hashicorp/go-getter/client.go index 5a039dbac..8cfc038ce 100644 --- a/vendor/github.com/hashicorp/go-getter/client.go +++ b/vendor/github.com/hashicorp/go-getter/client.go @@ -222,13 +222,18 @@ func (c *Client) Get() error { checksumValue = b } - // For now, any means file. In the future, we'll ask the getter - // what it thinks it is. if mode == ClientModeAny { - mode = ClientModeFile + // Ask the getter which client mode to use + mode, err = g.ClientMode(u) + if err != nil { + return err + } - // Destination is the base name of the URL path - dst = filepath.Join(dst, filepath.Base(u.Path)) + // Destination is the base name of the URL path in "any" mode when + // a file source is detected. + if mode == ClientModeFile { + dst = filepath.Join(dst, filepath.Base(u.Path)) + } } // If we're not downloading a directory, then just download the file diff --git a/vendor/github.com/hashicorp/go-getter/get.go b/vendor/github.com/hashicorp/go-getter/get.go index cd9a411be..c3236f553 100644 --- a/vendor/github.com/hashicorp/go-getter/get.go +++ b/vendor/github.com/hashicorp/go-getter/get.go @@ -35,6 +35,10 @@ type Getter interface { // reference a single file. If possible, the Getter should check if // the remote end contains the same file and no-op this operation. GetFile(string, *url.URL) error + + // ClientMode returns the mode based on the given URL. This is used to + // allow clients to let the getters decide which mode to use. + ClientMode(*url.URL) (ClientMode, error) } // Getters is the mapping of scheme to the Getter implementation that will diff --git a/vendor/github.com/hashicorp/go-getter/get_file.go b/vendor/github.com/hashicorp/go-getter/get_file.go index 341cd0ed8..e5d2d61d7 100644 --- a/vendor/github.com/hashicorp/go-getter/get_file.go +++ b/vendor/github.com/hashicorp/go-getter/get_file.go @@ -1,8 +1,32 @@ package getter +import ( + "net/url" + "os" +) + // FileGetter is a Getter implementation that will download a module from // a file scheme. type FileGetter struct { // Copy, if set to true, will copy data instead of using a symlink Copy bool } + +func (g *FileGetter) ClientMode(u *url.URL) (ClientMode, error) { + path := u.Path + if u.RawPath != "" { + path = u.RawPath + } + + fi, err := os.Stat(path) + if err != nil { + return 0, err + } + + // Check if the source is a directory. + if fi.IsDir() { + return ClientModeDir, nil + } + + return ClientModeFile, nil +} diff --git a/vendor/github.com/hashicorp/go-getter/get_file_unix.go b/vendor/github.com/hashicorp/go-getter/get_file_unix.go index 693680ac2..c89a2d5a4 100644 --- a/vendor/github.com/hashicorp/go-getter/get_file_unix.go +++ b/vendor/github.com/hashicorp/go-getter/get_file_unix.go @@ -55,7 +55,7 @@ func (g *FileGetter) GetFile(dst string, u *url.URL) error { path = u.RawPath } - // The source path must exist and be a directory to be usable. + // The source path must exist and be a file to be usable. if fi, err := os.Stat(path); err != nil { return fmt.Errorf("source path error: %s", err) } else if fi.IsDir() { diff --git a/vendor/github.com/hashicorp/go-getter/get_git.go b/vendor/github.com/hashicorp/go-getter/get_git.go index 1bf0dc7e6..072813983 100644 --- a/vendor/github.com/hashicorp/go-getter/get_git.go +++ b/vendor/github.com/hashicorp/go-getter/get_git.go @@ -1,58 +1,105 @@ package getter import ( + "encoding/base64" "fmt" "io/ioutil" "net/url" "os" "os/exec" "path/filepath" + "strings" urlhelper "github.com/hashicorp/go-getter/helper/url" + "github.com/hashicorp/go-version" ) // GitGetter is a Getter implementation that will download a module from // a git repository. type GitGetter struct{} +func (g *GitGetter) ClientMode(_ *url.URL) (ClientMode, error) { + return ClientModeDir, nil +} + func (g *GitGetter) Get(dst string, u *url.URL) error { if _, err := exec.LookPath("git"); err != nil { return fmt.Errorf("git must be available and on the PATH") } // Extract some query parameters we use - var ref string + var ref, sshKey string q := u.Query() if len(q) > 0 { ref = q.Get("ref") q.Del("ref") + sshKey = q.Get("sshkey") + q.Del("sshkey") + // Copy the URL var newU url.URL = *u u = &newU u.RawQuery = q.Encode() } - // First: clone or update the repository + var sshKeyFile string + if sshKey != "" { + // Check that the git version is sufficiently new. + if err := checkGitVersion("2.3"); err != nil { + return fmt.Errorf("Error using ssh key: %v", err) + } + + // We have an SSH key - decode it. + raw, err := base64.StdEncoding.DecodeString(sshKey) + if err != nil { + return err + } + + // Create a temp file for the key and ensure it is removed. + fh, err := ioutil.TempFile("", "go-getter") + if err != nil { + return err + } + sshKeyFile = fh.Name() + defer os.Remove(sshKeyFile) + + // Set the permissions prior to writing the key material. + if err := os.Chmod(sshKeyFile, 0600); err != nil { + return err + } + + // Write the raw key into the temp file. + _, err = fh.Write(raw) + fh.Close() + if err != nil { + return err + } + } + + // Clone or update the repository _, err := os.Stat(dst) if err != nil && !os.IsNotExist(err) { return err } if err == nil { - err = g.update(dst, ref) + err = g.update(dst, sshKeyFile, ref) } else { - err = g.clone(dst, u) + err = g.clone(dst, sshKeyFile, u) } if err != nil { return err } // Next: check out the proper tag/branch if it is specified, and checkout - if ref == "" { - return nil + if ref != "" { + if err := g.checkout(dst, ref); err != nil { + return err + } } - return g.checkout(dst, ref) + // Lastly, download any/all submodules. + return g.fetchSubmodules(dst, sshKeyFile) } // GetFile for Git doesn't support updating at this time. It will download @@ -92,16 +139,18 @@ func (g *GitGetter) checkout(dst string, ref string) error { return getRunCommand(cmd) } -func (g *GitGetter) clone(dst string, u *url.URL) error { +func (g *GitGetter) clone(dst, sshKeyFile string, u *url.URL) error { cmd := exec.Command("git", "clone", u.String(), dst) + setupGitEnv(cmd, sshKeyFile) return getRunCommand(cmd) } -func (g *GitGetter) update(dst string, ref string) error { +func (g *GitGetter) update(dst, sshKeyFile, ref string) error { // Determine if we're a branch. If we're NOT a branch, then we just // switch to master prior to checking out cmd := exec.Command("git", "show-ref", "-q", "--verify", "refs/heads/"+ref) cmd.Dir = dst + if getRunCommand(cmd) != nil { // Not a branch, switch to master. This will also catch non-existent // branches, in which case we want to switch to master and then @@ -116,5 +165,61 @@ func (g *GitGetter) update(dst string, ref string) error { cmd = exec.Command("git", "pull", "--ff-only") cmd.Dir = dst + setupGitEnv(cmd, sshKeyFile) return getRunCommand(cmd) } + +// fetchSubmodules downloads any configured submodules recursively. +func (g *GitGetter) fetchSubmodules(dst, sshKeyFile string) error { + cmd := exec.Command("git", "submodule", "update", "--init", "--recursive") + cmd.Dir = dst + setupGitEnv(cmd, sshKeyFile) + return getRunCommand(cmd) +} + +// setupGitEnv sets up the environment for the given command. This is used to +// pass configuration data to git and ssh and enables advanced cloning methods. +func setupGitEnv(cmd *exec.Cmd, sshKeyFile string) { + var sshOpts []string + + if sshKeyFile != "" { + // We have an SSH key temp file configured, tell ssh about this. + sshOpts = append(sshOpts, "-i", sshKeyFile) + } + + cmd.Env = append(os.Environ(), + // Set the ssh command to use for clones. + "GIT_SSH_COMMAND=ssh "+strings.Join(sshOpts, " "), + ) +} + +// checkGitVersion is used to check the version of git installed on the system +// against a known minimum version. Returns an error if the installed version +// is older than the given minimum. +func checkGitVersion(min string) error { + want, err := version.NewVersion(min) + if err != nil { + return err + } + + out, err := exec.Command("git", "version").Output() + if err != nil { + return err + } + + fields := strings.Fields(string(out)) + if len(fields) != 3 { + return fmt.Errorf("Unexpected 'git version' output: %q", string(out)) + } + + have, err := version.NewVersion(fields[2]) + if err != nil { + return err + } + + if have.LessThan(want) { + return fmt.Errorf("Required git version = %s, have %s", want, have) + } + + return nil +} diff --git a/vendor/github.com/hashicorp/go-getter/get_hg.go b/vendor/github.com/hashicorp/go-getter/get_hg.go index 542bef1f9..820bdd488 100644 --- a/vendor/github.com/hashicorp/go-getter/get_hg.go +++ b/vendor/github.com/hashicorp/go-getter/get_hg.go @@ -16,6 +16,10 @@ import ( // a Mercurial repository. type HgGetter struct{} +func (g *HgGetter) ClientMode(_ *url.URL) (ClientMode, error) { + return ClientModeDir, nil +} + func (g *HgGetter) Get(dst string, u *url.URL) error { if _, err := exec.LookPath("hg"); err != nil { return fmt.Errorf("hg must be available and on the PATH") diff --git a/vendor/github.com/hashicorp/go-getter/get_http.go b/vendor/github.com/hashicorp/go-getter/get_http.go index d64b23835..3c020343e 100644 --- a/vendor/github.com/hashicorp/go-getter/get_http.go +++ b/vendor/github.com/hashicorp/go-getter/get_http.go @@ -38,6 +38,13 @@ type HttpGetter struct { Netrc bool } +func (g *HttpGetter) ClientMode(u *url.URL) (ClientMode, error) { + if strings.HasSuffix(u.Path, "/") { + return ClientModeDir, nil + } + return ClientModeFile, nil +} + func (g *HttpGetter) Get(dst string, u *url.URL) error { // Copy the URL so we can modify it var newU url.URL = *u diff --git a/vendor/github.com/hashicorp/go-getter/get_mock.go b/vendor/github.com/hashicorp/go-getter/get_mock.go index a7d3d3052..882e694dc 100644 --- a/vendor/github.com/hashicorp/go-getter/get_mock.go +++ b/vendor/github.com/hashicorp/go-getter/get_mock.go @@ -43,3 +43,10 @@ func (g *MockGetter) GetFile(dst string, u *url.URL) error { } return g.GetFileErr } + +func (g *MockGetter) ClientMode(u *url.URL) (ClientMode, error) { + if l := len(u.Path); l > 0 && u.Path[l-1:] == "/" { + return ClientModeDir, nil + } + return ClientModeFile, nil +} diff --git a/vendor/github.com/hashicorp/go-getter/get_s3.go b/vendor/github.com/hashicorp/go-getter/get_s3.go index bcfcbfc90..d3bffeb17 100644 --- a/vendor/github.com/hashicorp/go-getter/get_s3.go +++ b/vendor/github.com/hashicorp/go-getter/get_s3.go @@ -20,6 +20,45 @@ import ( // a S3 bucket. type S3Getter struct{} +func (g *S3Getter) ClientMode(u *url.URL) (ClientMode, error) { + // Parse URL + region, bucket, path, _, creds, err := g.parseUrl(u) + if err != nil { + return 0, err + } + + // Create client config + config := g.getAWSConfig(region, creds) + sess := session.New(config) + client := s3.New(sess) + + // List the object(s) at the given prefix + req := &s3.ListObjectsInput{ + Bucket: aws.String(bucket), + Prefix: aws.String(path), + } + resp, err := client.ListObjects(req) + if err != nil { + return 0, err + } + + for _, o := range resp.Contents { + // Use file mode on exact match. + if *o.Key == path { + return ClientModeFile, nil + } + + // Use dir mode if child keys are found. + if strings.HasPrefix(*o.Key, path+"/") { + return ClientModeDir, nil + } + } + + // There was no match, so just return file mode. The download is going + // to fail but we will let S3 return the proper error later. + return ClientModeFile, nil +} + func (g *S3Getter) Get(dst string, u *url.URL) error { // Parse URL region, bucket, path, _, creds, err := g.parseUrl(u) diff --git a/vendor/vendor.json b/vendor/vendor.json index de1323840..94e7feaa8 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -1619,10 +1619,10 @@ "revision": "875fb671b3ddc66f8e2f0acc33829c8cb989a38d" }, { - "checksumSHA1": "aOWXSbYAdK3PBSMNFiK2ze4lPEc=", + "checksumSHA1": "N+MSF+0nq3XFXaUCOooF9acE+ZI=", "path": "github.com/hashicorp/go-getter", - "revision": "2fbd997432e72fe36060c8f07ec1eaf98d098177", - "revisionTime": "2016-09-12T21:42:52Z" + "revision": "cc80f38c726badeae53775d179755e1c4953d6cf", + "revisionTime": "2017-01-26T00:13:12Z" }, { "checksumSHA1": "9J+kDr29yDrwsdu2ULzewmqGjpA=",