Merge pull request #11388 from hashicorp/f-go-getter

dependency: update go-getter
This commit is contained in:
Clint 2017-01-26 14:34:31 -06:00 committed by GitHub
commit 731696b002
11 changed files with 219 additions and 18 deletions

View File

@ -210,6 +210,12 @@ None
a commit SHA, a branch name, etc. If it is a named ref such as a branch a commit SHA, a branch name, etc. If it is a named ref such as a branch
name, go-getter will update it to the latest on each get. name, go-getter will update it to the latest on each get.
* `sshkey` - An SSH private key to use during clones. The provided key must
be a base64-encoded string. For example, to generate a suitable `sshkey`
from a private key file on disk, you would run `base64 -w0 <file>`.
**Note**: Git 2.3+ is required to use this feature.
### Mercurial (`hg`) ### Mercurial (`hg`)
* `rev` - The Mercurial revision to checkout. * `rev` - The Mercurial revision to checkout.

View File

@ -222,13 +222,18 @@ func (c *Client) Get() error {
checksumValue = b checksumValue = b
} }
// For now, any means file. In the future, we'll ask the getter
// what it thinks it is.
if mode == ClientModeAny { if mode == ClientModeAny {
mode = ClientModeFile // Ask the getter which client mode to use
mode, err = g.ClientMode(u)
if err != nil {
return err
}
// Destination is the base name of the URL path // Destination is the base name of the URL path in "any" mode when
dst = filepath.Join(dst, filepath.Base(u.Path)) // a file source is detected.
if mode == ClientModeFile {
dst = filepath.Join(dst, filepath.Base(u.Path))
}
} }
// If we're not downloading a directory, then just download the file // If we're not downloading a directory, then just download the file

View File

@ -35,6 +35,10 @@ type Getter interface {
// reference a single file. If possible, the Getter should check if // reference a single file. If possible, the Getter should check if
// the remote end contains the same file and no-op this operation. // the remote end contains the same file and no-op this operation.
GetFile(string, *url.URL) error GetFile(string, *url.URL) error
// ClientMode returns the mode based on the given URL. This is used to
// allow clients to let the getters decide which mode to use.
ClientMode(*url.URL) (ClientMode, error)
} }
// Getters is the mapping of scheme to the Getter implementation that will // Getters is the mapping of scheme to the Getter implementation that will

View File

@ -1,8 +1,32 @@
package getter package getter
import (
"net/url"
"os"
)
// FileGetter is a Getter implementation that will download a module from // FileGetter is a Getter implementation that will download a module from
// a file scheme. // a file scheme.
type FileGetter struct { type FileGetter struct {
// Copy, if set to true, will copy data instead of using a symlink // Copy, if set to true, will copy data instead of using a symlink
Copy bool Copy bool
} }
func (g *FileGetter) ClientMode(u *url.URL) (ClientMode, error) {
path := u.Path
if u.RawPath != "" {
path = u.RawPath
}
fi, err := os.Stat(path)
if err != nil {
return 0, err
}
// Check if the source is a directory.
if fi.IsDir() {
return ClientModeDir, nil
}
return ClientModeFile, nil
}

View File

@ -55,7 +55,7 @@ func (g *FileGetter) GetFile(dst string, u *url.URL) error {
path = u.RawPath path = u.RawPath
} }
// The source path must exist and be a directory to be usable. // The source path must exist and be a file to be usable.
if fi, err := os.Stat(path); err != nil { if fi, err := os.Stat(path); err != nil {
return fmt.Errorf("source path error: %s", err) return fmt.Errorf("source path error: %s", err)
} else if fi.IsDir() { } else if fi.IsDir() {

View File

@ -1,58 +1,105 @@
package getter package getter
import ( import (
"encoding/base64"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"net/url" "net/url"
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"strings"
urlhelper "github.com/hashicorp/go-getter/helper/url" urlhelper "github.com/hashicorp/go-getter/helper/url"
"github.com/hashicorp/go-version"
) )
// GitGetter is a Getter implementation that will download a module from // GitGetter is a Getter implementation that will download a module from
// a git repository. // a git repository.
type GitGetter struct{} type GitGetter struct{}
func (g *GitGetter) ClientMode(_ *url.URL) (ClientMode, error) {
return ClientModeDir, nil
}
func (g *GitGetter) Get(dst string, u *url.URL) error { func (g *GitGetter) Get(dst string, u *url.URL) error {
if _, err := exec.LookPath("git"); err != nil { if _, err := exec.LookPath("git"); err != nil {
return fmt.Errorf("git must be available and on the PATH") return fmt.Errorf("git must be available and on the PATH")
} }
// Extract some query parameters we use // Extract some query parameters we use
var ref string var ref, sshKey string
q := u.Query() q := u.Query()
if len(q) > 0 { if len(q) > 0 {
ref = q.Get("ref") ref = q.Get("ref")
q.Del("ref") q.Del("ref")
sshKey = q.Get("sshkey")
q.Del("sshkey")
// Copy the URL // Copy the URL
var newU url.URL = *u var newU url.URL = *u
u = &newU u = &newU
u.RawQuery = q.Encode() u.RawQuery = q.Encode()
} }
// First: clone or update the repository var sshKeyFile string
if sshKey != "" {
// Check that the git version is sufficiently new.
if err := checkGitVersion("2.3"); err != nil {
return fmt.Errorf("Error using ssh key: %v", err)
}
// We have an SSH key - decode it.
raw, err := base64.StdEncoding.DecodeString(sshKey)
if err != nil {
return err
}
// Create a temp file for the key and ensure it is removed.
fh, err := ioutil.TempFile("", "go-getter")
if err != nil {
return err
}
sshKeyFile = fh.Name()
defer os.Remove(sshKeyFile)
// Set the permissions prior to writing the key material.
if err := os.Chmod(sshKeyFile, 0600); err != nil {
return err
}
// Write the raw key into the temp file.
_, err = fh.Write(raw)
fh.Close()
if err != nil {
return err
}
}
// Clone or update the repository
_, err := os.Stat(dst) _, err := os.Stat(dst)
if err != nil && !os.IsNotExist(err) { if err != nil && !os.IsNotExist(err) {
return err return err
} }
if err == nil { if err == nil {
err = g.update(dst, ref) err = g.update(dst, sshKeyFile, ref)
} else { } else {
err = g.clone(dst, u) err = g.clone(dst, sshKeyFile, u)
} }
if err != nil { if err != nil {
return err return err
} }
// Next: check out the proper tag/branch if it is specified, and checkout // Next: check out the proper tag/branch if it is specified, and checkout
if ref == "" { if ref != "" {
return nil if err := g.checkout(dst, ref); err != nil {
return err
}
} }
return g.checkout(dst, ref) // Lastly, download any/all submodules.
return g.fetchSubmodules(dst, sshKeyFile)
} }
// GetFile for Git doesn't support updating at this time. It will download // GetFile for Git doesn't support updating at this time. It will download
@ -92,16 +139,18 @@ func (g *GitGetter) checkout(dst string, ref string) error {
return getRunCommand(cmd) return getRunCommand(cmd)
} }
func (g *GitGetter) clone(dst string, u *url.URL) error { func (g *GitGetter) clone(dst, sshKeyFile string, u *url.URL) error {
cmd := exec.Command("git", "clone", u.String(), dst) cmd := exec.Command("git", "clone", u.String(), dst)
setupGitEnv(cmd, sshKeyFile)
return getRunCommand(cmd) return getRunCommand(cmd)
} }
func (g *GitGetter) update(dst string, ref string) error { func (g *GitGetter) update(dst, sshKeyFile, ref string) error {
// Determine if we're a branch. If we're NOT a branch, then we just // Determine if we're a branch. If we're NOT a branch, then we just
// switch to master prior to checking out // switch to master prior to checking out
cmd := exec.Command("git", "show-ref", "-q", "--verify", "refs/heads/"+ref) cmd := exec.Command("git", "show-ref", "-q", "--verify", "refs/heads/"+ref)
cmd.Dir = dst cmd.Dir = dst
if getRunCommand(cmd) != nil { if getRunCommand(cmd) != nil {
// Not a branch, switch to master. This will also catch non-existent // Not a branch, switch to master. This will also catch non-existent
// branches, in which case we want to switch to master and then // branches, in which case we want to switch to master and then
@ -116,5 +165,61 @@ func (g *GitGetter) update(dst string, ref string) error {
cmd = exec.Command("git", "pull", "--ff-only") cmd = exec.Command("git", "pull", "--ff-only")
cmd.Dir = dst cmd.Dir = dst
setupGitEnv(cmd, sshKeyFile)
return getRunCommand(cmd) return getRunCommand(cmd)
} }
// fetchSubmodules downloads any configured submodules recursively.
func (g *GitGetter) fetchSubmodules(dst, sshKeyFile string) error {
cmd := exec.Command("git", "submodule", "update", "--init", "--recursive")
cmd.Dir = dst
setupGitEnv(cmd, sshKeyFile)
return getRunCommand(cmd)
}
// setupGitEnv sets up the environment for the given command. This is used to
// pass configuration data to git and ssh and enables advanced cloning methods.
func setupGitEnv(cmd *exec.Cmd, sshKeyFile string) {
var sshOpts []string
if sshKeyFile != "" {
// We have an SSH key temp file configured, tell ssh about this.
sshOpts = append(sshOpts, "-i", sshKeyFile)
}
cmd.Env = append(os.Environ(),
// Set the ssh command to use for clones.
"GIT_SSH_COMMAND=ssh "+strings.Join(sshOpts, " "),
)
}
// checkGitVersion is used to check the version of git installed on the system
// against a known minimum version. Returns an error if the installed version
// is older than the given minimum.
func checkGitVersion(min string) error {
want, err := version.NewVersion(min)
if err != nil {
return err
}
out, err := exec.Command("git", "version").Output()
if err != nil {
return err
}
fields := strings.Fields(string(out))
if len(fields) != 3 {
return fmt.Errorf("Unexpected 'git version' output: %q", string(out))
}
have, err := version.NewVersion(fields[2])
if err != nil {
return err
}
if have.LessThan(want) {
return fmt.Errorf("Required git version = %s, have %s", want, have)
}
return nil
}

View File

@ -16,6 +16,10 @@ import (
// a Mercurial repository. // a Mercurial repository.
type HgGetter struct{} type HgGetter struct{}
func (g *HgGetter) ClientMode(_ *url.URL) (ClientMode, error) {
return ClientModeDir, nil
}
func (g *HgGetter) Get(dst string, u *url.URL) error { func (g *HgGetter) Get(dst string, u *url.URL) error {
if _, err := exec.LookPath("hg"); err != nil { if _, err := exec.LookPath("hg"); err != nil {
return fmt.Errorf("hg must be available and on the PATH") return fmt.Errorf("hg must be available and on the PATH")

View File

@ -38,6 +38,13 @@ type HttpGetter struct {
Netrc bool Netrc bool
} }
func (g *HttpGetter) ClientMode(u *url.URL) (ClientMode, error) {
if strings.HasSuffix(u.Path, "/") {
return ClientModeDir, nil
}
return ClientModeFile, nil
}
func (g *HttpGetter) Get(dst string, u *url.URL) error { func (g *HttpGetter) Get(dst string, u *url.URL) error {
// Copy the URL so we can modify it // Copy the URL so we can modify it
var newU url.URL = *u var newU url.URL = *u

View File

@ -43,3 +43,10 @@ func (g *MockGetter) GetFile(dst string, u *url.URL) error {
} }
return g.GetFileErr return g.GetFileErr
} }
func (g *MockGetter) ClientMode(u *url.URL) (ClientMode, error) {
if l := len(u.Path); l > 0 && u.Path[l-1:] == "/" {
return ClientModeDir, nil
}
return ClientModeFile, nil
}

View File

@ -20,6 +20,45 @@ import (
// a S3 bucket. // a S3 bucket.
type S3Getter struct{} type S3Getter struct{}
func (g *S3Getter) ClientMode(u *url.URL) (ClientMode, error) {
// Parse URL
region, bucket, path, _, creds, err := g.parseUrl(u)
if err != nil {
return 0, err
}
// Create client config
config := g.getAWSConfig(region, creds)
sess := session.New(config)
client := s3.New(sess)
// List the object(s) at the given prefix
req := &s3.ListObjectsInput{
Bucket: aws.String(bucket),
Prefix: aws.String(path),
}
resp, err := client.ListObjects(req)
if err != nil {
return 0, err
}
for _, o := range resp.Contents {
// Use file mode on exact match.
if *o.Key == path {
return ClientModeFile, nil
}
// Use dir mode if child keys are found.
if strings.HasPrefix(*o.Key, path+"/") {
return ClientModeDir, nil
}
}
// There was no match, so just return file mode. The download is going
// to fail but we will let S3 return the proper error later.
return ClientModeFile, nil
}
func (g *S3Getter) Get(dst string, u *url.URL) error { func (g *S3Getter) Get(dst string, u *url.URL) error {
// Parse URL // Parse URL
region, bucket, path, _, creds, err := g.parseUrl(u) region, bucket, path, _, creds, err := g.parseUrl(u)

6
vendor/vendor.json vendored
View File

@ -1619,10 +1619,10 @@
"revision": "875fb671b3ddc66f8e2f0acc33829c8cb989a38d" "revision": "875fb671b3ddc66f8e2f0acc33829c8cb989a38d"
}, },
{ {
"checksumSHA1": "aOWXSbYAdK3PBSMNFiK2ze4lPEc=", "checksumSHA1": "N+MSF+0nq3XFXaUCOooF9acE+ZI=",
"path": "github.com/hashicorp/go-getter", "path": "github.com/hashicorp/go-getter",
"revision": "2fbd997432e72fe36060c8f07ec1eaf98d098177", "revision": "cc80f38c726badeae53775d179755e1c4953d6cf",
"revisionTime": "2016-09-12T21:42:52Z" "revisionTime": "2017-01-26T00:13:12Z"
}, },
{ {
"checksumSHA1": "9J+kDr29yDrwsdu2ULzewmqGjpA=", "checksumSHA1": "9J+kDr29yDrwsdu2ULzewmqGjpA=",