plugin/discovery: plugin caching mechanism

For users that have metered or slow internet connections it is annoying
to have Terraform constantly re-downloading the same files when they
initialize many separate directories.

To help such users, here we add an opt-in mechanism to use a local
directory as a read-through cache. When enabled, any plugin download will
be skipped if a suitable file already exists in the cache directory. If
the desired plugin isn't in the cache, it will be downloaded into the
cache for use next time.

This mechanism also serves to reduce total disk usage by allowing
plugin files to be shared between many configurations, as long as the
target system isn't Windows and supports either hardlinks or symlinks.
This commit is contained in:
Martin Atkins 2017-09-01 16:03:45 -07:00
parent 12d6bc8c30
commit 879899d434
4 changed files with 183 additions and 2 deletions

View File

@ -3,10 +3,12 @@ package discovery
import ( import (
"errors" "errors"
"fmt" "fmt"
"io"
"io/ioutil" "io/ioutil"
"log" "log"
"net/http" "net/http"
"os" "os"
"path/filepath"
"runtime" "runtime"
"strconv" "strconv"
"strings" "strings"
@ -48,6 +50,10 @@ type Installer interface {
type ProviderInstaller struct { type ProviderInstaller struct {
Dir string Dir string
// Cache is used to access and update a local cache of plugins if non-nil.
// Can be nil to disable caching.
Cache PluginCache
PluginProtocolVersion uint PluginProtocolVersion uint
// OS and Arch specify the OS and architecture that should be used when // OS and Arch specify the OS and architecture that should be used when
@ -101,6 +107,12 @@ func (i *ProviderInstaller) Get(provider string, req Constraints) (PluginMeta, e
// sort them newest to oldest // sort them newest to oldest
Versions(versions).Sort() Versions(versions).Sort()
// Ensure that our installation directory exists
err = os.MkdirAll(i.Dir, os.ModePerm)
if err != nil {
return PluginMeta{}, fmt.Errorf("failed to create plugin dir %s: %s", i.Dir, err)
}
// take the first matching plugin we find // take the first matching plugin we find
for _, v := range versions { for _, v := range versions {
url := i.providerURL(provider, v.String()) url := i.providerURL(provider, v.String())
@ -120,8 +132,8 @@ func (i *ProviderInstaller) Get(provider string, req Constraints) (PluginMeta, e
log.Printf("[DEBUG] fetching provider info for %s version %s", provider, v) log.Printf("[DEBUG] fetching provider info for %s version %s", provider, v)
if checkPlugin(url, i.PluginProtocolVersion) { if checkPlugin(url, i.PluginProtocolVersion) {
i.Ui.Info(fmt.Sprintf("- Downloading plugin for provider %q (%s)...", provider, v.String())) i.Ui.Info(fmt.Sprintf("- Downloading plugin for provider %q (%s)...", provider, v.String()))
log.Printf("[DEBUG] getting provider %q version %q at %s", provider, v, url) log.Printf("[DEBUG] getting provider %q version %q", provider, v)
err := getter.Get(i.Dir, url) err := i.install(provider, v, url)
if err != nil { if err != nil {
return PluginMeta{}, err return PluginMeta{}, err
} }
@ -168,6 +180,98 @@ func (i *ProviderInstaller) Get(provider string, req Constraints) (PluginMeta, e
return PluginMeta{}, ErrorNoVersionCompatible return PluginMeta{}, ErrorNoVersionCompatible
} }
func (i *ProviderInstaller) install(provider string, version Version, url string) error {
if i.Cache != nil {
log.Printf("[DEBUG] looking for provider %s %s in plugin cache", provider, version)
cached := i.Cache.CachedPluginPath("provider", provider, version)
if cached == "" {
log.Printf("[DEBUG] %s %s not yet in cache, so downloading %s", provider, version, url)
err := getter.Get(i.Cache.InstallDir(), url)
if err != nil {
return err
}
// should now be in cache
cached = i.Cache.CachedPluginPath("provider", provider, version)
if cached == "" {
// should never happen if the getter is behaving properly
// and the plugins are packaged properly.
return fmt.Errorf("failed to find downloaded plugin in cache %s", i.Cache.InstallDir())
}
}
// Link or copy the cached binary into our install dir so the
// normal resolution machinery can find it.
filename := filepath.Base(cached)
targetPath := filepath.Join(i.Dir, filename)
log.Printf("[DEBUG] installing %s %s to %s from local cache %s", provider, version, targetPath, cached)
// Delete if we can. If there's nothing there already then no harm done.
// This is important because we can't create a link if there's
// already a file of the same name present.
// (any other error here we'll catch below when we try to write here)
os.Remove(targetPath)
// We don't attempt linking on Windows because links are not
// comprehensively supported by all tools/apps in Windows and
// so we choose to be conservative to avoid creating any
// weird issues for Windows users.
linkErr := errors.New("link not supported for Windows") // placeholder error, never actually returned
if runtime.GOOS != "windows" {
// Try hard linking first. Hard links are preferable because this
// creates a self-contained directory that doesn't depend on the
// cache after install.
linkErr = os.Link(cached, targetPath)
// If that failed, try a symlink. This _does_ depend on the cache
// after install, so the user must manage the cache more carefully
// in this case, but avoids creating redundant copies of the
// plugins on disk.
if linkErr != nil {
linkErr = os.Symlink(cached, targetPath)
}
}
// If we still have an error then we'll try a copy as a fallback.
// In this case either the OS is Windows or the target filesystem
// can't support symlinks.
if linkErr != nil {
srcFile, err := os.Open(cached)
if err != nil {
return fmt.Errorf("failed to open cached plugin %s: %s", cached, err)
}
defer srcFile.Close()
destFile, err := os.OpenFile(targetPath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, os.ModePerm)
if err != nil {
return fmt.Errorf("failed to create %s: %s", targetPath, err)
}
_, err = io.Copy(destFile, srcFile)
if err != nil {
destFile.Close()
return fmt.Errorf("failed to copy cached plugin from %s to %s: %s", cached, targetPath, err)
}
err = destFile.Close()
if err != nil {
return fmt.Errorf("error creating %s: %s", targetPath, err)
}
}
// One way or another, by the time we get here we should have either
// a link or a copy of the cached plugin within i.Dir, as expected.
} else {
log.Printf("[DEBUG] plugin cache is disabled, so downloading %s %s from %s", provider, version, url)
err := getter.Get(i.Dir, url)
if err != nil {
return err
}
}
return nil
}
func (i *ProviderInstaller) PurgeUnused(used map[string]PluginMeta) (PluginMetaSet, error) { func (i *ProviderInstaller) PurgeUnused(used map[string]PluginMeta) (PluginMetaSet, error) {
purge := make(PluginMetaSet) purge := make(PluginMetaSet)

View File

@ -0,0 +1,48 @@
package discovery
// PluginCache is an interface implemented by objects that are able to maintain
// a cache of plugins.
type PluginCache interface {
// CachedPluginPath returns a path where the requested plugin is already
// cached, or an empty string if the requested plugin is not yet cached.
CachedPluginPath(kind string, name string, version Version) string
// InstallDir returns the directory that new plugins should be installed into
// in order to populate the cache. This directory should be used as the
// first argument to getter.Get when downloading plugins with go-getter.
//
// After installing into this directory, use CachedPluginPath to obtain the
// path where the plugin was installed.
InstallDir() string
}
// NewLocalPluginCache returns a PluginCache that caches plugins in a
// given local directory.
func NewLocalPluginCache(dir string) PluginCache {
return &pluginCache{
Dir: dir,
}
}
type pluginCache struct {
Dir string
}
func (c *pluginCache) CachedPluginPath(kind string, name string, version Version) string {
allPlugins := FindPlugins(kind, []string{c.Dir})
plugins := allPlugins.WithName(name).WithVersion(version)
if plugins.Count() == 0 {
// nothing cached
return ""
}
// There should generally be only one plugin here; if there's more than
// one match for some reason then we'll just choose one arbitrarily.
plugin := plugins.Newest()
return plugin.Path
}
func (c *pluginCache) InstallDir() string {
return c.Dir
}

View File

@ -0,0 +1,29 @@
package discovery
import (
"testing"
)
func TestLocalPluginCache(t *testing.T) {
cache := NewLocalPluginCache("test-fixtures/plugin-cache")
foo1Path := cache.CachedPluginPath("provider", "foo", VersionStr("v0.0.1").MustParse())
if foo1Path == "" {
t.Errorf("foo v0.0.1 not found; should have been found")
}
foo2Path := cache.CachedPluginPath("provider", "foo", VersionStr("v0.0.2").MustParse())
if foo2Path != "" {
t.Errorf("foo v0.0.2 found at %s; should not have been found", foo2Path)
}
baz1Path := cache.CachedPluginPath("provider", "baz", VersionStr("v0.0.1").MustParse())
if baz1Path != "" {
t.Errorf("baz v0.0.1 found at %s; should not have been found", baz1Path)
}
baz2Path := cache.CachedPluginPath("provider", "baz", VersionStr("v0.0.2").MustParse())
if baz1Path != "" {
t.Errorf("baz v0.0.2 found at %s; should not have been found", baz2Path)
}
}