resilien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

export ModuleStorage and use it for Tree.Load 

Exporting ModuleStorage allows us to explicitly pass in the storgae
location rather than extracting it out of the getter.Storage interface,
set a UI for communiating actions back to the user, and accepting a
services Disco for discovery.
This commit is contained in:
James Bardin 2017-10-27 11:29:29 -04:00
parent 70410d48ef
commit 36eb40a432
7 changed files with 127 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
config/module/get_test.go
View File

@ -196,7 +196,8 @@ func TestRegistryGitHubArchive(t *testing.T) {
storage := testStorage(t)
tree := NewTree("", testConfig(t, "registry-tar-subdir"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -204,7 +205,8 @@ func TestRegistryGitHubArchive(t *testing.T) {
t.Fatal("should be loaded")
}
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -212,7 +214,8 @@ func TestRegistryGitHubArchive(t *testing.T) {
server.Close()
tree = NewTree("", testConfig(t, "registry-tar-subdir"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -242,7 +245,8 @@ func TestRegisryModuleSubdir(t *testing.T) {
storage := testStorage(t)
tree := NewTree("", testConfig(t, "registry-subdir"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -250,7 +254,8 @@ func TestRegisryModuleSubdir(t *testing.T) {
t.Fatal("should be loaded")
}
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -299,7 +304,8 @@ func TestAccRegistryLoad(t *testing.T) {
storage := testStorage(t)
tree := NewTree("", testConfig(t, "registry-load"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -307,7 +313,8 @@ func TestAccRegistryLoad(t *testing.T) {
t.Fatal("should be loaded")
}
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}

5
config/module/module_test.go
View File

@ -7,7 +7,6 @@ import (
"path/filepath"
"testing"
"github.com/hashicorp/go-getter"
"github.com/hashicorp/terraform/config"
)
@ -42,7 +41,7 @@ func testConfig(t *testing.T, n string) *config.Config {
return c
}
func testStorage(t *testing.T) getter.Storage {
func testStorage(t *testing.T) *ModuleStorage {
t.Helper()
return &getter.FolderStorage{StorageDir: tempDir(t)}
return &ModuleStorage{StorageDir: tempDir(t)}
}

78
config/module/storage.go
View File

@ -7,10 +7,11 @@ import (
"log"
"os"
"path/filepath"
"reflect"
getter "github.com/hashicorp/go-getter"
"github.com/hashicorp/terraform/registry/regsrc"
"github.com/hashicorp/terraform/svchost/disco"
"github.com/mitchellh/cli"
)
const manifestName = "modules.json"
@ -54,49 +55,22 @@ type moduleRecord struct {
registry bool
}
// moduleStorage implements methods to record and fetch metadata about the
// ModuleStorage implements methods to record and fetch metadata about the
// modules that have been fetched and stored locally. The getter.Storgae
// abstraction doesn't provide the information needed to know which versions of
// a module have been stored, or their location.
type moduleStorage struct {
getter.Storage
storageDir string
mode GetMode
}
func newModuleStorage(s getter.Storage, mode GetMode) moduleStorage {
return moduleStorage{
Storage: s,
storageDir: storageDir(s),
mode: mode,
}
}
// The Tree needs to know where to store the module manifest.
// Th Storage abstraction doesn't provide access to the storage root directory,
// so we extract it here.
func storageDir(s getter.Storage) string {
// get the StorageDir directly if possible
switch t := s.(type) {
case *getter.FolderStorage:
return t.StorageDir
case moduleStorage:
return t.storageDir
case nil:
return ""
}
// this should be our UI wrapper which is exported here, so we need to
// extract the FolderStorage via reflection.
fs := reflect.ValueOf(s).Elem().FieldByName("Storage").Interface()
return storageDir(fs.(getter.Storage))
type ModuleStorage struct {
StorageDir string
Services *disco.Disco
Ui cli.Ui
Mode GetMode
}
// loadManifest returns the moduleManifest file from the parent directory.
func (m moduleStorage) loadManifest() (moduleManifest, error) {
func (m ModuleStorage) loadManifest() (moduleManifest, error) {
manifest := moduleManifest{}
manifestPath := filepath.Join(m.storageDir, manifestName)
manifestPath := filepath.Join(m.StorageDir, manifestName)
data, err := ioutil.ReadFile(manifestPath)
if err != nil && !os.IsNotExist(err) {
return manifest, err
@ -116,7 +90,7 @@ func (m moduleStorage) loadManifest() (moduleManifest, error) {
// root directory. The storage method loads the entire file and rewrites it
// each time. This is only done a few times during init, so efficiency is
// not a concern.
func (m moduleStorage) recordModule(rec moduleRecord) error {
func (m ModuleStorage) recordModule(rec moduleRecord) error {
manifest, err := m.loadManifest()
if err != nil {
// if there was a problem with the file, we will attempt to write a new
@ -146,14 +120,14 @@ func (m moduleStorage) recordModule(rec moduleRecord) error {
panic(err)
}
manifestPath := filepath.Join(m.storageDir, manifestName)
manifestPath := filepath.Join(m.StorageDir, manifestName)
return ioutil.WriteFile(manifestPath, js, 0644)
}
// load the manifest from dir, and return all module versions matching the
// provided source. Records with no version info will be skipped, as they need
// to be uniquely identified by other means.
func (m moduleStorage) moduleVersions(source string) ([]moduleRecord, error) {
func (m ModuleStorage) moduleVersions(source string) ([]moduleRecord, error) {
manifest, err := m.loadManifest()
if err != nil {
return manifest.Modules, err
@ -170,7 +144,7 @@ func (m moduleStorage) moduleVersions(source string) ([]moduleRecord, error) {
return matching, nil
}
func (m moduleStorage) moduleDir(key string) (string, error) {
func (m ModuleStorage) moduleDir(key string) (string, error) {
manifest, err := m.loadManifest()
if err != nil {
return "", err
@ -186,7 +160,7 @@ func (m moduleStorage) moduleDir(key string) (string, error) {
}
// return only the root directory of the module stored in dir.
func (m moduleStorage) getModuleRoot(dir string) (string, error) {
func (m ModuleStorage) getModuleRoot(dir string) (string, error) {
manifest, err := m.loadManifest()
if err != nil {
return "", err
@ -202,7 +176,7 @@ func (m moduleStorage) getModuleRoot(dir string) (string, error) {
// record only the Root directory for the module stored at dir.
// TODO: remove this compatibility function to store the full moduleRecord.
func (m moduleStorage) recordModuleRoot(dir, root string) error {
func (m ModuleStorage) recordModuleRoot(dir, root string) error {
rec := moduleRecord{
Dir: dir,
Root: root,
@ -211,24 +185,28 @@ func (m moduleStorage) recordModuleRoot(dir, root string) error {
return m.recordModule(rec)
}
func (m moduleStorage) getStorage(key string, src string) (string, bool, error) {
func (m ModuleStorage) getStorage(key string, src string) (string, bool, error) {
storage := &getter.FolderStorage{
StorageDir: m.StorageDir,
}
// Get the module with the level specified if we were told to.
if m.mode > GetModeNone {
if m.Mode > GetModeNone {
log.Printf("[DEBUG] fetching %q with key %q", src, key)
if err := m.Storage.Get(key, src, m.mode == GetModeUpdate); err != nil {
if err := storage.Get(key, src, m.Mode == GetModeUpdate); err != nil {
return "", false, err
}
}
// Get the directory where the module is.
dir, found, err := m.Storage.Dir(key)
dir, found, err := storage.Dir(key)
log.Printf("[DEBUG] found %q in %q: %t", src, dir, found)
return dir, found, err
}
// find a stored module that's not from a registry
func (m moduleStorage) findModule(key string) (string, error) {
if m.mode == GetModeUpdate {
func (m ModuleStorage) findModule(key string) (string, error) {
if m.Mode == GetModeUpdate {
return "", nil
}
@ -236,7 +214,7 @@ func (m moduleStorage) findModule(key string) (string, error) {
}
// find a registry module
func (m moduleStorage) findRegistryModule(mSource, constraint string) (moduleRecord, error) {
func (m ModuleStorage) findRegistryModule(mSource, constraint string) (moduleRecord, error) {
rec := moduleRecord{
Source: mSource,
}
@ -272,7 +250,7 @@ func (m moduleStorage) findRegistryModule(mSource, constraint string) (moduleRec
// we need to lookup available versions
// Only on Get if it's not found, on unconditionally on Update
if (m.mode == GetModeGet && !found) || (m.mode == GetModeUpdate) {
if (m.Mode == GetModeGet && !found) || (m.Mode == GetModeUpdate) {
resp, err := lookupModuleVersions(nil, mod)
if err != nil {
return rec, err

6
config/module/testing.go
View File

@ -4,8 +4,6 @@ import (
"io/ioutil"
"os"
"testing"
"github.com/hashicorp/go-getter"
)
// TestTree loads a module at the given path and returns the tree as well
@ -26,8 +24,8 @@ func TestTree(t *testing.T, path string) (*Tree, func()) {
}
// Get the child modules
s := &getter.FolderStorage{StorageDir: dir}
if err := mod.Load(s, GetModeGet); err != nil {
s := &ModuleStorage{StorageDir: dir, Mode: GetModeGet}
if err := mod.Load(s); err != nil {
t.Fatalf("err: %s", err)
return nil, nil
}

10
config/module/tree.go
View File

@ -52,7 +52,7 @@ func NewEmptyTree() *Tree {
// We do this dummy load so that the tree is marked as "loaded". It
// should never fail because this is just about a no-op. If it does fail
// we panic so we can know its a bug.
if err := t.Load(nil, GetModeGet); err != nil {
if err := t.Load(&ModuleStorage{Mode: GetModeGet}); err != nil {
panic(err)
}
@ -169,12 +169,10 @@ func (t *Tree) Name() string {
// module trees inherently require the configuration to be in a reasonably
// sane state: no circular dependencies, proper module sources, etc. A full
// suite of validations can be done by running Validate (after loading).
func (t *Tree) Load(storage getter.Storage, mode GetMode) error {
func (t *Tree) Load(s *ModuleStorage) error {
t.lock.Lock()
defer t.lock.Unlock()
s := newModuleStorage(storage, mode)
children, err := t.getChildren(s)
if err != nil {
return err
@ -182,7 +180,7 @@ func (t *Tree) Load(storage getter.Storage, mode GetMode) error {
// Go through all the children and load them.
for _, c := range children {
if err := c.Load(storage, mode); err != nil {
if err := c.Load(s); err != nil {
return err
}
}
@ -198,7 +196,7 @@ func (t *Tree) Load(storage getter.Storage, mode GetMode) error {
return nil
}
func (t *Tree) getChildren(s moduleStorage) (map[string]*Tree, error) {
func (t *Tree) getChildren(s *ModuleStorage) (map[string]*Tree, error) {
children := make(map[string]*Tree)
// Go through all the modules and get the directory for them.

3
config/module/tree_gob_test.go
View File

@ -12,7 +12,8 @@ func TestTreeEncodeDecodeGob(t *testing.T) {
tree := NewTree("", testConfig(t, "basic"))
// This should get things
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}

110
config/module/tree_test.go
View File

@ -10,7 +10,6 @@ import (
"strings"
"testing"
"github.com/hashicorp/go-getter"
"github.com/hashicorp/terraform/config"
"github.com/hashicorp/terraform/helper/copy"
)
@ -22,8 +21,9 @@ func TestTreeChild(t *testing.T) {
}
storage := testStorage(t)
storage.Mode = GetModeGet
tree := NewTree("", testConfig(t, "child"))
if err := tree.Load(storage, GetModeGet); err != nil {
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -73,7 +73,7 @@ func TestTreeLoad(t *testing.T) {
}
// This should error because we haven't gotten things yet
if err := tree.Load(storage, GetModeNone); err == nil {
if err := tree.Load(storage); err == nil {
t.Fatal("should error")
}
@ -82,7 +82,8 @@ func TestTreeLoad(t *testing.T) {
}
// This should get things
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -91,7 +92,8 @@ func TestTreeLoad(t *testing.T) {
}
// This should no longer error
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -111,19 +113,23 @@ func TestTreeLoad_duplicate(t *testing.T) {
}
// This should get things
if err := tree.Load(storage, GetModeGet); err == nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err == nil {
t.Fatalf("should error")
}
}
func TestTreeLoad_copyable(t *testing.T) {
dir := tempDir(t)
storage := &getter.FolderStorage{StorageDir: dir}
storage := &ModuleStorage{
StorageDir: dir,
Mode: GetModeGet,
}
cfg := testConfig(t, "basic")
tree := NewTree("", cfg)
// This should get things
if err := tree.Load(storage, GetModeGet); err != nil {
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -132,7 +138,8 @@ func TestTreeLoad_copyable(t *testing.T) {
}
// This should no longer error
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -160,10 +167,13 @@ func TestTreeLoad_copyable(t *testing.T) {
}
tree := NewTree("", cfg)
storage := &getter.FolderStorage{StorageDir: dir2}
storage := &ModuleStorage{
StorageDir: dir2,
Mode: GetModeNone,
}
// This should not error since we already got it!
if err := tree.Load(storage, GetModeNone); err != nil {
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -182,7 +192,8 @@ func TestTreeLoad_parentRef(t *testing.T) {
}
// This should error because we haven't gotten things yet
if err := tree.Load(storage, GetModeNone); err == nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err == nil {
t.Fatal("should error")
}
@ -191,7 +202,8 @@ func TestTreeLoad_parentRef(t *testing.T) {
}
// This should get things
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -200,7 +212,8 @@ func TestTreeLoad_parentRef(t *testing.T) {
}
// This should no longer error
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -233,7 +246,8 @@ func TestTreeLoad_subdir(t *testing.T) {
}
// This should error because we haven't gotten things yet
if err := tree.Load(storage, GetModeNone); err == nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err == nil {
t.Fatal("should error")
}
@ -242,7 +256,8 @@ func TestTreeLoad_subdir(t *testing.T) {
}
// This should get things
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -251,7 +266,8 @@ func TestTreeLoad_subdir(t *testing.T) {
}
// This should no longer error
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -271,7 +287,7 @@ func TestTree_recordManifest(t *testing.T) {
}
defer os.RemoveAll(td)
storage := moduleStorage{storageDir: td}
storage := ModuleStorage{StorageDir: td}
dir := filepath.Join(td, "0131bf0fef686e090b16bdbab4910ddf")
@ -388,7 +404,9 @@ func TestTreeValidate_table(t *testing.T) {
for i, tc := range cases {
t.Run(fmt.Sprintf("%d-%s", i, tc.Name), func(t *testing.T) {
tree := NewTree("", testConfig(t, tc.Fixture))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -409,7 +427,9 @@ func TestTreeValidate_table(t *testing.T) {
func TestTreeValidate_badChild(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-child-bad"))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -421,7 +441,9 @@ func TestTreeValidate_badChild(t *testing.T) {
func TestTreeValidate_badChildOutput(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-bad-output"))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -433,7 +455,9 @@ func TestTreeValidate_badChildOutput(t *testing.T) {
func TestTreeValidate_badChildOutputToModule(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-bad-output-to-module"))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -445,7 +469,9 @@ func TestTreeValidate_badChildOutputToModule(t *testing.T) {
func TestTreeValidate_badChildVar(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-bad-var"))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -457,7 +483,9 @@ func TestTreeValidate_badChildVar(t *testing.T) {
func TestTreeValidate_badRoot(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-root-bad"))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -469,7 +497,9 @@ func TestTreeValidate_badRoot(t *testing.T) {
func TestTreeValidate_good(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-child-good"))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -489,7 +519,9 @@ func TestTreeValidate_notLoaded(t *testing.T) {
func TestTreeValidate_requiredChildVar(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-required-var"))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -510,7 +542,9 @@ func TestTreeValidate_requiredChildVar(t *testing.T) {
func TestTreeValidate_unknownModule(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-module-unknown"))
if err := tree.Load(testStorage(t), GetModeNone); err != nil {
storage := testStorage(t)
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -523,7 +557,8 @@ func TestTreeProviders_basic(t *testing.T) {
storage := testStorage(t)
tree := NewTree("", testConfig(t, "basic-parent-providers"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -585,7 +620,8 @@ func TestTreeProviders_implicit(t *testing.T) {
storage := testStorage(t)
tree := NewTree("", testConfig(t, "implicit-parent-providers"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -625,7 +661,8 @@ func TestTreeProviders_implicitMultiLevel(t *testing.T) {
storage := testStorage(t)
tree := NewTree("", testConfig(t, "implicit-grandparent-providers"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -694,7 +731,8 @@ func TestTreeLoad_conflictingSubmoduleNames(t *testing.T) {
storage := testStorage(t)
tree := NewTree("", testConfig(t, "conficting-submodule-names"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("load failed: %s", err)
}
@ -703,7 +741,8 @@ func TestTreeLoad_conflictingSubmoduleNames(t *testing.T) {
}
// Try to reload
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("reload failed: %s", err)
}
@ -753,13 +792,14 @@ func TestTreeLoad_changeIntermediateSource(t *testing.T) {
if err := os.MkdirAll(".terraform/modules", 0777); err != nil {
t.Fatal(err)
}
storage := &getter.FolderStorage{StorageDir: ".terraform/modules"}
storage := &ModuleStorage{StorageDir: ".terraform/modules"}
cfg, err := config.LoadDir("./")
if err != nil {
t.Fatal(err)
}
tree := NewTree("", cfg)
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("load failed: %s", err)
}
@ -774,7 +814,7 @@ func TestTreeLoad_changeIntermediateSource(t *testing.T) {
t.Fatal(err)
}
tree = NewTree("", cfg)
if err := tree.Load(storage, GetModeGet); err != nil {
if err := tree.Load(storage); err != nil {
t.Fatalf("load failed: %s", err)
}