From 2a6990e2b9c6ea9885ecca5ecb804238f180f40c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Sep 2014 15:58:30 -0700 Subject: [PATCH 01/41] config: `module` structures parse --- config/config.go | 17 +++++++ config/loader_hcl.go | 85 +++++++++++++++++++++++++++++++++ config/loader_test.go | 57 ++++++++++++++++++++++ config/test-fixtures/modules.tf | 4 ++ 4 files changed, 163 insertions(+) create mode 100644 config/test-fixtures/modules.tf diff --git a/config/config.go b/config/config.go index 102e18163..c055189a5 100644 --- a/config/config.go +++ b/config/config.go @@ -15,6 +15,7 @@ import ( // Config is the configuration that comes from loading a collection // of Terraform templates. type Config struct { + Modules []*Module ProviderConfigs []*ProviderConfig Resources []*Resource Variables []*Variable @@ -25,6 +26,17 @@ type Config struct { unknownKeys []string } +// Module is a module used within a configuration. +// +// This does not represent a module itself, this represents a module +// call-site within an existing configuration. +type Module struct { + Name string + Type string + Source string + RawConfig *RawConfig +} + // ProviderConfig is the configuration for a resource provider. // // For example, Terraform needs to set the AWS access keys for the AWS @@ -92,6 +104,11 @@ func ProviderConfigName(t string, pcs []*ProviderConfig) string { return lk } +// A unique identifier for this module. +func (r *Module) Id() string { + return fmt.Sprintf("%s.%s", r.Type, r.Name) +} + // A unique identifier for this resource. func (r *Resource) Id() string { return fmt.Sprintf("%s.%s", r.Type, r.Name) diff --git a/config/loader_hcl.go b/config/loader_hcl.go index 380d50294..13337006a 100644 --- a/config/loader_hcl.go +++ b/config/loader_hcl.go @@ -17,6 +17,7 @@ type hclConfigurable struct { func (t *hclConfigurable) Config() (*Config, error) { validKeys := map[string]struct{}{ + "module": struct{}{}, "output": struct{}{}, "provider": struct{}{}, "resource": struct{}{}, @@ -69,6 +70,15 @@ func (t *hclConfigurable) Config() (*Config, error) { } } + // Build the modules + if modules := t.Object.Get("module", false); modules != nil { + var err error + config.Modules, err = loadModulesHcl(modules) + if err != nil { + return nil, err + } + } + // Build the provider configs if providers := t.Object.Get("provider", false); providers != nil { var err error @@ -177,6 +187,81 @@ func loadFileHcl(root string) (configurable, []string, error) { return result, nil, nil } +// Given a handle to a HCL object, this recurses into the structure +// and pulls out a list of modules. +// +// The resulting modules may not be unique, but each module +// represents exactly one module definition in the HCL configuration. +// We leave it up to another pass to merge them together. +func loadModulesHcl(os *hclobj.Object) ([]*Module, error) { + var allTypes []*hclobj.Object + + // See loadResourcesHcl for why this exists. Don't touch this. + for _, o1 := range os.Elem(false) { + // Iterate the inner to get the list of types + for _, o2 := range o1.Elem(true) { + // Iterate all of this type to get _all_ the types + for _, o3 := range o2.Elem(false) { + allTypes = append(allTypes, o3) + } + } + } + + // Where all the results will go + var result []*Module + + // Now go over all the types and their children in order to get + // all of the actual resources. + for _, t := range allTypes { + for _, obj := range t.Elem(true) { + k := obj.Key + + var config map[string]interface{} + if err := hcl.DecodeObject(&config, obj); err != nil { + return nil, fmt.Errorf( + "Error reading config for %s[%s]: %s", + t.Key, + k, + err) + } + + // Remove the fields we handle specially + delete(config, "source") + + rawConfig, err := NewRawConfig(config) + if err != nil { + return nil, fmt.Errorf( + "Error reading config for %s[%s]: %s", + t.Key, + k, + err) + } + + // If we have a count, then figure it out + var source string + if o := obj.Get("source", false); o != nil { + err = hcl.DecodeObject(&source, o) + if err != nil { + return nil, fmt.Errorf( + "Error parsing source for %s[%s]: %s", + t.Key, + k, + err) + } + } + + result = append(result, &Module{ + Name: k, + Type: t.Key, + Source: source, + RawConfig: rawConfig, + }) + } + } + + return result, nil +} + // LoadOutputsHcl recurses into the given HCL object and turns // it into a mapping of outputs. func loadOutputsHcl(os *hclobj.Object) ([]*Output, error) { diff --git a/config/loader_test.go b/config/loader_test.go index b20096d6f..e4843e5c6 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -106,6 +106,22 @@ func TestLoadBasic_json(t *testing.T) { } } +func TestLoadBasic_modules(t *testing.T) { + c, err := Load(filepath.Join(fixtureDir, "modules.tf")) + if err != nil { + t.Fatalf("err: %s", err) + } + + if c == nil { + t.Fatal("config should not be nil") + } + + actual := modulesStr(c.Modules) + if actual != strings.TrimSpace(modulesModulesStr) { + t.Fatalf("bad:\n%s", actual) + } +} + func TestLoad_variables(t *testing.T) { c, err := Load(filepath.Join(fixtureDir, "variables.tf")) if err != nil { @@ -302,6 +318,41 @@ func TestLoad_connections(t *testing.T) { } } +func modulesStr(ms []*Module) string { + result := "" + order := make([]int, 0, len(ms)) + ks := make([]string, 0, len(ms)) + mapping := make(map[string]int) + for i, m := range ms { + k := m.Id() + ks = append(ks, k) + mapping[k] = i + } + sort.Strings(ks) + for _, k := range ks { + order = append(order, mapping[k]) + } + + for _, i := range order { + m := ms[i] + result += fmt.Sprintf("%s\n", m.Id()) + + ks := make([]string, 0, len(m.RawConfig.Raw)) + for k, _ := range m.RawConfig.Raw { + ks = append(ks, k) + } + sort.Strings(ks) + + result += fmt.Sprintf(" source = %s\n", m.Source) + + for _, k := range ks { + result += fmt.Sprintf(" %s\n", k) + } + } + + return strings.TrimSpace(result) +} + // This helper turns a provider configs field into a deterministic // string value for comparison in tests. func providerConfigsStr(pcs []*ProviderConfig) string { @@ -611,6 +662,12 @@ foo bar ` +const modulesModulesStr = ` +foo.bar + source = baz + memory +` + const provisionerResourcesStr = ` aws_instance[web] (x1) ami diff --git a/config/test-fixtures/modules.tf b/config/test-fixtures/modules.tf new file mode 100644 index 000000000..48e7ad09e --- /dev/null +++ b/config/test-fixtures/modules.tf @@ -0,0 +1,4 @@ +module "foo" "bar" { + memory = "1G" + source = "baz" +} From 610e92cab2c4fdf94d5f671ddcdbf0570c8a79a3 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Sep 2014 16:02:36 -0700 Subject: [PATCH 02/41] config: validate no duplicate modules --- config/config.go | 20 ++++++++++++++++++- config/config_test.go | 7 +++++++ .../test-fixtures/validate-dup-module/main.tf | 7 +++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 config/test-fixtures/validate-dup-module/main.tf diff --git a/config/config.go b/config/config.go index c055189a5..cdc2237c3 100644 --- a/config/config.go +++ b/config/config.go @@ -173,9 +173,27 @@ func (c *Config) Validate() error { } } + // Check that all references to modules are valid + modules := make(map[string]*Module) + dupped := make(map[string]struct{}) + for _, m := range c.Modules { + if _, ok := modules[m.Id()]; ok { + if _, ok := dupped[m.Id()]; !ok { + dupped[m.Id()] = struct{}{} + + errs = append(errs, fmt.Errorf( + "%s: module repeated multiple times", + m.Id())) + } + } + + modules[m.Id()] = m + } + dupped = nil + // Check that all references to resources are valid resources := make(map[string]*Resource) - dupped := make(map[string]struct{}) + dupped = make(map[string]struct{}) for _, r := range c.Resources { if _, ok := resources[r.Id()]; ok { if _, ok := dupped[r.Id()]; !ok { diff --git a/config/config_test.go b/config/config_test.go index 59242277b..581109e9c 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -44,6 +44,13 @@ func TestConfigValidate_countZero(t *testing.T) { } } +func TestConfigValidate_dupModule(t *testing.T) { + c := testConfig(t, "validate-dup-module") + if err := c.Validate(); err == nil { + t.Fatal("should not be valid") + } +} + func TestConfigValidate_dupResource(t *testing.T) { c := testConfig(t, "validate-dup-resource") if err := c.Validate(); err == nil { diff --git a/config/test-fixtures/validate-dup-module/main.tf b/config/test-fixtures/validate-dup-module/main.tf new file mode 100644 index 000000000..cf7fd3d9c --- /dev/null +++ b/config/test-fixtures/validate-dup-module/main.tf @@ -0,0 +1,7 @@ +module "aws_instance" "web" { + source = "foo" +} + +module "aws_instance" "web" { + source = "bar" +} From e7fe5aa4524566afb40c8de01425eff956f4c9a5 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Sep 2014 19:50:41 -0700 Subject: [PATCH 03/41] config: Append works with modules --- config/append.go | 7 +++++++ config/append_test.go | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/config/append.go b/config/append.go index de7b2c824..f87e67748 100644 --- a/config/append.go +++ b/config/append.go @@ -29,6 +29,13 @@ func Append(c1, c2 *Config) (*Config, error) { } } + if len(c1.Modules) > 0 || len(c2.Modules) > 0 { + c.Modules = make( + []*Module, 0, len(c1.Modules)+len(c2.Modules)) + c.Modules = append(c.Modules, c1.Modules...) + c.Modules = append(c.Modules, c2.Modules...) + } + if len(c1.Outputs) > 0 || len(c2.Outputs) > 0 { c.Outputs = make( []*Output, 0, len(c1.Outputs)+len(c2.Outputs)) diff --git a/config/append_test.go b/config/append_test.go index 4a7fb9d1e..e7aea9d21 100644 --- a/config/append_test.go +++ b/config/append_test.go @@ -12,6 +12,9 @@ func TestAppend(t *testing.T) { }{ { &Config{ + Modules: []*Module{ + &Module{Name: "foo"}, + }, Outputs: []*Output{ &Output{Name: "foo"}, }, @@ -29,6 +32,9 @@ func TestAppend(t *testing.T) { }, &Config{ + Modules: []*Module{ + &Module{Name: "bar"}, + }, Outputs: []*Output{ &Output{Name: "bar"}, }, @@ -46,6 +52,10 @@ func TestAppend(t *testing.T) { }, &Config{ + Modules: []*Module{ + &Module{Name: "foo"}, + &Module{Name: "bar"}, + }, Outputs: []*Output{ &Output{Name: "foo"}, &Output{Name: "bar"}, From dd6f536fab70defc103e0c6ad4b738eb8e517378 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Sep 2014 19:54:02 -0700 Subject: [PATCH 04/41] config: Merge for modules works --- config/config.go | 19 +++++++++++++++++++ config/merge.go | 17 +++++++++++++++++ config/merge_test.go | 10 ++++++++++ 3 files changed, 46 insertions(+) diff --git a/config/config.go b/config/config.go index cdc2237c3..51a033427 100644 --- a/config/config.go +++ b/config/config.go @@ -309,6 +309,25 @@ func (c *Config) allVariables() map[string][]InterpolatedVariable { return result } +func (m *Module) mergerName() string { + return m.Id() +} + +func (m *Module) mergerMerge(other merger) merger { + m2 := other.(*Module) + + result := *m + result.Name = m2.Name + result.Type = m2.Type + result.RawConfig = result.RawConfig.merge(m2.RawConfig) + + if m2.Source != "" { + result.Source = m2.Source + } + + return &result +} + func (o *Output) mergerName() string { return o.Name } diff --git a/config/merge.go b/config/merge.go index 835d652a9..c43f13c04 100644 --- a/config/merge.go +++ b/config/merge.go @@ -35,6 +35,23 @@ func Merge(c1, c2 *Config) (*Config, error) { var m1, m2, mresult []merger + // Modules + m1 = make([]merger, 0, len(c1.Modules)) + m2 = make([]merger, 0, len(c2.Modules)) + for _, v := range c1.Modules { + m1 = append(m1, v) + } + for _, v := range c2.Modules { + m2 = append(m2, v) + } + mresult = mergeSlice(m1, m2) + if len(mresult) > 0 { + c.Modules = make([]*Module, len(mresult)) + for i, v := range mresult { + c.Modules[i] = v.(*Module) + } + } + // Outputs m1 = make([]merger, 0, len(c1.Outputs)) m2 = make([]merger, 0, len(c2.Outputs)) diff --git a/config/merge_test.go b/config/merge_test.go index 10ef5f37a..2dbe5aee9 100644 --- a/config/merge_test.go +++ b/config/merge_test.go @@ -13,6 +13,9 @@ func TestMerge(t *testing.T) { // Normal good case. { &Config{ + Modules: []*Module{ + &Module{Name: "foo"}, + }, Outputs: []*Output{ &Output{Name: "foo"}, }, @@ -30,6 +33,9 @@ func TestMerge(t *testing.T) { }, &Config{ + Modules: []*Module{ + &Module{Name: "bar"}, + }, Outputs: []*Output{ &Output{Name: "bar"}, }, @@ -47,6 +53,10 @@ func TestMerge(t *testing.T) { }, &Config{ + Modules: []*Module{ + &Module{Name: "foo"}, + &Module{Name: "bar"}, + }, Outputs: []*Output{ &Output{Name: "foo"}, &Output{Name: "bar"}, From bb2209004026cd96b3ccc0c7818d0b53d8514eef Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Sep 2014 17:45:56 -0700 Subject: [PATCH 05/41] config/module: start, lots of initial work --- config/module/folder_storage.go | 65 +++++++++++++++++++++ config/module/folder_storage_test.go | 60 +++++++++++++++++++ config/module/get.go | 53 +++++++++++++++++ config/module/get_file.go | 41 +++++++++++++ config/module/module.go | 7 +++ config/module/module_test.go | 21 +++++++ config/module/storage.go | 13 +++++ config/module/test-fixtures/basic/main.tf | 1 + config/module/tree.go | 71 +++++++++++++++++++++++ 9 files changed, 332 insertions(+) create mode 100644 config/module/folder_storage.go create mode 100644 config/module/folder_storage_test.go create mode 100644 config/module/get.go create mode 100644 config/module/get_file.go create mode 100644 config/module/module.go create mode 100644 config/module/module_test.go create mode 100644 config/module/storage.go create mode 100644 config/module/test-fixtures/basic/main.tf create mode 100644 config/module/tree.go diff --git a/config/module/folder_storage.go b/config/module/folder_storage.go new file mode 100644 index 000000000..dfb79748a --- /dev/null +++ b/config/module/folder_storage.go @@ -0,0 +1,65 @@ +package module + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "os" + "path/filepath" +) + +// FolderStorage is an implementation of the Storage interface that manages +// modules on the disk. +type FolderStorage struct { + // StorageDir is the directory where the modules will be stored. + StorageDir string +} + +// Dir implements Storage.Dir +func (s *FolderStorage) Dir(source string) (d string, e bool, err error) { + d = s.dir(source) + _, err = os.Stat(d) + if err == nil { + // Directory exists + e = true + return + } + if os.IsNotExist(err) { + // Directory doesn't exist + d = "" + e = false + err = nil + return + } + + // An error + d = "" + e = false + return +} + +// Get implements Storage.Get +func (s *FolderStorage) Get(source string, update bool) error { + dir := s.dir(source) + if !update { + if _, err := os.Stat(dir); err == nil { + // If the directory already exists, then we're done since + // we're not updating. + return nil + } else if !os.IsNotExist(err) { + // If the error we got wasn't a file-not-exist error, then + // something went wrong and we should report it. + return fmt.Errorf("Error reading module directory: %s", err) + } + } + + // Get the source. This always forces an update. + return Get(dir, source) +} + +// dir returns the directory name internally that we'll use to map to +// internally. +func (s *FolderStorage) dir(source string) string { + sum := md5.Sum([]byte(source)) + return filepath.Join(s.StorageDir, hex.EncodeToString(sum[:])) +} diff --git a/config/module/folder_storage_test.go b/config/module/folder_storage_test.go new file mode 100644 index 000000000..7be9cbe7b --- /dev/null +++ b/config/module/folder_storage_test.go @@ -0,0 +1,60 @@ +package module + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func TestFolderStorage_impl(t *testing.T) { + var _ Storage = new(FolderStorage) +} + +func TestFolderStorage(t *testing.T) { + s := &FolderStorage{StorageDir: tempDir(t)} + + module := testModule("basic") + + // A module shouldn't exist at first... + _, ok, err := s.Dir(module) + if err != nil { + t.Fatalf("err: %s", err) + } + if ok { + t.Fatal("should not exist") + } + + // We can get it + err = s.Get(module, false) + if err != nil { + t.Fatalf("err: %s", err) + } + + // Now the module exists + dir, ok, err := s.Dir(module) + if err != nil { + t.Fatalf("err: %s", err) + } + if !ok { + t.Fatal("should exist") + } + + mainPath := filepath.Join(dir, "main.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } + +} + +func tempDir(t *testing.T) string { + dir, err := ioutil.TempDir("", "tf") + if err != nil { + t.Fatalf("err: %s", err) + } + if err := os.RemoveAll(dir); err != nil { + t.Fatalf("err: %s", err) + } + + return dir +} diff --git a/config/module/get.go b/config/module/get.go new file mode 100644 index 000000000..270ca815a --- /dev/null +++ b/config/module/get.go @@ -0,0 +1,53 @@ +package module + +import ( + "fmt" + "net/url" +) + +// Getter defines the interface that schemes must implement to download +// and update modules. +type Getter interface { + // Get downloads the given URL into the given directory. This always + // assumes that we're updating and gets the latest version that it can. + // + // The directory may already exist (if we're updating). If it is in a + // format that isn't understood, an error should be returned. Get shouldn't + // simply nuke the directory. + Get(string, *url.URL) error +} + +// Getters is the mapping of scheme to the Getter implementation that will +// be used to get a dependency. +var Getters map[string]Getter + +func init() { + Getters = map[string]Getter{ + "file": new(FileGetter), + } +} + +// Get downloads the module specified by src into the folder specified by +// dst. If dst already exists, Get will attempt to update it. +// +// src is a URL, whereas dst is always just a file path to a folder. This +// folder doesn't need to exist. It will be created if it doesn't exist. +func Get(dst, src string) error { + u, err := url.Parse(src) + if err != nil { + return err + } + + g, ok := Getters[u.Scheme] + if !ok { + return fmt.Errorf( + "module download not supported for scheme '%s'", u.Scheme) + } + + err = g.Get(dst, u) + if err != nil { + err = fmt.Errorf("error downloading module '%s': %s", src, err) + } + + return err +} diff --git a/config/module/get_file.go b/config/module/get_file.go new file mode 100644 index 000000000..66ead7e51 --- /dev/null +++ b/config/module/get_file.go @@ -0,0 +1,41 @@ +package module + +import ( + "fmt" + "net/url" + "os" + "path/filepath" +) + +// FileGetter is a Getter implementation that will download a module from +// a file scheme. +type FileGetter struct{} + +func (g *FileGetter) Get(dst string, u *url.URL) error { + fi, err := os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return err + } + + // If the destination already exists, it must be a symlink + if err == nil { + mode := fi.Mode() + if mode&os.ModeSymlink != 0 { + return fmt.Errorf("destination exists and is not a symlink") + } + } + + // The source path must exist and be a directory to be usable. + if fi, err := os.Stat(u.Path); err != nil { + return fmt.Errorf("source path error: %s", err) + } else if !fi.IsDir() { + return fmt.Errorf("source path must be a directory") + } + + // Create all the parent directories + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + + return os.Symlink(u.Path, dst) +} diff --git a/config/module/module.go b/config/module/module.go new file mode 100644 index 000000000..f8649f6e9 --- /dev/null +++ b/config/module/module.go @@ -0,0 +1,7 @@ +package module + +// Module represents the metadata for a single module. +type Module struct { + Name string + Source string +} diff --git a/config/module/module_test.go b/config/module/module_test.go new file mode 100644 index 000000000..01035f269 --- /dev/null +++ b/config/module/module_test.go @@ -0,0 +1,21 @@ +package module + +import ( + "net/url" + "path/filepath" +) + +const fixtureDir = "./test-fixtures" + +func testModule(n string) string { + p := filepath.Join(fixtureDir, n) + p, err := filepath.Abs(p) + if err != nil { + panic(err) + } + + var url url.URL + url.Scheme = "file" + url.Path = p + return url.String() +} diff --git a/config/module/storage.go b/config/module/storage.go new file mode 100644 index 000000000..14b5181e5 --- /dev/null +++ b/config/module/storage.go @@ -0,0 +1,13 @@ +package module + +// Storage is an interface that knows how to lookup downloaded modules +// as well as download and update modules from their sources into the +// proper location. +type Storage interface { + // Dir returns the directory on local disk where the modulue source + // can be loaded from. + Dir(string) (string, bool, error) + + // Get will download and optionally update the given module. + Get(string, bool) error +} diff --git a/config/module/test-fixtures/basic/main.tf b/config/module/test-fixtures/basic/main.tf new file mode 100644 index 000000000..fec56017d --- /dev/null +++ b/config/module/test-fixtures/basic/main.tf @@ -0,0 +1 @@ +# Hello diff --git a/config/module/tree.go b/config/module/tree.go new file mode 100644 index 000000000..851057c0f --- /dev/null +++ b/config/module/tree.go @@ -0,0 +1,71 @@ +package module + +import ( + "github.com/hashicorp/terraform/config" +) + +// Tree represents the module import tree of configurations. +// +// This Tree structure can be used to get (download) new modules, load +// all the modules without getting, flatten the tree into something +// Terraform can use, etc. +type Tree struct { + Config *config.Config + Children []*Tree +} + +// GetMode is an enum that describes how modules are loaded. +// +// GetModeLoad says that modules will not be downloaded or updated, they will +// only be loaded from the storage. +// +// GetModeGet says that modules can be initially downloaded if they don't +// exist, but otherwise to just load from the current version in storage. +// +// GetModeUpdate says that modules should be checked for updates and +// downloaded prior to loading. If there are no updates, we load the version +// from disk, otherwise we download first and then load. +type GetMode byte + +const ( + GetModeNone GetMode = iota + GetModeGet + GetModeUpdate +) + +// Flatten takes the entire module tree and flattens it into a single +// namespace in *config.Config with no module imports. +// +// Validate is called here implicitly, since it is important that semantic +// checks pass before flattening the configuration. Otherwise, encapsulation +// breaks in horrible ways and the errors that come out the other side +// will be surprising. +func (t *Tree) Flatten() (*config.Config, error) { + return nil, nil +} + +// Modules returns the list of modules that this tree imports. +func (t *Tree) Modules() []*Module { + return nil +} + +// Load loads the configuration of the entire tree. +// +// The parameters are used to tell the tree where to find modules and +// whether it can download/update modules along the way. +// +// Various semantic-like checks are made along the way of loading since +// module trees inherently require the configuration to be in a reasonably +// sane state: no circular dependencies, proper module sources, etc. A full +// suite of validations can be done by running Validate (after loading). +func (t *Tree) Load(s Storage, mode GetMode) error { + return nil +} + +// Validate does semantic checks on the entire tree of configurations. +// +// This will call the respective config.Config.Validate() functions as well +// as verifying things such as parameters/outputs between the various modules. +func (t *Tree) Validate() error { + return nil +} From c2fe35e74ea178691baaab9f216b2fa82098e96f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Sep 2014 18:54:12 -0700 Subject: [PATCH 06/41] config/module: FileGetter tests --- config/module/folder_storage_test.go | 14 ------ config/module/get_file_test.go | 75 ++++++++++++++++++++++++++++ config/module/module_test.go | 15 ++++++ 3 files changed, 90 insertions(+), 14 deletions(-) create mode 100644 config/module/get_file_test.go diff --git a/config/module/folder_storage_test.go b/config/module/folder_storage_test.go index 7be9cbe7b..4ffaac2bb 100644 --- a/config/module/folder_storage_test.go +++ b/config/module/folder_storage_test.go @@ -1,7 +1,6 @@ package module import ( - "io/ioutil" "os" "path/filepath" "testing" @@ -44,17 +43,4 @@ func TestFolderStorage(t *testing.T) { if _, err := os.Stat(mainPath); err != nil { t.Fatalf("err: %s", err) } - -} - -func tempDir(t *testing.T) string { - dir, err := ioutil.TempDir("", "tf") - if err != nil { - t.Fatalf("err: %s", err) - } - if err := os.RemoveAll(dir); err != nil { - t.Fatalf("err: %s", err) - } - - return dir } diff --git a/config/module/get_file_test.go b/config/module/get_file_test.go new file mode 100644 index 000000000..c805cd5c3 --- /dev/null +++ b/config/module/get_file_test.go @@ -0,0 +1,75 @@ +package module + +import ( + "net/url" + "os" + "path/filepath" + "testing" +) + +func TestFileGetter_impl(t *testing.T) { + var _ Getter = new(FileGetter) +} + +func TestFileGetter(t *testing.T) { + g := new(FileGetter) + dst := tempDir(t) + + // With a dir that doesn't exist + if err := g.Get(dst, testModuleURL("basic")); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath := filepath.Join(dst, "main.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestFileGetter_sourceFile(t *testing.T) { + g := new(FileGetter) + dst := tempDir(t) + + // With a source URL that is a path to a file + u := testModuleURL("basic") + u.Path += "/main.tf" + if err := g.Get(dst, u); err == nil { + t.Fatal("should error") + } +} + +func TestFileGetter_sourceNoExist(t *testing.T) { + g := new(FileGetter) + dst := tempDir(t) + + // With a source URL that doesn't exist + u := testModuleURL("basic") + u.Path += "/main" + if err := g.Get(dst, u); err == nil { + t.Fatal("should error") + } +} + +func TestFileGetter_dir(t *testing.T) { + g := new(FileGetter) + dst := tempDir(t) + + if err := os.MkdirAll(dst, 0755); err != nil { + t.Fatalf("err: %s", err) + } + + // With a dir that exists that isn't a symlink + if err := g.Get(dst, testModuleURL("basic")); err == nil { + t.Fatal("should error") + } +} + +func testModuleURL(n string) *url.URL { + u, err := url.Parse(testModule(n)) + if err != nil { + panic(err) + } + + return u +} diff --git a/config/module/module_test.go b/config/module/module_test.go index 01035f269..8360a8bf2 100644 --- a/config/module/module_test.go +++ b/config/module/module_test.go @@ -1,12 +1,27 @@ package module import ( + "io/ioutil" "net/url" + "os" "path/filepath" + "testing" ) const fixtureDir = "./test-fixtures" +func tempDir(t *testing.T) string { + dir, err := ioutil.TempDir("", "tf") + if err != nil { + t.Fatalf("err: %s", err) + } + if err := os.RemoveAll(dir); err != nil { + t.Fatalf("err: %s", err) + } + + return dir +} + func testModule(n string) string { p := filepath.Join(fixtureDir, n) p, err := filepath.Abs(p) From fa997525c2e156e54f0540f08c607999ff4ff886 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Sep 2014 18:58:21 -0700 Subject: [PATCH 07/41] config/module: Get tests --- config/module/get_test.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 config/module/get_test.go diff --git a/config/module/get_test.go b/config/module/get_test.go new file mode 100644 index 000000000..94d131a3d --- /dev/null +++ b/config/module/get_test.go @@ -0,0 +1,32 @@ +package module + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestGet_badSchema(t *testing.T) { + dst := tempDir(t) + u := testModule("basic") + u = strings.Replace(u, "file", "nope", -1) + + if err := Get(dst, u); err == nil { + t.Fatal("should error") + } +} + +func TestGet_file(t *testing.T) { + dst := tempDir(t) + u := testModule("basic") + + if err := Get(dst, u); err != nil { + t.Fatalf("err: %s", err) + } + + mainPath := filepath.Join(dst, "main.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} From 5e4c2b4f49a013ef6fae76573284495dc979ecef Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 14 Sep 2014 10:36:02 -0700 Subject: [PATCH 08/41] config/module: test that symlink that exists will be overwritten --- config/module/get_file.go | 21 +++++++++++------- config/module/get_file_test.go | 39 ++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/config/module/get_file.go b/config/module/get_file.go index 66ead7e51..73cb85834 100644 --- a/config/module/get_file.go +++ b/config/module/get_file.go @@ -12,7 +12,14 @@ import ( type FileGetter struct{} func (g *FileGetter) Get(dst string, u *url.URL) error { - fi, err := os.Stat(dst) + // The source path must exist and be a directory to be usable. + if fi, err := os.Stat(u.Path); err != nil { + return fmt.Errorf("source path error: %s", err) + } else if !fi.IsDir() { + return fmt.Errorf("source path must be a directory") + } + + fi, err := os.Lstat(dst) if err != nil && !os.IsNotExist(err) { return err } @@ -20,16 +27,14 @@ func (g *FileGetter) Get(dst string, u *url.URL) error { // If the destination already exists, it must be a symlink if err == nil { mode := fi.Mode() - if mode&os.ModeSymlink != 0 { + if mode&os.ModeSymlink == 0 { return fmt.Errorf("destination exists and is not a symlink") } - } - // The source path must exist and be a directory to be usable. - if fi, err := os.Stat(u.Path); err != nil { - return fmt.Errorf("source path error: %s", err) - } else if !fi.IsDir() { - return fmt.Errorf("source path must be a directory") + // Remove the destination + if err := os.Remove(dst); err != nil { + return err + } } // Create all the parent directories diff --git a/config/module/get_file_test.go b/config/module/get_file_test.go index c805cd5c3..6cde4befc 100644 --- a/config/module/get_file_test.go +++ b/config/module/get_file_test.go @@ -20,6 +20,15 @@ func TestFileGetter(t *testing.T) { t.Fatalf("err: %s", err) } + // Verify the destination folder is a symlink + fi, err := os.Lstat(dst) + if err != nil { + t.Fatalf("err: %s", err) + } + if fi.Mode()&os.ModeSymlink == 0 { + t.Fatal("destination is not a symlink") + } + // Verify the main file exists mainPath := filepath.Join(dst, "main.tf") if _, err := os.Stat(mainPath); err != nil { @@ -65,6 +74,36 @@ func TestFileGetter_dir(t *testing.T) { } } +func TestFileGetter_dirSymlink(t *testing.T) { + g := new(FileGetter) + dst := tempDir(t) + dst2 := tempDir(t) + + // Make parents + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + t.Fatalf("err: %s", err) + } + if err := os.MkdirAll(dst2, 0755); err != nil { + t.Fatalf("err: %s", err) + } + + // Make a symlink + if err := os.Symlink(dst2, dst); err != nil { + t.Fatalf("err: %s") + } + + // With a dir that exists that isn't a symlink + if err := g.Get(dst, testModuleURL("basic")); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath := filepath.Join(dst, "main.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} + func testModuleURL(n string) *url.URL { u, err := url.Parse(testModule(n)) if err != nil { From 8dc8eac4bf90c52159146acd89f20540842246fa Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 14 Sep 2014 14:43:54 -0700 Subject: [PATCH 09/41] config: change module syntax --- config/config.go | 4 +- config/loader_hcl.go | 70 +++++++++++++++------------------ config/loader_test.go | 2 +- config/test-fixtures/modules.tf | 2 +- 4 files changed, 35 insertions(+), 43 deletions(-) diff --git a/config/config.go b/config/config.go index 51a033427..60cb8818a 100644 --- a/config/config.go +++ b/config/config.go @@ -32,7 +32,6 @@ type Config struct { // call-site within an existing configuration. type Module struct { Name string - Type string Source string RawConfig *RawConfig } @@ -106,7 +105,7 @@ func ProviderConfigName(t string, pcs []*ProviderConfig) string { // A unique identifier for this module. func (r *Module) Id() string { - return fmt.Sprintf("%s.%s", r.Type, r.Name) + return fmt.Sprintf("module.%s", r.Name) } // A unique identifier for this resource. @@ -318,7 +317,6 @@ func (m *Module) mergerMerge(other merger) merger { result := *m result.Name = m2.Name - result.Type = m2.Type result.RawConfig = result.RawConfig.merge(m2.RawConfig) if m2.Source != "" { diff --git a/config/loader_hcl.go b/config/loader_hcl.go index 13337006a..babc34ca6 100644 --- a/config/loader_hcl.go +++ b/config/loader_hcl.go @@ -194,7 +194,7 @@ func loadFileHcl(root string) (configurable, []string, error) { // represents exactly one module definition in the HCL configuration. // We leave it up to another pass to merge them together. func loadModulesHcl(os *hclobj.Object) ([]*Module, error) { - var allTypes []*hclobj.Object + var allNames []*hclobj.Object // See loadResourcesHcl for why this exists. Don't touch this. for _, o1 := range os.Elem(false) { @@ -202,7 +202,7 @@ func loadModulesHcl(os *hclobj.Object) ([]*Module, error) { for _, o2 := range o1.Elem(true) { // Iterate all of this type to get _all_ the types for _, o3 := range o2.Elem(false) { - allTypes = append(allTypes, o3) + allNames = append(allNames, o3) } } } @@ -212,51 +212,45 @@ func loadModulesHcl(os *hclobj.Object) ([]*Module, error) { // Now go over all the types and their children in order to get // all of the actual resources. - for _, t := range allTypes { - for _, obj := range t.Elem(true) { - k := obj.Key + for _, obj := range allNames { + k := obj.Key - var config map[string]interface{} - if err := hcl.DecodeObject(&config, obj); err != nil { - return nil, fmt.Errorf( - "Error reading config for %s[%s]: %s", - t.Key, - k, - err) - } + var config map[string]interface{} + if err := hcl.DecodeObject(&config, obj); err != nil { + return nil, fmt.Errorf( + "Error reading config for %s: %s", + k, + err) + } - // Remove the fields we handle specially - delete(config, "source") + // Remove the fields we handle specially + delete(config, "source") - rawConfig, err := NewRawConfig(config) + rawConfig, err := NewRawConfig(config) + if err != nil { + return nil, fmt.Errorf( + "Error reading config for %s: %s", + k, + err) + } + + // If we have a count, then figure it out + var source string + if o := obj.Get("source", false); o != nil { + err = hcl.DecodeObject(&source, o) if err != nil { return nil, fmt.Errorf( - "Error reading config for %s[%s]: %s", - t.Key, + "Error parsing source for %s: %s", k, err) } - - // If we have a count, then figure it out - var source string - if o := obj.Get("source", false); o != nil { - err = hcl.DecodeObject(&source, o) - if err != nil { - return nil, fmt.Errorf( - "Error parsing source for %s[%s]: %s", - t.Key, - k, - err) - } - } - - result = append(result, &Module{ - Name: k, - Type: t.Key, - Source: source, - RawConfig: rawConfig, - }) } + + result = append(result, &Module{ + Name: k, + Source: source, + RawConfig: rawConfig, + }) } return result, nil diff --git a/config/loader_test.go b/config/loader_test.go index e4843e5c6..3dad74940 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -663,7 +663,7 @@ foo ` const modulesModulesStr = ` -foo.bar +module.bar source = baz memory ` diff --git a/config/test-fixtures/modules.tf b/config/test-fixtures/modules.tf index 48e7ad09e..dc1fb6a79 100644 --- a/config/test-fixtures/modules.tf +++ b/config/test-fixtures/modules.tf @@ -1,4 +1,4 @@ -module "foo" "bar" { +module "bar" { memory = "1G" source = "baz" } From 799ffbb3acb04c82fb994bae8c160adaefaa7c25 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 14 Sep 2014 14:46:45 -0700 Subject: [PATCH 10/41] config/module: tree.Modules() --- config/module/module_test.go | 11 +++++++++++ config/module/test-fixtures/basic/main.tf | 4 ++++ config/module/tree.go | 18 +++++++++++++++++- config/module/tree_test.go | 19 +++++++++++++++++++ 4 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 config/module/tree_test.go diff --git a/config/module/module_test.go b/config/module/module_test.go index 8360a8bf2..2ce7229de 100644 --- a/config/module/module_test.go +++ b/config/module/module_test.go @@ -6,6 +6,8 @@ import ( "os" "path/filepath" "testing" + + "github.com/hashicorp/terraform/config" ) const fixtureDir = "./test-fixtures" @@ -22,6 +24,15 @@ func tempDir(t *testing.T) string { return dir } +func testConfig(t *testing.T, n string) *config.Config { + c, err := config.LoadDir(filepath.Join(fixtureDir, n)) + if err != nil { + t.Fatalf("err: %s", err) + } + + return c +} + func testModule(n string) string { p := filepath.Join(fixtureDir, n) p, err := filepath.Abs(p) diff --git a/config/module/test-fixtures/basic/main.tf b/config/module/test-fixtures/basic/main.tf index fec56017d..383063715 100644 --- a/config/module/test-fixtures/basic/main.tf +++ b/config/module/test-fixtures/basic/main.tf @@ -1 +1,5 @@ # Hello + +module "foo" { + source = "./foo" +} diff --git a/config/module/tree.go b/config/module/tree.go index 851057c0f..9a64a7dc8 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -33,6 +33,11 @@ const ( GetModeUpdate ) +// NewTree returns a new Tree for the given config structure. +func NewTree(c *config.Config) *Tree { + return &Tree{Config: c} +} + // Flatten takes the entire module tree and flattens it into a single // namespace in *config.Config with no module imports. // @@ -45,8 +50,19 @@ func (t *Tree) Flatten() (*config.Config, error) { } // Modules returns the list of modules that this tree imports. +// +// This is only the imports of _this_ level of the tree. To retrieve the +// full nested imports, you'll have to traverse the tree. func (t *Tree) Modules() []*Module { - return nil + result := make([]*Module, len(t.Config.Modules)) + for i, m := range t.Config.Modules { + result[i] = &Module{ + Name: m.Name, + Source: m.Source, + } + } + + return result } // Load loads the configuration of the entire tree. diff --git a/config/module/tree_test.go b/config/module/tree_test.go new file mode 100644 index 000000000..27fd1f06d --- /dev/null +++ b/config/module/tree_test.go @@ -0,0 +1,19 @@ +package module + +import ( + "reflect" + "testing" +) + +func TestTree(t *testing.T) { + tree := NewTree(testConfig(t, "basic")) + actual := tree.Modules() + + expected := []*Module{ + &Module{Name: "foo", Source: "./foo"}, + } + + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("bad: %#v", actual) + } +} From a35a9262d4b4c5a8fe2fa01e3712f2b8f0e7fef8 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 14 Sep 2014 16:17:29 -0700 Subject: [PATCH 11/41] config/module: detectors, some more work on Tree --- config/module/detect.go | 50 +++++++++++++++++++ config/module/detect_file.go | 28 +++++++++++ config/module/detect_file_test.go | 32 +++++++++++++ config/module/module.go | 1 + config/module/module_test.go | 4 ++ config/module/tree.go | 79 ++++++++++++++++++++++++++++--- config/module/tree_test.go | 9 +++- 7 files changed, 196 insertions(+), 7 deletions(-) create mode 100644 config/module/detect.go create mode 100644 config/module/detect_file.go create mode 100644 config/module/detect_file_test.go diff --git a/config/module/detect.go b/config/module/detect.go new file mode 100644 index 000000000..855b34991 --- /dev/null +++ b/config/module/detect.go @@ -0,0 +1,50 @@ +package module + +import ( + "fmt" + "net/url" +) + +// Detector defines the interface that an invalid URL or a URL with a blank +// scheme is passed through in order to determine if its shorthand for +// something else well-known. +type Detector interface { + // Detect will detect whether the string matches a known pattern to + // turn it into a proper URL. + Detect(string, string) (string, bool, error) +} + +// Detectors is the list of detectors that are tried on an invalid URL. +// This is also the order they're tried (index 0 is first). +var Detectors []Detector + +func init() { + Detectors = []Detector{ + new(FileDetector), + } +} + +// Detect turns a source string into another source string if it is +// detected to be of a known pattern. +// +// This is safe to be called with an already valid source string: Detect +// will just return it. +func Detect(src string, pwd string) (string, error) { + u, err := url.Parse(src) + if err == nil && u.Scheme != "" { + // Valid URL + return src, nil + } + + for _, d := range Detectors { + result, ok, err := d.Detect(src, pwd) + if err != nil { + return "", err + } + if ok { + return result, nil + } + } + + return "", fmt.Errorf("invalid source string: %s", src) +} diff --git a/config/module/detect_file.go b/config/module/detect_file.go new file mode 100644 index 000000000..7f1632faa --- /dev/null +++ b/config/module/detect_file.go @@ -0,0 +1,28 @@ +package module + +import ( + "fmt" + "path/filepath" +) + +// FileDetector implements Detector to detect file paths. +type FileDetector struct{} + +func (d *FileDetector) Detect(src, pwd string) (string, bool, error) { + if len(src) == 0 { + return "", false, nil + } + + // Make sure we're using "/" even on Windows. URLs are "/"-based. + src = filepath.ToSlash(src) + if !filepath.IsAbs(src) { + src = filepath.Join(pwd, src) + } + + // Make sure that we don't start with "/" since we add that below + if src[0] == '/' { + src = src[1:] + } + + return fmt.Sprintf("file:///%s", src), true, nil +} diff --git a/config/module/detect_file_test.go b/config/module/detect_file_test.go new file mode 100644 index 000000000..0f76bc05b --- /dev/null +++ b/config/module/detect_file_test.go @@ -0,0 +1,32 @@ +package module + +import ( + "testing" +) + +func TestFileDetector(t *testing.T) { + cases := []struct { + Input string + Output string + }{ + {"./foo", "file:///pwd/foo"}, + {"foo", "file:///pwd/foo"}, + {"/foo", "file:///foo"}, + } + + pwd := "/pwd" + f := new(FileDetector) + for i, tc := range cases { + output, ok, err := f.Detect(tc.Input, pwd) + if err != nil { + t.Fatalf("err: %s", err) + } + if !ok { + t.Fatal("not ok") + } + + if output != tc.Output { + t.Fatalf("%d: bad: %#v", i, output) + } + } +} diff --git a/config/module/module.go b/config/module/module.go index f8649f6e9..6d01ed85d 100644 --- a/config/module/module.go +++ b/config/module/module.go @@ -4,4 +4,5 @@ package module type Module struct { Name string Source string + Dir string } diff --git a/config/module/module_test.go b/config/module/module_test.go index 2ce7229de..41f15e315 100644 --- a/config/module/module_test.go +++ b/config/module/module_test.go @@ -45,3 +45,7 @@ func testModule(n string) string { url.Path = p return url.String() } + +func testStorage(t *testing.T) Storage { + return &FolderStorage{StorageDir: tempDir(t)} +} diff --git a/config/module/tree.go b/config/module/tree.go index 9a64a7dc8..8cb5b967a 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -1,6 +1,9 @@ package module import ( + "fmt" + "sync" + "github.com/hashicorp/terraform/config" ) @@ -10,8 +13,9 @@ import ( // all the modules without getting, flatten the tree into something // Terraform can use, etc. type Tree struct { - Config *config.Config - Children []*Tree + config *config.Config + children []*Tree + lock sync.Mutex } // GetMode is an enum that describes how modules are loaded. @@ -35,7 +39,15 @@ const ( // NewTree returns a new Tree for the given config structure. func NewTree(c *config.Config) *Tree { - return &Tree{Config: c} + return &Tree{config: c} +} + +// Children returns the children of this tree (the modules that are +// imported by this root). +// +// This will only return a non-nil value after Load is called. +func (t *Tree) Children() []*Tree { + return nil } // Flatten takes the entire module tree and flattens it into a single @@ -54,10 +66,10 @@ func (t *Tree) Flatten() (*config.Config, error) { // This is only the imports of _this_ level of the tree. To retrieve the // full nested imports, you'll have to traverse the tree. func (t *Tree) Modules() []*Module { - result := make([]*Module, len(t.Config.Modules)) - for i, m := range t.Config.Modules { + result := make([]*Module, len(t.config.Modules)) + for i, m := range t.config.Modules { result[i] = &Module{ - Name: m.Name, + Name: m.Name, Source: m.Source, } } @@ -70,11 +82,66 @@ func (t *Tree) Modules() []*Module { // The parameters are used to tell the tree where to find modules and // whether it can download/update modules along the way. // +// Calling this multiple times will reload the tree. +// // Various semantic-like checks are made along the way of loading since // module trees inherently require the configuration to be in a reasonably // sane state: no circular dependencies, proper module sources, etc. A full // suite of validations can be done by running Validate (after loading). func (t *Tree) Load(s Storage, mode GetMode) error { + t.lock.Lock() + defer t.lock.Unlock() + + // Reset the children if we have any + t.children = nil + + modules := t.Modules() + children := make([]*Tree, len(modules)) + + // Go through all the modules and get the directory for them. + update := mode == GetModeUpdate + for i, m := range modules { + source, err := Detect(m.Source, m.Dir) + if err != nil { + return fmt.Errorf("module %s: %s", m.Name, err) + } + + if mode > GetModeNone { + // Get the module since we specified we should + if err := s.Get(source, update); err != nil { + return err + } + } + + // Get the directory where this module is so we can load it + dir, ok, err := s.Dir(source) + if err != nil { + return err + } + if !ok { + return fmt.Errorf( + "module %s: not found, may need to be downloaded", m.Name) + } + + // Load the configuration + c, err := config.LoadDir(dir) + if err != nil { + return fmt.Errorf( + "module %s: %s", m.Name, err) + } + children[i] = NewTree(c) + } + + // Go through all the children and load them. + for _, c := range children { + if err := c.Load(s, mode); err != nil { + return err + } + } + + // Set our tree up + t.children = children + return nil } diff --git a/config/module/tree_test.go b/config/module/tree_test.go index 27fd1f06d..fbb5e73fe 100644 --- a/config/module/tree_test.go +++ b/config/module/tree_test.go @@ -5,7 +5,14 @@ import ( "testing" ) -func TestTree(t *testing.T) { +func TestTree_Load(t *testing.T) { + tree := NewTree(testConfig(t, "basic")) + if err := tree.Load(testStorage(t), GetModeGet); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestTree_Modules(t *testing.T) { tree := NewTree(testConfig(t, "basic")) actual := tree.Modules() From 6eee9fbcb3d684e0bc64934d6025fda22ef822c4 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 14 Sep 2014 19:28:18 -0700 Subject: [PATCH 12/41] config/module: file paths require pwd --- config/module/detect_file.go | 5 +++++ config/module/detect_file_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/config/module/detect_file.go b/config/module/detect_file.go index 7f1632faa..ac9ad8e67 100644 --- a/config/module/detect_file.go +++ b/config/module/detect_file.go @@ -16,6 +16,11 @@ func (d *FileDetector) Detect(src, pwd string) (string, bool, error) { // Make sure we're using "/" even on Windows. URLs are "/"-based. src = filepath.ToSlash(src) if !filepath.IsAbs(src) { + if pwd == "" { + return "", true, fmt.Errorf( + "relative paths require a module with a pwd") + } + src = filepath.Join(pwd, src) } diff --git a/config/module/detect_file_test.go b/config/module/detect_file_test.go index 0f76bc05b..6ccd49f4f 100644 --- a/config/module/detect_file_test.go +++ b/config/module/detect_file_test.go @@ -30,3 +30,31 @@ func TestFileDetector(t *testing.T) { } } } + +func TestFileDetector_noPwd(t *testing.T) { + cases := []struct { + Input string + Output string + Err bool + }{ + {"./foo", "", true}, + {"foo", "", true}, + {"/foo", "file:///foo", false}, + } + + pwd := "" + f := new(FileDetector) + for i, tc := range cases { + output, ok, err := f.Detect(tc.Input, pwd) + if (err != nil) != tc.Err { + t.Fatalf("%d: err: %s", i, err) + } + if !ok { + t.Fatal("not ok") + } + + if output != tc.Output { + t.Fatalf("%d: bad: %#v", i, output) + } + } +} From f8836290da2d8b9ac744483b8ec488ee8ed5a785 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 14 Sep 2014 19:35:38 -0700 Subject: [PATCH 13/41] config: not directory that config was loaded from --- config/config.go | 5 +++++ config/loader.go | 3 +++ config/loader_test.go | 23 ++++++++++++++++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 60cb8818a..431807c06 100644 --- a/config/config.go +++ b/config/config.go @@ -15,6 +15,11 @@ import ( // Config is the configuration that comes from loading a collection // of Terraform templates. type Config struct { + // Dir is the path to the directory where this configuration was + // loaded from. If it is blank, this configuration wasn't loaded from + // any meaningful directory. + Dir string + Modules []*Module ProviderConfigs []*ProviderConfig Resources []*Resource diff --git a/config/loader.go b/config/loader.go index 7b3112ad6..18d5856d6 100644 --- a/config/loader.go +++ b/config/loader.go @@ -140,6 +140,9 @@ func LoadDir(root string) (*Config, error) { } } + // Mark the directory + result.Dir = root + return result, nil } diff --git a/config/loader_test.go b/config/loader_test.go index 3dad74940..50e792ea0 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -25,6 +25,10 @@ func TestLoadBasic(t *testing.T) { t.Fatal("config should not be nil") } + if c.Dir != "" { + t.Fatalf("bad: %#v", c.Dir) + } + actual := variablesStr(c.Variables) if actual != strings.TrimSpace(basicVariablesStr) { t.Fatalf("bad:\n%s", actual) @@ -85,6 +89,10 @@ func TestLoadBasic_json(t *testing.T) { t.Fatal("config should not be nil") } + if c.Dir != "" { + t.Fatalf("bad: %#v", c.Dir) + } + actual := variablesStr(c.Variables) if actual != strings.TrimSpace(basicVariablesStr) { t.Fatalf("bad:\n%s", actual) @@ -116,6 +124,10 @@ func TestLoadBasic_modules(t *testing.T) { t.Fatal("config should not be nil") } + if c.Dir != "" { + t.Fatalf("bad: %#v", c.Dir) + } + actual := modulesStr(c.Modules) if actual != strings.TrimSpace(modulesModulesStr) { t.Fatalf("bad:\n%s", actual) @@ -131,6 +143,10 @@ func TestLoad_variables(t *testing.T) { t.Fatal("config should not be nil") } + if c.Dir != "" { + t.Fatalf("bad: %#v", c.Dir) + } + actual := variablesStr(c.Variables) if actual != strings.TrimSpace(variablesVariablesStr) { t.Fatalf("bad:\n%s", actual) @@ -138,7 +154,8 @@ func TestLoad_variables(t *testing.T) { } func TestLoadDir_basic(t *testing.T) { - c, err := LoadDir(filepath.Join(fixtureDir, "dir-basic")) + dir := filepath.Join(fixtureDir, "dir-basic") + c, err := LoadDir(dir) if err != nil { t.Fatalf("err: %s", err) } @@ -147,6 +164,10 @@ func TestLoadDir_basic(t *testing.T) { t.Fatal("config should not be nil") } + if c.Dir != dir { + t.Fatalf("bad: %#v", c.Dir) + } + actual := variablesStr(c.Variables) if actual != strings.TrimSpace(dirBasicVariablesStr) { t.Fatalf("bad:\n%s", actual) From e96fe43814d7af1587da65122951b9386cf1cbd2 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 14 Sep 2014 19:55:38 -0700 Subject: [PATCH 14/41] config: dir on Config should be an absolute path --- config/loader.go | 8 +++++++- config/loader_test.go | 6 +++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/config/loader.go b/config/loader.go index 18d5856d6..5cc27e998 100644 --- a/config/loader.go +++ b/config/loader.go @@ -104,6 +104,12 @@ func LoadDir(root string) (*Config, error) { root) } + // Determine the absolute path to the directory. + rootAbs, err := filepath.Abs(root) + if err != nil { + return nil, err + } + var result *Config // Sort the files and overrides so we have a deterministic order @@ -141,7 +147,7 @@ func LoadDir(root string) (*Config, error) { } // Mark the directory - result.Dir = root + result.Dir = rootAbs return result, nil } diff --git a/config/loader_test.go b/config/loader_test.go index 50e792ea0..320d1ea06 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -164,7 +164,11 @@ func TestLoadDir_basic(t *testing.T) { t.Fatal("config should not be nil") } - if c.Dir != dir { + dirAbs, err := filepath.Abs(dir) + if err != nil { + t.Fatalf("err: %s", err) + } + if c.Dir != dirAbs { t.Fatalf("bad: %#v", c.Dir) } From 85d1e406444edf070ff6468de9c628ab5888de88 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 14 Sep 2014 20:00:17 -0700 Subject: [PATCH 15/41] config/module: can load a tree properly --- config/module/module.go | 1 - config/module/test-fixtures/basic/foo/main.tf | 1 + config/module/tree.go | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 config/module/test-fixtures/basic/foo/main.tf diff --git a/config/module/module.go b/config/module/module.go index 6d01ed85d..f8649f6e9 100644 --- a/config/module/module.go +++ b/config/module/module.go @@ -4,5 +4,4 @@ package module type Module struct { Name string Source string - Dir string } diff --git a/config/module/test-fixtures/basic/foo/main.tf b/config/module/test-fixtures/basic/foo/main.tf new file mode 100644 index 000000000..fec56017d --- /dev/null +++ b/config/module/test-fixtures/basic/foo/main.tf @@ -0,0 +1 @@ +# Hello diff --git a/config/module/tree.go b/config/module/tree.go index 8cb5b967a..639a925b5 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -69,7 +69,7 @@ func (t *Tree) Modules() []*Module { result := make([]*Module, len(t.config.Modules)) for i, m := range t.config.Modules { result[i] = &Module{ - Name: m.Name, + Name: m.Name, Source: m.Source, } } @@ -101,7 +101,7 @@ func (t *Tree) Load(s Storage, mode GetMode) error { // Go through all the modules and get the directory for them. update := mode == GetModeUpdate for i, m := range modules { - source, err := Detect(m.Source, m.Dir) + source, err := Detect(m.Source, t.config.Dir) if err != nil { return fmt.Errorf("module %s: %s", m.Name, err) } From 30b76ef8202a6fe3e47f8b47012533dc1d4915d9 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 14 Sep 2014 20:14:37 -0700 Subject: [PATCH 16/41] config/module: tree.String() --- config/module/tree.go | 46 ++++++++++++++++++++++++++++++++++++-- config/module/tree_test.go | 36 ++++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/config/module/tree.go b/config/module/tree.go index 639a925b5..d08b047bd 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -1,7 +1,10 @@ package module import ( + "bufio" + "bytes" "fmt" + "strings" "sync" "github.com/hashicorp/terraform/config" @@ -13,9 +16,10 @@ import ( // all the modules without getting, flatten the tree into something // Terraform can use, etc. type Tree struct { + name string config *config.Config children []*Tree - lock sync.Mutex + lock sync.RWMutex } // GetMode is an enum that describes how modules are loaded. @@ -47,7 +51,9 @@ func NewTree(c *config.Config) *Tree { // // This will only return a non-nil value after Load is called. func (t *Tree) Children() []*Tree { - return nil + t.lock.RLock() + defer t.lock.RUnlock() + return t.children } // Flatten takes the entire module tree and flattens it into a single @@ -77,6 +83,16 @@ func (t *Tree) Modules() []*Module { return result } +// Name returns the name of the tree. This will be "" for the root +// tree and then the module name given for any children. +func (t *Tree) Name() string { + if t.name == "" { + return "" + } + + return t.name +} + // Load loads the configuration of the entire tree. // // The parameters are used to tell the tree where to find modules and @@ -130,6 +146,7 @@ func (t *Tree) Load(s Storage, mode GetMode) error { "module %s: %s", m.Name, err) } children[i] = NewTree(c) + children[i].name = m.Name } // Go through all the children and load them. @@ -145,6 +162,31 @@ func (t *Tree) Load(s Storage, mode GetMode) error { return nil } +// String gives a nice output to describe the tree. +func (t *Tree) String() string { + var result bytes.Buffer + result.WriteString(t.Name() + "\n") + + cs := t.Children() + if cs == nil { + result.WriteString(" not loaded") + } else { + // Go through each child and get its string value, then indent it + // by two. + for _, c := range cs { + r := strings.NewReader(c.String()) + scanner := bufio.NewScanner(r) + for scanner.Scan() { + result.WriteString(" ") + result.WriteString(scanner.Text()) + result.WriteString("\n") + } + } + } + + return result.String() +} + // Validate does semantic checks on the entire tree of configurations. // // This will call the respective config.Config.Validate() functions as well diff --git a/config/module/tree_test.go b/config/module/tree_test.go index fbb5e73fe..fe9e09604 100644 --- a/config/module/tree_test.go +++ b/config/module/tree_test.go @@ -2,14 +2,34 @@ package module import ( "reflect" + "strings" "testing" ) func TestTree_Load(t *testing.T) { + storage := testStorage(t) tree := NewTree(testConfig(t, "basic")) - if err := tree.Load(testStorage(t), GetModeGet); err != nil { + + // This should error because we haven't gotten things yet + if err := tree.Load(storage, GetModeNone); err == nil { + t.Fatal("should error") + } + + // This should get things + if err := tree.Load(storage, GetModeGet); err != nil { t.Fatalf("err: %s", err) } + + // This should no longer error + if err := tree.Load(storage, GetModeNone); err != nil { + t.Fatalf("err: %s", err) + } + + actual := strings.TrimSpace(tree.String()) + expected := strings.TrimSpace(treeLoadStr) + if actual != expected { + t.Fatalf("bad: \n\n%s", actual) + } } func TestTree_Modules(t *testing.T) { @@ -24,3 +44,17 @@ func TestTree_Modules(t *testing.T) { t.Fatalf("bad: %#v", actual) } } + +func TestTree_Name(t *testing.T) { + tree := NewTree(testConfig(t, "basic")) + actual := tree.Name() + + if actual != "" { + t.Fatalf("bad: %#v", actual) + } +} + +const treeLoadStr = ` + + foo +` From c9fd910c41dc1eae4f6c8013577bb7bc256eb7bc Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Sep 2014 09:37:40 -0700 Subject: [PATCH 17/41] config/module: Validate --- .../validate-child-bad/child/main.tf | 3 + .../test-fixtures/validate-child-bad/main.tf | 3 + .../validate-child-good/child/main.tf | 1 + .../test-fixtures/validate-child-good/main.tf | 3 + .../test-fixtures/validate-root-bad/main.tf | 3 + config/module/tree.go | 59 ++++++++++++++++++ config/module/tree_test.go | 62 ++++++++++++++++++- 7 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 config/module/test-fixtures/validate-child-bad/child/main.tf create mode 100644 config/module/test-fixtures/validate-child-bad/main.tf create mode 100644 config/module/test-fixtures/validate-child-good/child/main.tf create mode 100644 config/module/test-fixtures/validate-child-good/main.tf create mode 100644 config/module/test-fixtures/validate-root-bad/main.tf diff --git a/config/module/test-fixtures/validate-child-bad/child/main.tf b/config/module/test-fixtures/validate-child-bad/child/main.tf new file mode 100644 index 000000000..93b365403 --- /dev/null +++ b/config/module/test-fixtures/validate-child-bad/child/main.tf @@ -0,0 +1,3 @@ +# Duplicate resources +resource "aws_instance" "foo" {} +resource "aws_instance" "foo" {} diff --git a/config/module/test-fixtures/validate-child-bad/main.tf b/config/module/test-fixtures/validate-child-bad/main.tf new file mode 100644 index 000000000..813f7ef8e --- /dev/null +++ b/config/module/test-fixtures/validate-child-bad/main.tf @@ -0,0 +1,3 @@ +module "foo" { + source = "./child" +} diff --git a/config/module/test-fixtures/validate-child-good/child/main.tf b/config/module/test-fixtures/validate-child-good/child/main.tf new file mode 100644 index 000000000..dd0d95b59 --- /dev/null +++ b/config/module/test-fixtures/validate-child-good/child/main.tf @@ -0,0 +1 @@ +# Good diff --git a/config/module/test-fixtures/validate-child-good/main.tf b/config/module/test-fixtures/validate-child-good/main.tf new file mode 100644 index 000000000..0f6991c53 --- /dev/null +++ b/config/module/test-fixtures/validate-child-good/main.tf @@ -0,0 +1,3 @@ +module "child" { + source = "./child" +} diff --git a/config/module/test-fixtures/validate-root-bad/main.tf b/config/module/test-fixtures/validate-root-bad/main.tf new file mode 100644 index 000000000..93b365403 --- /dev/null +++ b/config/module/test-fixtures/validate-root-bad/main.tf @@ -0,0 +1,3 @@ +# Duplicate resources +resource "aws_instance" "foo" {} +resource "aws_instance" "foo" {} diff --git a/config/module/tree.go b/config/module/tree.go index d08b047bd..2ac3f8b18 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -67,6 +67,13 @@ func (t *Tree) Flatten() (*config.Config, error) { return nil, nil } +// Loaded says whether or not this tree has been loaded or not yet. +func (t *Tree) Loaded() bool { + t.lock.RLock() + defer t.lock.RUnlock() + return t.children != nil +} + // Modules returns the list of modules that this tree imports. // // This is only the imports of _this_ level of the tree. To retrieve the @@ -191,6 +198,58 @@ func (t *Tree) String() string { // // This will call the respective config.Config.Validate() functions as well // as verifying things such as parameters/outputs between the various modules. +// +// Load must be called prior to calling Validate or an error will be returned. func (t *Tree) Validate() error { + if !t.Loaded() { + return fmt.Errorf("tree must be loaded before calling Validate") + } + + // Validate our configuration first. + if err := t.config.Validate(); err != nil { + return &ValidateError{ + Name: []string{t.Name()}, + Err: err, + } + } + + // Validate all our children + for _, c := range t.Children() { + err := c.Validate() + if err == nil { + continue + } + + verr, ok := err.(*ValidateError) + if !ok { + // Unknown error, just return... + return err + } + + // Append ourselves to the error and then return + verr.Name = append(verr.Name, t.Name()) + return verr + } + return nil } + +// ValidateError is an error returned by Tree.Validate if an error occurs +// with validation. +type ValidateError struct { + Name []string + Err error +} + +func (e *ValidateError) Error() string { + // Build up the name + var buf bytes.Buffer + for _, n := range e.Name { + buf.WriteString(n) + buf.WriteString(".") + } + buf.Truncate(buf.Len()-1) + + // Format the value + return fmt.Sprintf("module %s: %s", buf.String(), e.Err) +} diff --git a/config/module/tree_test.go b/config/module/tree_test.go index fe9e09604..8795e16b9 100644 --- a/config/module/tree_test.go +++ b/config/module/tree_test.go @@ -6,20 +6,32 @@ import ( "testing" ) -func TestTree_Load(t *testing.T) { +func TestTreeLoad(t *testing.T) { storage := testStorage(t) tree := NewTree(testConfig(t, "basic")) + if tree.Loaded() { + t.Fatal("should not be loaded") + } + // This should error because we haven't gotten things yet if err := tree.Load(storage, GetModeNone); err == nil { t.Fatal("should error") } + if tree.Loaded() { + t.Fatal("should not be loaded") + } + // This should get things if err := tree.Load(storage, GetModeGet); err != nil { t.Fatalf("err: %s", err) } + if !tree.Loaded() { + t.Fatal("should be loaded") + } + // This should no longer error if err := tree.Load(storage, GetModeNone); err != nil { t.Fatalf("err: %s", err) @@ -32,7 +44,7 @@ func TestTree_Load(t *testing.T) { } } -func TestTree_Modules(t *testing.T) { +func TestTreeModules(t *testing.T) { tree := NewTree(testConfig(t, "basic")) actual := tree.Modules() @@ -45,7 +57,7 @@ func TestTree_Modules(t *testing.T) { } } -func TestTree_Name(t *testing.T) { +func TestTreeName(t *testing.T) { tree := NewTree(testConfig(t, "basic")) actual := tree.Name() @@ -54,6 +66,50 @@ func TestTree_Name(t *testing.T) { } } +func TestTreeValidate_badChild(t *testing.T) { + tree := NewTree(testConfig(t, "validate-child-bad")) + + if err := tree.Load(testStorage(t), GetModeGet); err != nil { + t.Fatalf("err: %s", err) + } + + if err := tree.Validate(); err == nil { + t.Fatal("should error") + } +} + +func TestTreeValidate_badRoot(t *testing.T) { + tree := NewTree(testConfig(t, "validate-root-bad")) + + if err := tree.Load(testStorage(t), GetModeGet); err != nil { + t.Fatalf("err: %s", err) + } + + if err := tree.Validate(); err == nil { + t.Fatal("should error") + } +} + +func TestTreeValidate_good(t *testing.T) { + tree := NewTree(testConfig(t, "validate-child-good")) + + if err := tree.Load(testStorage(t), GetModeGet); err != nil { + t.Fatalf("err: %s", err) + } + + if err := tree.Validate(); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestTreeValidate_notLoaded(t *testing.T) { + tree := NewTree(testConfig(t, "basic")) + + if err := tree.Validate(); err == nil { + t.Fatal("should error") + } +} + const treeLoadStr = ` foo From 4fdb6d1b52fca72e4a4d5207220e0edd7c58a87b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Sep 2014 09:41:00 -0700 Subject: [PATCH 18/41] config: add test for empty file --- config/loader_test.go | 11 +++++++++++ config/test-fixtures/empty.tf | 0 2 files changed, 11 insertions(+) create mode 100644 config/test-fixtures/empty.tf diff --git a/config/loader_test.go b/config/loader_test.go index 320d1ea06..a07d506a8 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -50,6 +50,17 @@ func TestLoadBasic(t *testing.T) { } } +func TestLoadBasic_empty(t *testing.T) { + c, err := Load(filepath.Join(fixtureDir, "empty.tf")) + if err != nil { + t.Fatalf("err: %s", err) + } + + if c == nil { + t.Fatal("config should not be nil") + } +} + func TestLoadBasic_import(t *testing.T) { // Skip because we disabled importing t.Skip() diff --git a/config/test-fixtures/empty.tf b/config/test-fixtures/empty.tf new file mode 100644 index 000000000..e69de29bb From 2419bf79f23d6181da05f7a60fb49c766e7c52a4 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Sep 2014 09:53:29 -0700 Subject: [PATCH 19/41] config/module: error if duplicate module --- config/module/test-fixtures/dup/foo/main.tf | 0 config/module/test-fixtures/dup/main.tf | 7 +++++++ config/module/tree.go | 20 +++++++++++++------- config/module/tree_test.go | 14 ++++++++++++++ 4 files changed, 34 insertions(+), 7 deletions(-) create mode 100644 config/module/test-fixtures/dup/foo/main.tf create mode 100644 config/module/test-fixtures/dup/main.tf diff --git a/config/module/test-fixtures/dup/foo/main.tf b/config/module/test-fixtures/dup/foo/main.tf new file mode 100644 index 000000000..e69de29bb diff --git a/config/module/test-fixtures/dup/main.tf b/config/module/test-fixtures/dup/main.tf new file mode 100644 index 000000000..98efd6e4f --- /dev/null +++ b/config/module/test-fixtures/dup/main.tf @@ -0,0 +1,7 @@ +module "foo" { + source = "./foo" +} + +module "foo" { + source = "./foo" +} diff --git a/config/module/tree.go b/config/module/tree.go index 2ac3f8b18..019f5c1c0 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -18,7 +18,7 @@ import ( type Tree struct { name string config *config.Config - children []*Tree + children map[string]*Tree lock sync.RWMutex } @@ -50,7 +50,7 @@ func NewTree(c *config.Config) *Tree { // imported by this root). // // This will only return a non-nil value after Load is called. -func (t *Tree) Children() []*Tree { +func (t *Tree) Children() map[string]*Tree { t.lock.RLock() defer t.lock.RUnlock() return t.children @@ -119,11 +119,16 @@ func (t *Tree) Load(s Storage, mode GetMode) error { t.children = nil modules := t.Modules() - children := make([]*Tree, len(modules)) + children := make(map[string]*Tree) // Go through all the modules and get the directory for them. update := mode == GetModeUpdate - for i, m := range modules { + for _, m := range modules { + if _, ok := children[m.Name]; ok { + return fmt.Errorf( + "module %s: duplicated. module names must be unique", m.Name) + } + source, err := Detect(m.Source, t.config.Dir) if err != nil { return fmt.Errorf("module %s: %s", m.Name, err) @@ -152,8 +157,9 @@ func (t *Tree) Load(s Storage, mode GetMode) error { return fmt.Errorf( "module %s: %s", m.Name, err) } - children[i] = NewTree(c) - children[i].name = m.Name + + children[m.Name] = NewTree(c) + children[m.Name].name = m.Name } // Go through all the children and load them. @@ -248,7 +254,7 @@ func (e *ValidateError) Error() string { buf.WriteString(n) buf.WriteString(".") } - buf.Truncate(buf.Len()-1) + buf.Truncate(buf.Len() - 1) // Format the value return fmt.Sprintf("module %s: %s", buf.String(), e.Err) diff --git a/config/module/tree_test.go b/config/module/tree_test.go index 8795e16b9..ed74eb66c 100644 --- a/config/module/tree_test.go +++ b/config/module/tree_test.go @@ -44,6 +44,20 @@ func TestTreeLoad(t *testing.T) { } } +func TestTreeLoad_duplicate(t *testing.T) { + storage := testStorage(t) + tree := NewTree(testConfig(t, "dup")) + + if tree.Loaded() { + t.Fatal("should not be loaded") + } + + // This should get things + if err := tree.Load(storage, GetModeGet); err == nil { + t.Fatalf("should error") + } +} + func TestTreeModules(t *testing.T) { tree := NewTree(testConfig(t, "basic")) actual := tree.Modules() From 12e7c75211410abbf00f2f858756860928c8f860 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Sep 2014 10:32:41 -0700 Subject: [PATCH 20/41] config/module: validate that parameters into modules valid --- .../validate-bad-var/child/main.tf | 0 .../test-fixtures/validate-bad-var/main.tf | 5 +++ .../validate-child-good/child/main.tf | 2 +- .../test-fixtures/validate-child-good/main.tf | 1 + config/module/tree.go | 40 ++++++++++++++++--- config/module/tree_test.go | 12 ++++++ 6 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 config/module/test-fixtures/validate-bad-var/child/main.tf create mode 100644 config/module/test-fixtures/validate-bad-var/main.tf diff --git a/config/module/test-fixtures/validate-bad-var/child/main.tf b/config/module/test-fixtures/validate-bad-var/child/main.tf new file mode 100644 index 000000000..e69de29bb diff --git a/config/module/test-fixtures/validate-bad-var/main.tf b/config/module/test-fixtures/validate-bad-var/main.tf new file mode 100644 index 000000000..7cc785d17 --- /dev/null +++ b/config/module/test-fixtures/validate-bad-var/main.tf @@ -0,0 +1,5 @@ +module "child" { + source = "./child" + + memory = "foo" +} diff --git a/config/module/test-fixtures/validate-child-good/child/main.tf b/config/module/test-fixtures/validate-child-good/child/main.tf index dd0d95b59..618ae3c42 100644 --- a/config/module/test-fixtures/validate-child-good/child/main.tf +++ b/config/module/test-fixtures/validate-child-good/child/main.tf @@ -1 +1 @@ -# Good +variable "memory" {} diff --git a/config/module/test-fixtures/validate-child-good/main.tf b/config/module/test-fixtures/validate-child-good/main.tf index 0f6991c53..7c70782f1 100644 --- a/config/module/test-fixtures/validate-child-good/main.tf +++ b/config/module/test-fixtures/validate-child-good/main.tf @@ -1,3 +1,4 @@ module "child" { source = "./child" + memory = "1G" } diff --git a/config/module/tree.go b/config/module/tree.go index 019f5c1c0..867f812c8 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -211,16 +211,20 @@ func (t *Tree) Validate() error { return fmt.Errorf("tree must be loaded before calling Validate") } + // If something goes wrong, here is our error template + newErr := &ValidateError{Name: []string{t.Name()}} + // Validate our configuration first. if err := t.config.Validate(); err != nil { - return &ValidateError{ - Name: []string{t.Name()}, - Err: err, - } + newErr.Err = err + return newErr } + // Get the child trees + children := t.Children() + // Validate all our children - for _, c := range t.Children() { + for _, c := range children { err := c.Validate() if err == nil { continue @@ -237,6 +241,32 @@ func (t *Tree) Validate() error { return verr } + // Go over all the modules and verify that any parameters are valid + // variables into the module in question. + for _, m := range t.config.Modules { + tree, ok := children[m.Name] + if !ok { + // This should never happen because Load watches us + panic("module not found in children: " + m.Name) + } + + // Build the variables that the module defines + varMap := make(map[string]struct{}) + for _, v := range tree.config.Variables { + varMap[v.Name] = struct{}{} + } + + // Compare to the keys in our raw config for the module + for k, _ := range m.RawConfig.Raw { + if _, ok := varMap[k]; !ok { + newErr.Err = fmt.Errorf( + "module %s: %s is not a valid parameter", + m.Name, k) + return newErr + } + } + } + return nil } diff --git a/config/module/tree_test.go b/config/module/tree_test.go index ed74eb66c..3fbad23d7 100644 --- a/config/module/tree_test.go +++ b/config/module/tree_test.go @@ -92,6 +92,18 @@ func TestTreeValidate_badChild(t *testing.T) { } } +func TestTreeValidate_badChildVar(t *testing.T) { + tree := NewTree(testConfig(t, "validate-bad-var")) + + if err := tree.Load(testStorage(t), GetModeGet); err != nil { + t.Fatalf("err: %s", err) + } + + if err := tree.Validate(); err == nil { + t.Fatal("should error") + } +} + func TestTreeValidate_badRoot(t *testing.T) { tree := NewTree(testConfig(t, "validate-root-bad")) From 46c140c7977385168deb5437b24124f30a964e17 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Sep 2014 11:40:25 -0700 Subject: [PATCH 21/41] config: can parse module variables --- config/expr_parse_test.go | 12 ++++++++++++ config/interpolate.go | 35 ++++++++++++++++++++++++++++++++--- config/interpolate_test.go | 9 +++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/config/expr_parse_test.go b/config/expr_parse_test.go index a8d34d553..da77c7bda 100644 --- a/config/expr_parse_test.go +++ b/config/expr_parse_test.go @@ -34,6 +34,18 @@ func TestExprParse(t *testing.T) { false, }, + { + "module.foo.bar", + &VariableInterpolation{ + Variable: &ModuleVariable{ + Name: "foo", + Field: "bar", + key: "module.foo.bar", + }, + }, + false, + }, + { "lookup(var.foo, var.bar)", &FunctionInterpolation{ diff --git a/config/interpolate.go b/config/interpolate.go index cffd58b17..b6d84d425 100644 --- a/config/interpolate.go +++ b/config/interpolate.go @@ -52,6 +52,14 @@ type VariableInterpolation struct { Variable InterpolatedVariable } +// A ModuleVariable is a variable that is referencing the output +// of a module, such as "${module.foo.bar}" +type ModuleVariable struct { + Name string + Field string + key string +} + // A ResourceVariable is a variable that is referencing the field // of a resource, such as "${aws_instance.foo.ami}" type ResourceVariable struct { @@ -76,11 +84,13 @@ type UserVariable struct { } func NewInterpolatedVariable(v string) (InterpolatedVariable, error) { - if !strings.HasPrefix(v, "var.") { + if strings.HasPrefix(v, "var.") { + return NewUserVariable(v) + } else if strings.HasPrefix(v, "module.") { + return NewModuleVariable(v) + } else { return NewResourceVariable(v) } - - return NewUserVariable(v) } func (i *FunctionInterpolation) Interpolate( @@ -142,6 +152,25 @@ func (i *VariableInterpolation) Variables() map[string]InterpolatedVariable { return map[string]InterpolatedVariable{i.Variable.FullKey(): i.Variable} } +func NewModuleVariable(key string) (*ModuleVariable, error) { + parts := strings.SplitN(key, ".", 3) + if len(parts) < 3 { + return nil, fmt.Errorf( + "%s: module variables must be three parts: module.name.attr", + key) + } + + return &ModuleVariable{ + Name: parts[1], + Field: parts[2], + key: key, + }, nil +} + +func (v *ModuleVariable) FullKey() string { + return v.key +} + func NewResourceVariable(key string) (*ResourceVariable, error) { parts := strings.SplitN(key, ".", 3) if len(parts) < 3 { diff --git a/config/interpolate_test.go b/config/interpolate_test.go index 92bb0f15c..61d188d22 100644 --- a/config/interpolate_test.go +++ b/config/interpolate_test.go @@ -20,6 +20,15 @@ func TestNewInterpolatedVariable(t *testing.T) { }, false, }, + { + "module.foo.bar", + &ModuleVariable{ + Name: "foo", + Field: "bar", + key: "module.foo.bar", + }, + false, + }, } for i, tc := range cases { From b60da29d486ce3b752a398bdc4b82c183367f250 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Sep 2014 11:45:41 -0700 Subject: [PATCH 22/41] config: validate that variables reference valid modules --- config/config.go | 20 ++++++++++++++++++- config/config_test.go | 14 +++++++++++++ config/loader_test.go | 2 +- .../validate-var-module-invalid/main.tf | 3 +++ .../test-fixtures/validate-var-module/main.tf | 5 +++++ 5 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 config/test-fixtures/validate-var-module-invalid/main.tf create mode 100644 config/test-fixtures/validate-var-module/main.tf diff --git a/config/config.go b/config/config.go index 431807c06..e0733081f 100644 --- a/config/config.go +++ b/config/config.go @@ -110,7 +110,7 @@ func ProviderConfigName(t string, pcs []*ProviderConfig) string { // A unique identifier for this module. func (r *Module) Id() string { - return fmt.Sprintf("module.%s", r.Name) + return fmt.Sprintf("%s", r.Name) } // A unique identifier for this resource. @@ -195,6 +195,24 @@ func (c *Config) Validate() error { } dupped = nil + // Check that all variables for modules reference modules that + // exist. + for source, vs := range vars { + for _, v := range vs { + mv, ok := v.(*ModuleVariable) + if !ok { + continue + } + + if _, ok := modules[mv.Name]; !ok { + errs = append(errs, fmt.Errorf( + "%s: unknown module referenced: %s", + source, + mv.Name)) + } + } + } + // Check that all references to resources are valid resources := make(map[string]*Resource) dupped = make(map[string]struct{}) diff --git a/config/config_test.go b/config/config_test.go index 581109e9c..26becdd30 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -114,6 +114,20 @@ func TestConfigValidate_varDefaultInterpolate(t *testing.T) { } } +func TestConfigValidate_varModule(t *testing.T) { + c := testConfig(t, "validate-var-module") + if err := c.Validate(); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestConfigValidate_varModuleInvalid(t *testing.T) { + c := testConfig(t, "validate-var-module-invalid") + if err := c.Validate(); err == nil { + t.Fatal("should not be valid") + } +} + func TestProviderConfigName(t *testing.T) { pcs := []*ProviderConfig{ &ProviderConfig{Name: "aw"}, diff --git a/config/loader_test.go b/config/loader_test.go index a07d506a8..b92307a70 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -699,7 +699,7 @@ foo ` const modulesModulesStr = ` -module.bar +bar source = baz memory ` diff --git a/config/test-fixtures/validate-var-module-invalid/main.tf b/config/test-fixtures/validate-var-module-invalid/main.tf new file mode 100644 index 000000000..73897c008 --- /dev/null +++ b/config/test-fixtures/validate-var-module-invalid/main.tf @@ -0,0 +1,3 @@ +resource "aws_instance" "foo" { + foo = "${module.foo.bar}" +} diff --git a/config/test-fixtures/validate-var-module/main.tf b/config/test-fixtures/validate-var-module/main.tf new file mode 100644 index 000000000..402c235a1 --- /dev/null +++ b/config/test-fixtures/validate-var-module/main.tf @@ -0,0 +1,5 @@ +module "foo" {} + +resource "aws_instance" "foo" { + foo = "${module.foo.bar}" +} From 292f57ea0a0ffa85cedd9afdf7ba4c08ed23a89d Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Sep 2014 13:57:07 -0700 Subject: [PATCH 23/41] config/module: validate outputs line up with ModuleVariables --- config/config.go | 6 ++-- .../validate-bad-output/child/main.tf | 0 .../test-fixtures/validate-bad-output/main.tf | 7 +++++ .../validate-child-good/child/main.tf | 2 ++ .../test-fixtures/validate-child-good/main.tf | 4 +++ config/module/tree.go | 31 +++++++++++++++++++ config/module/tree_test.go | 12 +++++++ 7 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 config/module/test-fixtures/validate-bad-output/child/main.tf create mode 100644 config/module/test-fixtures/validate-bad-output/main.tf diff --git a/config/config.go b/config/config.go index e0733081f..b4b405107 100644 --- a/config/config.go +++ b/config/config.go @@ -127,7 +127,7 @@ func (c *Config) Validate() error { "Unknown root level key: %s", k)) } - vars := c.allVariables() + vars := c.InterpolatedVariables() varMap := make(map[string]*Variable) for _, v := range c.Variables { varMap[v.Name] = v @@ -302,10 +302,10 @@ func (c *Config) Validate() error { return nil } -// allVariables is a helper that returns a mapping of all the interpolated +// InterpolatedVariables is a helper that returns a mapping of all the interpolated // variables within the configuration. This is used to verify references // are valid in the Validate step. -func (c *Config) allVariables() map[string][]InterpolatedVariable { +func (c *Config) InterpolatedVariables() map[string][]InterpolatedVariable { result := make(map[string][]InterpolatedVariable) for _, pc := range c.ProviderConfigs { source := fmt.Sprintf("provider config '%s'", pc.Name) diff --git a/config/module/test-fixtures/validate-bad-output/child/main.tf b/config/module/test-fixtures/validate-bad-output/child/main.tf new file mode 100644 index 000000000..e69de29bb diff --git a/config/module/test-fixtures/validate-bad-output/main.tf b/config/module/test-fixtures/validate-bad-output/main.tf new file mode 100644 index 000000000..a19233e12 --- /dev/null +++ b/config/module/test-fixtures/validate-bad-output/main.tf @@ -0,0 +1,7 @@ +module "child" { + source = "./child" +} + +resource "aws_instance" "foo" { + memory = "${module.child.memory}" +} diff --git a/config/module/test-fixtures/validate-child-good/child/main.tf b/config/module/test-fixtures/validate-child-good/child/main.tf index 618ae3c42..2cfd2a80f 100644 --- a/config/module/test-fixtures/validate-child-good/child/main.tf +++ b/config/module/test-fixtures/validate-child-good/child/main.tf @@ -1 +1,3 @@ variable "memory" {} + +output "result" {} diff --git a/config/module/test-fixtures/validate-child-good/main.tf b/config/module/test-fixtures/validate-child-good/main.tf index 7c70782f1..5f3ad8da5 100644 --- a/config/module/test-fixtures/validate-child-good/main.tf +++ b/config/module/test-fixtures/validate-child-good/main.tf @@ -2,3 +2,7 @@ module "child" { source = "./child" memory = "1G" } + +resource "aws_instance" "foo" { + memory = "${module.child.result}" +} diff --git a/config/module/tree.go b/config/module/tree.go index 867f812c8..ed1b2b3de 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -267,6 +267,37 @@ func (t *Tree) Validate() error { } } + // Go over all the variables used and make sure that any module + // variables represent outputs properly. + for source, vs := range t.config.InterpolatedVariables() { + for _, v := range vs { + mv, ok := v.(*config.ModuleVariable) + if !ok { + continue + } + + tree, ok := children[mv.Name] + if !ok { + // This should never happen because Load watches us + panic("module not found in children: " + mv.Name) + } + + found := false + for _, o := range tree.config.Outputs { + if o.Name == mv.Field { + found = true + break + } + } + if !found { + newErr.Err = fmt.Errorf( + "%s: %s is not a valid output for module %s", + source, mv.Field, mv.Name) + return newErr + } + } + } + return nil } diff --git a/config/module/tree_test.go b/config/module/tree_test.go index 3fbad23d7..d2fc41a5e 100644 --- a/config/module/tree_test.go +++ b/config/module/tree_test.go @@ -92,6 +92,18 @@ func TestTreeValidate_badChild(t *testing.T) { } } +func TestTreeValidate_badChildOutput(t *testing.T) { + tree := NewTree(testConfig(t, "validate-bad-output")) + + if err := tree.Load(testStorage(t), GetModeGet); err != nil { + t.Fatalf("err: %s", err) + } + + if err := tree.Validate(); err == nil { + t.Fatal("should error") + } +} + func TestTreeValidate_badChildVar(t *testing.T) { tree := NewTree(testConfig(t, "validate-bad-var")) From c0a30d333792e81cfd2e632be327f45204e82627 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Sep 2014 15:43:12 -0700 Subject: [PATCH 24/41] config: TestString --- config/config_string.go | 297 ++++++++++++++++++++++++++++++++++++++++ config/loader_test.go | 248 --------------------------------- config/module/tree.go | 21 +-- 3 files changed, 302 insertions(+), 264 deletions(-) create mode 100644 config/config_string.go diff --git a/config/config_string.go b/config/config_string.go new file mode 100644 index 000000000..89a990ae8 --- /dev/null +++ b/config/config_string.go @@ -0,0 +1,297 @@ +package config + +import ( + "bytes" + "fmt" + "sort" + "strings" +) + +// TestString is a Stringer-like function that outputs a string that can +// be used to easily compare multiple Config structures in unit tests. +// +// This function has no practical use outside of unit tests and debugging. +func (c *Config) TestString() string { + if c == nil { + return "" + } + + var buf bytes.Buffer + if len(c.Modules) > 0 { + buf.WriteString("Modules:\n\n") + buf.WriteString(modulesStr(c.Modules)) + buf.WriteString("\n\n") + } + + if len(c.Variables) > 0 { + buf.WriteString("Variables:\n\n") + buf.WriteString(variablesStr(c.Variables)) + buf.WriteString("\n\n") + } + + if len(c.ProviderConfigs) > 0 { + buf.WriteString("Provider Configs:\n\n") + buf.WriteString(providerConfigsStr(c.ProviderConfigs)) + buf.WriteString("\n\n") + } + + if len(c.Resources) > 0 { + buf.WriteString("Resources:\n\n") + buf.WriteString(resourcesStr(c.Resources)) + buf.WriteString("\n\n") + } + + if len(c.Outputs) > 0 { + buf.WriteString("Outputs:\n\n") + buf.WriteString(outputsStr(c.Outputs)) + buf.WriteString("\n") + } + + return strings.TrimSpace(buf.String()) +} + +func modulesStr(ms []*Module) string { + result := "" + order := make([]int, 0, len(ms)) + ks := make([]string, 0, len(ms)) + mapping := make(map[string]int) + for i, m := range ms { + k := m.Id() + ks = append(ks, k) + mapping[k] = i + } + sort.Strings(ks) + for _, k := range ks { + order = append(order, mapping[k]) + } + + for _, i := range order { + m := ms[i] + result += fmt.Sprintf("%s\n", m.Id()) + + ks := make([]string, 0, len(m.RawConfig.Raw)) + for k, _ := range m.RawConfig.Raw { + ks = append(ks, k) + } + sort.Strings(ks) + + result += fmt.Sprintf(" source = %s\n", m.Source) + + for _, k := range ks { + result += fmt.Sprintf(" %s\n", k) + } + } + + return strings.TrimSpace(result) +} + +func outputsStr(os []*Output) string { + ns := make([]string, 0, len(os)) + m := make(map[string]*Output) + for _, o := range os { + ns = append(ns, o.Name) + m[o.Name] = o + } + sort.Strings(ns) + + result := "" + for _, n := range ns { + o := m[n] + + result += fmt.Sprintf("%s\n", n) + + if len(o.RawConfig.Variables) > 0 { + result += fmt.Sprintf(" vars\n") + for _, rawV := range o.RawConfig.Variables { + kind := "unknown" + str := rawV.FullKey() + + switch rawV.(type) { + case *ResourceVariable: + kind = "resource" + case *UserVariable: + kind = "user" + } + + result += fmt.Sprintf(" %s: %s\n", kind, str) + } + } + } + + return strings.TrimSpace(result) +} + +// This helper turns a provider configs field into a deterministic +// string value for comparison in tests. +func providerConfigsStr(pcs []*ProviderConfig) string { + result := "" + + ns := make([]string, 0, len(pcs)) + m := make(map[string]*ProviderConfig) + for _, n := range pcs { + ns = append(ns, n.Name) + m[n.Name] = n + } + sort.Strings(ns) + + for _, n := range ns { + pc := m[n] + + result += fmt.Sprintf("%s\n", n) + + keys := make([]string, 0, len(pc.RawConfig.Raw)) + for k, _ := range pc.RawConfig.Raw { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + result += fmt.Sprintf(" %s\n", k) + } + + if len(pc.RawConfig.Variables) > 0 { + result += fmt.Sprintf(" vars\n") + for _, rawV := range pc.RawConfig.Variables { + kind := "unknown" + str := rawV.FullKey() + + switch rawV.(type) { + case *ResourceVariable: + kind = "resource" + case *UserVariable: + kind = "user" + } + + result += fmt.Sprintf(" %s: %s\n", kind, str) + } + } + } + + return strings.TrimSpace(result) +} + +// This helper turns a resources field into a deterministic +// string value for comparison in tests. +func resourcesStr(rs []*Resource) string { + result := "" + order := make([]int, 0, len(rs)) + ks := make([]string, 0, len(rs)) + mapping := make(map[string]int) + for i, r := range rs { + k := fmt.Sprintf("%s[%s]", r.Type, r.Name) + ks = append(ks, k) + mapping[k] = i + } + sort.Strings(ks) + for _, k := range ks { + order = append(order, mapping[k]) + } + + for _, i := range order { + r := rs[i] + result += fmt.Sprintf( + "%s[%s] (x%d)\n", + r.Type, + r.Name, + r.Count) + + ks := make([]string, 0, len(r.RawConfig.Raw)) + for k, _ := range r.RawConfig.Raw { + ks = append(ks, k) + } + sort.Strings(ks) + + for _, k := range ks { + result += fmt.Sprintf(" %s\n", k) + } + + if len(r.Provisioners) > 0 { + result += fmt.Sprintf(" provisioners\n") + for _, p := range r.Provisioners { + result += fmt.Sprintf(" %s\n", p.Type) + + ks := make([]string, 0, len(p.RawConfig.Raw)) + for k, _ := range p.RawConfig.Raw { + ks = append(ks, k) + } + sort.Strings(ks) + + for _, k := range ks { + result += fmt.Sprintf(" %s\n", k) + } + } + } + + if len(r.DependsOn) > 0 { + result += fmt.Sprintf(" dependsOn\n") + for _, d := range r.DependsOn { + result += fmt.Sprintf(" %s\n", d) + } + } + + if len(r.RawConfig.Variables) > 0 { + result += fmt.Sprintf(" vars\n") + + ks := make([]string, 0, len(r.RawConfig.Variables)) + for k, _ := range r.RawConfig.Variables { + ks = append(ks, k) + } + sort.Strings(ks) + + for _, k := range ks { + rawV := r.RawConfig.Variables[k] + kind := "unknown" + str := rawV.FullKey() + + switch rawV.(type) { + case *ResourceVariable: + kind = "resource" + case *UserVariable: + kind = "user" + } + + result += fmt.Sprintf(" %s: %s\n", kind, str) + } + } + } + + return strings.TrimSpace(result) +} + +// This helper turns a variables field into a deterministic +// string value for comparison in tests. +func variablesStr(vs []*Variable) string { + result := "" + ks := make([]string, 0, len(vs)) + m := make(map[string]*Variable) + for _, v := range vs { + ks = append(ks, v.Name) + m[v.Name] = v + } + sort.Strings(ks) + + for _, k := range ks { + v := m[k] + + required := "" + if v.Required() { + required = " (required)" + } + + if v.Default == nil || v.Default == "" { + v.Default = "<>" + } + if v.Description == "" { + v.Description = "<>" + } + + result += fmt.Sprintf( + "%s%s\n %v\n %s\n", + k, + required, + v.Default, + v.Description) + } + + return strings.TrimSpace(result) +} diff --git a/config/loader_test.go b/config/loader_test.go index b92307a70..f95235d66 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -1,9 +1,7 @@ package config import ( - "fmt" "path/filepath" - "sort" "strings" "testing" ) @@ -264,42 +262,6 @@ func TestLoadDir_override(t *testing.T) { } } -func outputsStr(os []*Output) string { - ns := make([]string, 0, len(os)) - m := make(map[string]*Output) - for _, o := range os { - ns = append(ns, o.Name) - m[o.Name] = o - } - sort.Strings(ns) - - result := "" - for _, n := range ns { - o := m[n] - - result += fmt.Sprintf("%s\n", n) - - if len(o.RawConfig.Variables) > 0 { - result += fmt.Sprintf(" vars\n") - for _, rawV := range o.RawConfig.Variables { - kind := "unknown" - str := rawV.FullKey() - - switch rawV.(type) { - case *ResourceVariable: - kind = "resource" - case *UserVariable: - kind = "user" - } - - result += fmt.Sprintf(" %s: %s\n", kind, str) - } - } - } - - return strings.TrimSpace(result) -} - func TestLoad_provisioners(t *testing.T) { c, err := Load(filepath.Join(fixtureDir, "provisioners.tf")) if err != nil { @@ -354,216 +316,6 @@ func TestLoad_connections(t *testing.T) { } } -func modulesStr(ms []*Module) string { - result := "" - order := make([]int, 0, len(ms)) - ks := make([]string, 0, len(ms)) - mapping := make(map[string]int) - for i, m := range ms { - k := m.Id() - ks = append(ks, k) - mapping[k] = i - } - sort.Strings(ks) - for _, k := range ks { - order = append(order, mapping[k]) - } - - for _, i := range order { - m := ms[i] - result += fmt.Sprintf("%s\n", m.Id()) - - ks := make([]string, 0, len(m.RawConfig.Raw)) - for k, _ := range m.RawConfig.Raw { - ks = append(ks, k) - } - sort.Strings(ks) - - result += fmt.Sprintf(" source = %s\n", m.Source) - - for _, k := range ks { - result += fmt.Sprintf(" %s\n", k) - } - } - - return strings.TrimSpace(result) -} - -// This helper turns a provider configs field into a deterministic -// string value for comparison in tests. -func providerConfigsStr(pcs []*ProviderConfig) string { - result := "" - - ns := make([]string, 0, len(pcs)) - m := make(map[string]*ProviderConfig) - for _, n := range pcs { - ns = append(ns, n.Name) - m[n.Name] = n - } - sort.Strings(ns) - - for _, n := range ns { - pc := m[n] - - result += fmt.Sprintf("%s\n", n) - - keys := make([]string, 0, len(pc.RawConfig.Raw)) - for k, _ := range pc.RawConfig.Raw { - keys = append(keys, k) - } - sort.Strings(keys) - - for _, k := range keys { - result += fmt.Sprintf(" %s\n", k) - } - - if len(pc.RawConfig.Variables) > 0 { - result += fmt.Sprintf(" vars\n") - for _, rawV := range pc.RawConfig.Variables { - kind := "unknown" - str := rawV.FullKey() - - switch rawV.(type) { - case *ResourceVariable: - kind = "resource" - case *UserVariable: - kind = "user" - } - - result += fmt.Sprintf(" %s: %s\n", kind, str) - } - } - } - - return strings.TrimSpace(result) -} - -// This helper turns a resources field into a deterministic -// string value for comparison in tests. -func resourcesStr(rs []*Resource) string { - result := "" - order := make([]int, 0, len(rs)) - ks := make([]string, 0, len(rs)) - mapping := make(map[string]int) - for i, r := range rs { - k := fmt.Sprintf("%s[%s]", r.Type, r.Name) - ks = append(ks, k) - mapping[k] = i - } - sort.Strings(ks) - for _, k := range ks { - order = append(order, mapping[k]) - } - - for _, i := range order { - r := rs[i] - result += fmt.Sprintf( - "%s[%s] (x%d)\n", - r.Type, - r.Name, - r.Count) - - ks := make([]string, 0, len(r.RawConfig.Raw)) - for k, _ := range r.RawConfig.Raw { - ks = append(ks, k) - } - sort.Strings(ks) - - for _, k := range ks { - result += fmt.Sprintf(" %s\n", k) - } - - if len(r.Provisioners) > 0 { - result += fmt.Sprintf(" provisioners\n") - for _, p := range r.Provisioners { - result += fmt.Sprintf(" %s\n", p.Type) - - ks := make([]string, 0, len(p.RawConfig.Raw)) - for k, _ := range p.RawConfig.Raw { - ks = append(ks, k) - } - sort.Strings(ks) - - for _, k := range ks { - result += fmt.Sprintf(" %s\n", k) - } - } - } - - if len(r.DependsOn) > 0 { - result += fmt.Sprintf(" dependsOn\n") - for _, d := range r.DependsOn { - result += fmt.Sprintf(" %s\n", d) - } - } - - if len(r.RawConfig.Variables) > 0 { - result += fmt.Sprintf(" vars\n") - - ks := make([]string, 0, len(r.RawConfig.Variables)) - for k, _ := range r.RawConfig.Variables { - ks = append(ks, k) - } - sort.Strings(ks) - - for _, k := range ks { - rawV := r.RawConfig.Variables[k] - kind := "unknown" - str := rawV.FullKey() - - switch rawV.(type) { - case *ResourceVariable: - kind = "resource" - case *UserVariable: - kind = "user" - } - - result += fmt.Sprintf(" %s: %s\n", kind, str) - } - } - } - - return strings.TrimSpace(result) -} - -// This helper turns a variables field into a deterministic -// string value for comparison in tests. -func variablesStr(vs []*Variable) string { - result := "" - ks := make([]string, 0, len(vs)) - m := make(map[string]*Variable) - for _, v := range vs { - ks = append(ks, v.Name) - m[v.Name] = v - } - sort.Strings(ks) - - for _, k := range ks { - v := m[k] - - required := "" - if v.Required() { - required = " (required)" - } - - if v.Default == nil || v.Default == "" { - v.Default = "<>" - } - if v.Description == "" { - v.Description = "<>" - } - - result += fmt.Sprintf( - "%s%s\n %v\n %s\n", - k, - required, - v.Default, - v.Description) - } - - return strings.TrimSpace(result) -} - const basicOutputsStr = ` web_ip vars diff --git a/config/module/tree.go b/config/module/tree.go index ed1b2b3de..f37dba649 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -56,17 +56,6 @@ func (t *Tree) Children() map[string]*Tree { return t.children } -// Flatten takes the entire module tree and flattens it into a single -// namespace in *config.Config with no module imports. -// -// Validate is called here implicitly, since it is important that semantic -// checks pass before flattening the configuration. Otherwise, encapsulation -// breaks in horrible ways and the errors that come out the other side -// will be surprising. -func (t *Tree) Flatten() (*config.Config, error) { - return nil, nil -} - // Loaded says whether or not this tree has been loaded or not yet. func (t *Tree) Loaded() bool { t.lock.RLock() @@ -212,7 +201,7 @@ func (t *Tree) Validate() error { } // If something goes wrong, here is our error template - newErr := &ValidateError{Name: []string{t.Name()}} + newErr := &TreeError{Name: []string{t.Name()}} // Validate our configuration first. if err := t.config.Validate(); err != nil { @@ -230,7 +219,7 @@ func (t *Tree) Validate() error { continue } - verr, ok := err.(*ValidateError) + verr, ok := err.(*TreeError) if !ok { // Unknown error, just return... return err @@ -301,14 +290,14 @@ func (t *Tree) Validate() error { return nil } -// ValidateError is an error returned by Tree.Validate if an error occurs +// TreeError is an error returned by Tree.Validate if an error occurs // with validation. -type ValidateError struct { +type TreeError struct { Name []string Err error } -func (e *ValidateError) Error() string { +func (e *TreeError) Error() string { // Build up the name var buf bytes.Buffer for _, n := range e.Name { From 7bbf6a0d3a38c3b4f554e1fd656a64e14d3e4673 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Sep 2014 15:49:07 -0700 Subject: [PATCH 25/41] config/module: NewTreeModule is easier to use --- config/module/tree.go | 21 +++++++++++++++------ config/module/tree_test.go | 21 +++++++++++---------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/config/module/tree.go b/config/module/tree.go index f37dba649..558dcce9d 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -42,8 +42,20 @@ const ( ) // NewTree returns a new Tree for the given config structure. -func NewTree(c *config.Config) *Tree { - return &Tree{config: c} +func NewTree(name string, c *config.Config) *Tree { + return &Tree{config: c, name: name} +} + +// NewTreeModule is like NewTree except it parses the configuration in +// the directory and gives it a specific name. Use a blank name "" to specify +// the root module. +func NewTreeModule(name, dir string) (*Tree, error) { + c, err := config.LoadDir(dir) + if err != nil { + return nil, err + } + + return NewTree(name, c), nil } // Children returns the children of this tree (the modules that are @@ -141,14 +153,11 @@ func (t *Tree) Load(s Storage, mode GetMode) error { } // Load the configuration - c, err := config.LoadDir(dir) + children[m.Name], err = NewTreeModule(m.Name, dir) if err != nil { return fmt.Errorf( "module %s: %s", m.Name, err) } - - children[m.Name] = NewTree(c) - children[m.Name].name = m.Name } // Go through all the children and load them. diff --git a/config/module/tree_test.go b/config/module/tree_test.go index d2fc41a5e..95c481dde 100644 --- a/config/module/tree_test.go +++ b/config/module/tree_test.go @@ -8,7 +8,7 @@ import ( func TestTreeLoad(t *testing.T) { storage := testStorage(t) - tree := NewTree(testConfig(t, "basic")) + tree := NewTree("", testConfig(t, "basic")) if tree.Loaded() { t.Fatal("should not be loaded") @@ -46,7 +46,7 @@ func TestTreeLoad(t *testing.T) { func TestTreeLoad_duplicate(t *testing.T) { storage := testStorage(t) - tree := NewTree(testConfig(t, "dup")) + tree := NewTree("", testConfig(t, "dup")) if tree.Loaded() { t.Fatal("should not be loaded") @@ -59,7 +59,7 @@ func TestTreeLoad_duplicate(t *testing.T) { } func TestTreeModules(t *testing.T) { - tree := NewTree(testConfig(t, "basic")) + tree := NewTree("", testConfig(t, "basic")) actual := tree.Modules() expected := []*Module{ @@ -72,7 +72,7 @@ func TestTreeModules(t *testing.T) { } func TestTreeName(t *testing.T) { - tree := NewTree(testConfig(t, "basic")) + tree := NewTree("", testConfig(t, "basic")) actual := tree.Name() if actual != "" { @@ -81,7 +81,7 @@ func TestTreeName(t *testing.T) { } func TestTreeValidate_badChild(t *testing.T) { - tree := NewTree(testConfig(t, "validate-child-bad")) + tree := NewTree("", testConfig(t, "validate-child-bad")) if err := tree.Load(testStorage(t), GetModeGet); err != nil { t.Fatalf("err: %s", err) @@ -93,7 +93,7 @@ func TestTreeValidate_badChild(t *testing.T) { } func TestTreeValidate_badChildOutput(t *testing.T) { - tree := NewTree(testConfig(t, "validate-bad-output")) + tree := NewTree("", testConfig(t, "validate-bad-output")) if err := tree.Load(testStorage(t), GetModeGet); err != nil { t.Fatalf("err: %s", err) @@ -105,7 +105,7 @@ func TestTreeValidate_badChildOutput(t *testing.T) { } func TestTreeValidate_badChildVar(t *testing.T) { - tree := NewTree(testConfig(t, "validate-bad-var")) + tree := NewTree("", testConfig(t, "validate-bad-var")) if err := tree.Load(testStorage(t), GetModeGet); err != nil { t.Fatalf("err: %s", err) @@ -117,7 +117,7 @@ func TestTreeValidate_badChildVar(t *testing.T) { } func TestTreeValidate_badRoot(t *testing.T) { - tree := NewTree(testConfig(t, "validate-root-bad")) + tree := NewTree("", testConfig(t, "validate-root-bad")) if err := tree.Load(testStorage(t), GetModeGet); err != nil { t.Fatalf("err: %s", err) @@ -129,7 +129,7 @@ func TestTreeValidate_badRoot(t *testing.T) { } func TestTreeValidate_good(t *testing.T) { - tree := NewTree(testConfig(t, "validate-child-good")) + tree := NewTree("", testConfig(t, "validate-child-good")) if err := tree.Load(testStorage(t), GetModeGet); err != nil { t.Fatalf("err: %s", err) @@ -141,13 +141,14 @@ func TestTreeValidate_good(t *testing.T) { } func TestTreeValidate_notLoaded(t *testing.T) { - tree := NewTree(testConfig(t, "basic")) + tree := NewTree("", testConfig(t, "basic")) if err := tree.Validate(); err == nil { t.Fatal("should error") } } + const treeLoadStr = ` foo From cf4885d2fd456a233c2cf495733e1f9bd6504b64 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Sep 2014 23:32:30 -0700 Subject: [PATCH 26/41] config/module: git support --- config/module/get.go | 28 ++++++++++++++++++++++++++++ config/module/get_git.go | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 config/module/get_git.go diff --git a/config/module/get.go b/config/module/get.go index 270ca815a..7c4761d6e 100644 --- a/config/module/get.go +++ b/config/module/get.go @@ -1,8 +1,11 @@ package module import ( + "bytes" "fmt" "net/url" + "os/exec" + "syscall" ) // Getter defines the interface that schemes must implement to download @@ -24,6 +27,7 @@ var Getters map[string]Getter func init() { Getters = map[string]Getter{ "file": new(FileGetter), + "git": new(GitGetter), } } @@ -51,3 +55,27 @@ func Get(dst, src string) error { return err } + +// getRunCommand is a helper that will run a command and capture the output +// in the case an error happens. +func getRunCommand(cmd *exec.Cmd) error { + var buf bytes.Buffer + cmd.Stdout = &buf + cmd.Stderr = &buf + err := cmd.Run() + if err == nil { + return nil + } + if exiterr, ok := err.(*exec.ExitError); ok { + // The program has exited with an exit code != 0 + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + return fmt.Errorf( + "%s exited with %d: %s", + cmd.Path, + status.ExitStatus(), + buf.String()) + } + } + + return fmt.Errorf("error running %s: %s", cmd.Path, buf.String()) +} diff --git a/config/module/get_git.go b/config/module/get_git.go new file mode 100644 index 000000000..132bd8e11 --- /dev/null +++ b/config/module/get_git.go @@ -0,0 +1,39 @@ +package module + +import ( + "fmt" + "net/url" + "os" + "os/exec" +) + +// GitGetter is a Getter implementation that will download a module from +// a git repository. +type GitGetter struct{} + +func (g *GitGetter) Get(dst string, u *url.URL) error { + if _, err := exec.LookPath("git"); err != nil { + return fmt.Errorf("git must be available and on the PATH") + } + + _, err := os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + return g.update(dst, u) + } + + return g.clone(dst, u) +} + +func (g *GitGetter) clone(dst string, u *url.URL) error { + cmd := exec.Command("git", "clone", u.String(), dst) + return getRunCommand(cmd) +} + +func (g *GitGetter) update(dst string, u *url.URL) error { + cmd := exec.Command("git", "pull", "--ff-only") + cmd.Dir = dst + return getRunCommand(cmd) +} From acb6d12a756dcdb92906cc0780065b0b939132a4 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Sep 2014 23:48:56 -0700 Subject: [PATCH 27/41] config/module: support forced getters with TYPE::URL syntax --- config/module/detect.go | 17 +++++++++++++---- config/module/detect_test.go | 27 +++++++++++++++++++++++++++ config/module/get.go | 27 +++++++++++++++++++++++++-- config/module/get_test.go | 15 +++++++++++++++ 4 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 config/module/detect_test.go diff --git a/config/module/detect.go b/config/module/detect.go index 855b34991..633bda246 100644 --- a/config/module/detect.go +++ b/config/module/detect.go @@ -30,20 +30,29 @@ func init() { // This is safe to be called with an already valid source string: Detect // will just return it. func Detect(src string, pwd string) (string, error) { - u, err := url.Parse(src) + getForce, getSrc := getForcedGetter(src) + + u, err := url.Parse(getSrc) if err == nil && u.Scheme != "" { // Valid URL return src, nil } for _, d := range Detectors { - result, ok, err := d.Detect(src, pwd) + result, ok, err := d.Detect(getSrc, pwd) if err != nil { return "", err } - if ok { - return result, nil + if !ok { + continue } + + // Preserve the forced getter if it exists + if getForce != "" { + result = fmt.Sprintf("%s::%s", getForce, result) + } + + return result, nil } return "", fmt.Errorf("invalid source string: %s", src) diff --git a/config/module/detect_test.go b/config/module/detect_test.go new file mode 100644 index 000000000..5fdf5dc74 --- /dev/null +++ b/config/module/detect_test.go @@ -0,0 +1,27 @@ +package module + +import ( + "testing" +) + +func TestDetect(t *testing.T) { + cases := []struct { + Input string + Pwd string + Output string + Err bool + }{ + {"./foo", "/foo", "file:///foo/foo", false}, + {"git::./foo", "/foo", "git::file:///foo/foo", false}, + } + + for i, tc := range cases { + output, err := Detect(tc.Input, tc.Pwd) + if (err != nil) != tc.Err { + t.Fatalf("%d: bad err: %s", i, err) + } + if output != tc.Output { + t.Fatalf("%d: bad output: %s", i, output) + } + } +} diff --git a/config/module/get.go b/config/module/get.go index 7c4761d6e..9a459760c 100644 --- a/config/module/get.go +++ b/config/module/get.go @@ -5,6 +5,7 @@ import ( "fmt" "net/url" "os/exec" + "regexp" "syscall" ) @@ -24,6 +25,10 @@ type Getter interface { // be used to get a dependency. var Getters map[string]Getter +// forcedRegexp is the regular expression that finds forced getters. This +// syntax is schema::url, example: git::https://foo.com +var forcedRegexp = regexp.MustCompile(`^([A-Za-z]+)::(.+)$`) + func init() { Getters = map[string]Getter{ "file": new(FileGetter), @@ -37,15 +42,21 @@ func init() { // src is a URL, whereas dst is always just a file path to a folder. This // folder doesn't need to exist. It will be created if it doesn't exist. func Get(dst, src string) error { + var force string + force, src = getForcedGetter(src) + u, err := url.Parse(src) if err != nil { return err } + if force == "" { + force = u.Scheme + } - g, ok := Getters[u.Scheme] + g, ok := Getters[force] if !ok { return fmt.Errorf( - "module download not supported for scheme '%s'", u.Scheme) + "module download not supported for scheme '%s'", force) } err = g.Get(dst, u) @@ -79,3 +90,15 @@ func getRunCommand(cmd *exec.Cmd) error { return fmt.Errorf("error running %s: %s", cmd.Path, buf.String()) } + +// getForcedGetter takes a source and returns the tuple of the forced +// getter and the raw URL (without the force syntax). +func getForcedGetter(src string) (string, string) { + var forced string + if ms := forcedRegexp.FindStringSubmatch(src); ms != nil { + forced = ms[1] + src = ms[2] + } + + return forced, src +} diff --git a/config/module/get_test.go b/config/module/get_test.go index 94d131a3d..742e9d794 100644 --- a/config/module/get_test.go +++ b/config/module/get_test.go @@ -30,3 +30,18 @@ func TestGet_file(t *testing.T) { t.Fatalf("err: %s", err) } } + +func TestGet_fileForced(t *testing.T) { + dst := tempDir(t) + u := testModule("basic") + u = "file::"+u + + if err := Get(dst, u); err != nil { + t.Fatalf("err: %s", err) + } + + mainPath := filepath.Join(dst, "main.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} From 3e2989daf1f165bd6a714781aafd769d4a2ea0fc Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Sep 2014 23:52:27 -0700 Subject: [PATCH 28/41] config/module: test Git --- config/module/get_file_test.go | 10 ---------- config/module/get_git_test.go | 27 +++++++++++++++++++++++++++ config/module/module_test.go | 9 +++++++++ config/module/test-fixtures/basic-git | 1 + 4 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 config/module/get_git_test.go create mode 160000 config/module/test-fixtures/basic-git diff --git a/config/module/get_file_test.go b/config/module/get_file_test.go index 6cde4befc..7cc69bccb 100644 --- a/config/module/get_file_test.go +++ b/config/module/get_file_test.go @@ -1,7 +1,6 @@ package module import ( - "net/url" "os" "path/filepath" "testing" @@ -103,12 +102,3 @@ func TestFileGetter_dirSymlink(t *testing.T) { t.Fatalf("err: %s", err) } } - -func testModuleURL(n string) *url.URL { - u, err := url.Parse(testModule(n)) - if err != nil { - panic(err) - } - - return u -} diff --git a/config/module/get_git_test.go b/config/module/get_git_test.go new file mode 100644 index 000000000..46f50a277 --- /dev/null +++ b/config/module/get_git_test.go @@ -0,0 +1,27 @@ +package module + +import ( + "os" + "path/filepath" + "testing" +) + +func TestGitGetter_impl(t *testing.T) { + var _ Getter = new(GitGetter) +} + +func TestGitGetter(t *testing.T) { + g := new(GitGetter) + dst := tempDir(t) + + // With a dir that doesn't exist + if err := g.Get(dst, testModuleURL("basic-git")); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath := filepath.Join(dst, "main.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} diff --git a/config/module/module_test.go b/config/module/module_test.go index 41f15e315..bd72f0862 100644 --- a/config/module/module_test.go +++ b/config/module/module_test.go @@ -46,6 +46,15 @@ func testModule(n string) string { return url.String() } +func testModuleURL(n string) *url.URL { + u, err := url.Parse(testModule(n)) + if err != nil { + panic(err) + } + + return u +} + func testStorage(t *testing.T) Storage { return &FolderStorage{StorageDir: tempDir(t)} } diff --git a/config/module/test-fixtures/basic-git b/config/module/test-fixtures/basic-git new file mode 160000 index 000000000..7f872b3fd --- /dev/null +++ b/config/module/test-fixtures/basic-git @@ -0,0 +1 @@ +Subproject commit 7f872b3fd8504ba63330261b35073a1c4f35383d From 96385113e7bbb81d51ce83e18cf02e72ecfcbeda Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 15 Sep 2014 23:55:07 -0700 Subject: [PATCH 29/41] config/module: delete weird subproject business --- config/module/test-fixtures/basic-git | 1 - 1 file changed, 1 deletion(-) delete mode 160000 config/module/test-fixtures/basic-git diff --git a/config/module/test-fixtures/basic-git b/config/module/test-fixtures/basic-git deleted file mode 160000 index 7f872b3fd..000000000 --- a/config/module/test-fixtures/basic-git +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7f872b3fd8504ba63330261b35073a1c4f35383d From fc71d7091f8f79ebdccfa51cda64df671eaf4234 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Sep 2014 00:03:20 -0700 Subject: [PATCH 30/41] config/module: Git test... this is kind of ghetto --- config/module/get_git_test.go | 24 +++ .../basic-git/DOTgit/COMMIT_EDITMSG | 10 ++ .../test-fixtures/basic-git/DOTgit/HEAD | 1 + .../test-fixtures/basic-git/DOTgit/config | 7 + .../basic-git/DOTgit/description | 1 + .../DOTgit/hooks/applypatch-msg.sample | 15 ++ .../basic-git/DOTgit/hooks/commit-msg.sample | 24 +++ .../basic-git/DOTgit/hooks/post-update.sample | 8 + .../DOTgit/hooks/pre-applypatch.sample | 14 ++ .../basic-git/DOTgit/hooks/pre-commit.sample | 49 +++++ .../basic-git/DOTgit/hooks/pre-push.sample | 54 ++++++ .../basic-git/DOTgit/hooks/pre-rebase.sample | 169 ++++++++++++++++++ .../DOTgit/hooks/prepare-commit-msg.sample | 36 ++++ .../basic-git/DOTgit/hooks/update.sample | 128 +++++++++++++ .../test-fixtures/basic-git/DOTgit/index | Bin 0 -> 104 bytes .../basic-git/DOTgit/info/exclude | 6 + .../test-fixtures/basic-git/DOTgit/logs/HEAD | 1 + .../basic-git/DOTgit/logs/refs/heads/master | 1 + .../38/30637158f774a20edcc0bf1c4d07b0bf87c43d | Bin 0 -> 59 bytes .../49/7bc37401eb3c9b11865b1768725b64066eccee | 2 + .../96/43088174e25a9bd91c27970a580af0085c9f32 | Bin 0 -> 52 bytes .../basic-git/DOTgit/refs/heads/master | 1 + config/module/test-fixtures/basic-git/main.tf | 5 + 23 files changed, 556 insertions(+) create mode 100644 config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG create mode 100644 config/module/test-fixtures/basic-git/DOTgit/HEAD create mode 100644 config/module/test-fixtures/basic-git/DOTgit/config create mode 100644 config/module/test-fixtures/basic-git/DOTgit/description create mode 100755 config/module/test-fixtures/basic-git/DOTgit/hooks/applypatch-msg.sample create mode 100755 config/module/test-fixtures/basic-git/DOTgit/hooks/commit-msg.sample create mode 100755 config/module/test-fixtures/basic-git/DOTgit/hooks/post-update.sample create mode 100755 config/module/test-fixtures/basic-git/DOTgit/hooks/pre-applypatch.sample create mode 100755 config/module/test-fixtures/basic-git/DOTgit/hooks/pre-commit.sample create mode 100755 config/module/test-fixtures/basic-git/DOTgit/hooks/pre-push.sample create mode 100755 config/module/test-fixtures/basic-git/DOTgit/hooks/pre-rebase.sample create mode 100755 config/module/test-fixtures/basic-git/DOTgit/hooks/prepare-commit-msg.sample create mode 100755 config/module/test-fixtures/basic-git/DOTgit/hooks/update.sample create mode 100644 config/module/test-fixtures/basic-git/DOTgit/index create mode 100644 config/module/test-fixtures/basic-git/DOTgit/info/exclude create mode 100644 config/module/test-fixtures/basic-git/DOTgit/logs/HEAD create mode 100644 config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master create mode 100644 config/module/test-fixtures/basic-git/DOTgit/objects/38/30637158f774a20edcc0bf1c4d07b0bf87c43d create mode 100644 config/module/test-fixtures/basic-git/DOTgit/objects/49/7bc37401eb3c9b11865b1768725b64066eccee create mode 100644 config/module/test-fixtures/basic-git/DOTgit/objects/96/43088174e25a9bd91c27970a580af0085c9f32 create mode 100644 config/module/test-fixtures/basic-git/DOTgit/refs/heads/master create mode 100644 config/module/test-fixtures/basic-git/main.tf diff --git a/config/module/get_git_test.go b/config/module/get_git_test.go index 46f50a277..33a0a98a6 100644 --- a/config/module/get_git_test.go +++ b/config/module/get_git_test.go @@ -2,18 +2,42 @@ package module import ( "os" + "os/exec" "path/filepath" "testing" ) +var testHasGit bool + +func init() { + if _, err := exec.LookPath("git"); err == nil { + testHasGit = true + } +} + func TestGitGetter_impl(t *testing.T) { var _ Getter = new(GitGetter) } func TestGitGetter(t *testing.T) { + if !testHasGit { + t.Log("git not found, skipping") + t.Skip() + } + g := new(GitGetter) dst := tempDir(t) + // Git doesn't allow nested ".git" directories so we do some hackiness + // here to get around that... + moduleDir := filepath.Join(fixtureDir, "basic-git") + oldName := filepath.Join(moduleDir, "DOTgit") + newName := filepath.Join(moduleDir, ".git") + if err := os.Rename(oldName, newName); err != nil { + t.Fatalf("err: %s", err) + } + defer os.Rename(newName, oldName) + // With a dir that doesn't exist if err := g.Get(dst, testModuleURL("basic-git")); err != nil { t.Fatalf("err: %s", err) diff --git a/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG b/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG new file mode 100644 index 000000000..fe6bc2b3d --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG @@ -0,0 +1,10 @@ +A commit +# Please enter the commit message for your changes. Lines starting +# with '#' will be ignored, and an empty message aborts the commit. +# On branch master +# +# Initial commit +# +# Changes to be committed: +# new file: main.tf +# diff --git a/config/module/test-fixtures/basic-git/DOTgit/HEAD b/config/module/test-fixtures/basic-git/DOTgit/HEAD new file mode 100644 index 000000000..cb089cd89 --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/config/module/test-fixtures/basic-git/DOTgit/config b/config/module/test-fixtures/basic-git/DOTgit/config new file mode 100644 index 000000000..6c9406b7d --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/config @@ -0,0 +1,7 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = false + logallrefupdates = true + ignorecase = true + precomposeunicode = true diff --git a/config/module/test-fixtures/basic-git/DOTgit/description b/config/module/test-fixtures/basic-git/DOTgit/description new file mode 100644 index 000000000..498b267a8 --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/description @@ -0,0 +1 @@ +Unnamed repository; edit this file 'description' to name the repository. diff --git a/config/module/test-fixtures/basic-git/DOTgit/hooks/applypatch-msg.sample b/config/module/test-fixtures/basic-git/DOTgit/hooks/applypatch-msg.sample new file mode 100755 index 000000000..8b2a2fe84 --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/hooks/applypatch-msg.sample @@ -0,0 +1,15 @@ +#!/bin/sh +# +# An example hook script to check the commit log message taken by +# applypatch from an e-mail message. +# +# The hook should exit with non-zero status after issuing an +# appropriate message if it wants to stop the commit. The hook is +# allowed to edit the commit message file. +# +# To enable this hook, rename this file to "applypatch-msg". + +. git-sh-setup +test -x "$GIT_DIR/hooks/commit-msg" && + exec "$GIT_DIR/hooks/commit-msg" ${1+"$@"} +: diff --git a/config/module/test-fixtures/basic-git/DOTgit/hooks/commit-msg.sample b/config/module/test-fixtures/basic-git/DOTgit/hooks/commit-msg.sample new file mode 100755 index 000000000..b58d1184a --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/hooks/commit-msg.sample @@ -0,0 +1,24 @@ +#!/bin/sh +# +# An example hook script to check the commit log message. +# Called by "git commit" with one argument, the name of the file +# that has the commit message. The hook should exit with non-zero +# status after issuing an appropriate message if it wants to stop the +# commit. The hook is allowed to edit the commit message file. +# +# To enable this hook, rename this file to "commit-msg". + +# Uncomment the below to add a Signed-off-by line to the message. +# Doing this in a hook is a bad idea in general, but the prepare-commit-msg +# hook is more suited to it. +# +# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1" + +# This example catches duplicate Signed-off-by lines. + +test "" = "$(grep '^Signed-off-by: ' "$1" | + sort | uniq -c | sed -e '/^[ ]*1[ ]/d')" || { + echo >&2 Duplicate Signed-off-by lines. + exit 1 +} diff --git a/config/module/test-fixtures/basic-git/DOTgit/hooks/post-update.sample b/config/module/test-fixtures/basic-git/DOTgit/hooks/post-update.sample new file mode 100755 index 000000000..ec17ec193 --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/hooks/post-update.sample @@ -0,0 +1,8 @@ +#!/bin/sh +# +# An example hook script to prepare a packed repository for use over +# dumb transports. +# +# To enable this hook, rename this file to "post-update". + +exec git update-server-info diff --git a/config/module/test-fixtures/basic-git/DOTgit/hooks/pre-applypatch.sample b/config/module/test-fixtures/basic-git/DOTgit/hooks/pre-applypatch.sample new file mode 100755 index 000000000..b1f187c2e --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/hooks/pre-applypatch.sample @@ -0,0 +1,14 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed +# by applypatch from an e-mail message. +# +# The hook should exit with non-zero status after issuing an +# appropriate message if it wants to stop the commit. +# +# To enable this hook, rename this file to "pre-applypatch". + +. git-sh-setup +test -x "$GIT_DIR/hooks/pre-commit" && + exec "$GIT_DIR/hooks/pre-commit" ${1+"$@"} +: diff --git a/config/module/test-fixtures/basic-git/DOTgit/hooks/pre-commit.sample b/config/module/test-fixtures/basic-git/DOTgit/hooks/pre-commit.sample new file mode 100755 index 000000000..68d62d544 --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/hooks/pre-commit.sample @@ -0,0 +1,49 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed. +# Called by "git commit" with no arguments. The hook should +# exit with non-zero status after issuing an appropriate message if +# it wants to stop the commit. +# +# To enable this hook, rename this file to "pre-commit". + +if git rev-parse --verify HEAD >/dev/null 2>&1 +then + against=HEAD +else + # Initial commit: diff against an empty tree object + against=4b825dc642cb6eb9a060e54bf8d69288fbee4904 +fi + +# If you want to allow non-ASCII filenames set this variable to true. +allownonascii=$(git config --bool hooks.allownonascii) + +# Redirect output to stderr. +exec 1>&2 + +# Cross platform projects tend to avoid non-ASCII filenames; prevent +# them from being added to the repository. We exploit the fact that the +# printable range starts at the space character and ends with tilde. +if [ "$allownonascii" != "true" ] && + # Note that the use of brackets around a tr range is ok here, (it's + # even required, for portability to Solaris 10's /usr/bin/tr), since + # the square bracket bytes happen to fall in the designated range. + test $(git diff --cached --name-only --diff-filter=A -z $against | + LC_ALL=C tr -d '[ -~]\0' | wc -c) != 0 +then + cat <<\EOF +Error: Attempt to add a non-ASCII file name. + +This can cause problems if you want to work with people on other platforms. + +To be portable it is advisable to rename the file. + +If you know what you are doing you can disable this check using: + + git config hooks.allownonascii true +EOF + exit 1 +fi + +# If there are whitespace errors, print the offending file names and fail. +exec git diff-index --check --cached $against -- diff --git a/config/module/test-fixtures/basic-git/DOTgit/hooks/pre-push.sample b/config/module/test-fixtures/basic-git/DOTgit/hooks/pre-push.sample new file mode 100755 index 000000000..1f3bcebfd --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/hooks/pre-push.sample @@ -0,0 +1,54 @@ +#!/bin/sh + +# An example hook script to verify what is about to be pushed. Called by "git +# push" after it has checked the remote status, but before anything has been +# pushed. If this script exits with a non-zero status nothing will be pushed. +# +# This hook is called with the following parameters: +# +# $1 -- Name of the remote to which the push is being done +# $2 -- URL to which the push is being done +# +# If pushing without using a named remote those arguments will be equal. +# +# Information about the commits which are being pushed is supplied as lines to +# the standard input in the form: +# +# +# +# This sample shows how to prevent push of commits where the log message starts +# with "WIP" (work in progress). + +remote="$1" +url="$2" + +z40=0000000000000000000000000000000000000000 + +IFS=' ' +while read local_ref local_sha remote_ref remote_sha +do + if [ "$local_sha" = $z40 ] + then + # Handle delete + : + else + if [ "$remote_sha" = $z40 ] + then + # New branch, examine all commits + range="$local_sha" + else + # Update to existing branch, examine new commits + range="$remote_sha..$local_sha" + fi + + # Check for WIP commit + commit=`git rev-list -n 1 --grep '^WIP' "$range"` + if [ -n "$commit" ] + then + echo "Found WIP commit in $local_ref, not pushing" + exit 1 + fi + fi +done + +exit 0 diff --git a/config/module/test-fixtures/basic-git/DOTgit/hooks/pre-rebase.sample b/config/module/test-fixtures/basic-git/DOTgit/hooks/pre-rebase.sample new file mode 100755 index 000000000..9773ed4cb --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/hooks/pre-rebase.sample @@ -0,0 +1,169 @@ +#!/bin/sh +# +# Copyright (c) 2006, 2008 Junio C Hamano +# +# The "pre-rebase" hook is run just before "git rebase" starts doing +# its job, and can prevent the command from running by exiting with +# non-zero status. +# +# The hook is called with the following parameters: +# +# $1 -- the upstream the series was forked from. +# $2 -- the branch being rebased (or empty when rebasing the current branch). +# +# This sample shows how to prevent topic branches that are already +# merged to 'next' branch from getting rebased, because allowing it +# would result in rebasing already published history. + +publish=next +basebranch="$1" +if test "$#" = 2 +then + topic="refs/heads/$2" +else + topic=`git symbolic-ref HEAD` || + exit 0 ;# we do not interrupt rebasing detached HEAD +fi + +case "$topic" in +refs/heads/??/*) + ;; +*) + exit 0 ;# we do not interrupt others. + ;; +esac + +# Now we are dealing with a topic branch being rebased +# on top of master. Is it OK to rebase it? + +# Does the topic really exist? +git show-ref -q "$topic" || { + echo >&2 "No such branch $topic" + exit 1 +} + +# Is topic fully merged to master? +not_in_master=`git rev-list --pretty=oneline ^master "$topic"` +if test -z "$not_in_master" +then + echo >&2 "$topic is fully merged to master; better remove it." + exit 1 ;# we could allow it, but there is no point. +fi + +# Is topic ever merged to next? If so you should not be rebasing it. +only_next_1=`git rev-list ^master "^$topic" ${publish} | sort` +only_next_2=`git rev-list ^master ${publish} | sort` +if test "$only_next_1" = "$only_next_2" +then + not_in_topic=`git rev-list "^$topic" master` + if test -z "$not_in_topic" + then + echo >&2 "$topic is already up-to-date with master" + exit 1 ;# we could allow it, but there is no point. + else + exit 0 + fi +else + not_in_next=`git rev-list --pretty=oneline ^${publish} "$topic"` + /usr/bin/perl -e ' + my $topic = $ARGV[0]; + my $msg = "* $topic has commits already merged to public branch:\n"; + my (%not_in_next) = map { + /^([0-9a-f]+) /; + ($1 => 1); + } split(/\n/, $ARGV[1]); + for my $elem (map { + /^([0-9a-f]+) (.*)$/; + [$1 => $2]; + } split(/\n/, $ARGV[2])) { + if (!exists $not_in_next{$elem->[0]}) { + if ($msg) { + print STDERR $msg; + undef $msg; + } + print STDERR " $elem->[1]\n"; + } + } + ' "$topic" "$not_in_next" "$not_in_master" + exit 1 +fi + +exit 0 + +################################################################ + +This sample hook safeguards topic branches that have been +published from being rewound. + +The workflow assumed here is: + + * Once a topic branch forks from "master", "master" is never + merged into it again (either directly or indirectly). + + * Once a topic branch is fully cooked and merged into "master", + it is deleted. If you need to build on top of it to correct + earlier mistakes, a new topic branch is created by forking at + the tip of the "master". This is not strictly necessary, but + it makes it easier to keep your history simple. + + * Whenever you need to test or publish your changes to topic + branches, merge them into "next" branch. + +The script, being an example, hardcodes the publish branch name +to be "next", but it is trivial to make it configurable via +$GIT_DIR/config mechanism. + +With this workflow, you would want to know: + +(1) ... if a topic branch has ever been merged to "next". Young + topic branches can have stupid mistakes you would rather + clean up before publishing, and things that have not been + merged into other branches can be easily rebased without + affecting other people. But once it is published, you would + not want to rewind it. + +(2) ... if a topic branch has been fully merged to "master". + Then you can delete it. More importantly, you should not + build on top of it -- other people may already want to + change things related to the topic as patches against your + "master", so if you need further changes, it is better to + fork the topic (perhaps with the same name) afresh from the + tip of "master". + +Let's look at this example: + + o---o---o---o---o---o---o---o---o---o "next" + / / / / + / a---a---b A / / + / / / / + / / c---c---c---c B / + / / / \ / + / / / b---b C \ / + / / / / \ / + ---o---o---o---o---o---o---o---o---o---o---o "master" + + +A, B and C are topic branches. + + * A has one fix since it was merged up to "next". + + * B has finished. It has been fully merged up to "master" and "next", + and is ready to be deleted. + + * C has not merged to "next" at all. + +We would want to allow C to be rebased, refuse A, and encourage +B to be deleted. + +To compute (1): + + git rev-list ^master ^topic next + git rev-list ^master next + + if these match, topic has not merged in next at all. + +To compute (2): + + git rev-list master..topic + + if this is empty, it is fully merged to "master". diff --git a/config/module/test-fixtures/basic-git/DOTgit/hooks/prepare-commit-msg.sample b/config/module/test-fixtures/basic-git/DOTgit/hooks/prepare-commit-msg.sample new file mode 100755 index 000000000..f093a02ec --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/hooks/prepare-commit-msg.sample @@ -0,0 +1,36 @@ +#!/bin/sh +# +# An example hook script to prepare the commit log message. +# Called by "git commit" with the name of the file that has the +# commit message, followed by the description of the commit +# message's source. The hook's purpose is to edit the commit +# message file. If the hook fails with a non-zero status, +# the commit is aborted. +# +# To enable this hook, rename this file to "prepare-commit-msg". + +# This hook includes three examples. The first comments out the +# "Conflicts:" part of a merge commit. +# +# The second includes the output of "git diff --name-status -r" +# into the message, just before the "git status" output. It is +# commented because it doesn't cope with --amend or with squashed +# commits. +# +# The third example adds a Signed-off-by line to the message, that can +# still be edited. This is rarely a good idea. + +case "$2,$3" in + merge,) + /usr/bin/perl -i.bak -ne 's/^/# /, s/^# #/#/ if /^Conflicts/ .. /#/; print' "$1" ;; + +# ,|template,) +# /usr/bin/perl -i.bak -pe ' +# print "\n" . `git diff --cached --name-status -r` +# if /^#/ && $first++ == 0' "$1" ;; + + *) ;; +esac + +# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1" diff --git a/config/module/test-fixtures/basic-git/DOTgit/hooks/update.sample b/config/module/test-fixtures/basic-git/DOTgit/hooks/update.sample new file mode 100755 index 000000000..d84758373 --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/hooks/update.sample @@ -0,0 +1,128 @@ +#!/bin/sh +# +# An example hook script to blocks unannotated tags from entering. +# Called by "git receive-pack" with arguments: refname sha1-old sha1-new +# +# To enable this hook, rename this file to "update". +# +# Config +# ------ +# hooks.allowunannotated +# This boolean sets whether unannotated tags will be allowed into the +# repository. By default they won't be. +# hooks.allowdeletetag +# This boolean sets whether deleting tags will be allowed in the +# repository. By default they won't be. +# hooks.allowmodifytag +# This boolean sets whether a tag may be modified after creation. By default +# it won't be. +# hooks.allowdeletebranch +# This boolean sets whether deleting branches will be allowed in the +# repository. By default they won't be. +# hooks.denycreatebranch +# This boolean sets whether remotely creating branches will be denied +# in the repository. By default this is allowed. +# + +# --- Command line +refname="$1" +oldrev="$2" +newrev="$3" + +# --- Safety check +if [ -z "$GIT_DIR" ]; then + echo "Don't run this script from the command line." >&2 + echo " (if you want, you could supply GIT_DIR then run" >&2 + echo " $0 )" >&2 + exit 1 +fi + +if [ -z "$refname" -o -z "$oldrev" -o -z "$newrev" ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +# --- Config +allowunannotated=$(git config --bool hooks.allowunannotated) +allowdeletebranch=$(git config --bool hooks.allowdeletebranch) +denycreatebranch=$(git config --bool hooks.denycreatebranch) +allowdeletetag=$(git config --bool hooks.allowdeletetag) +allowmodifytag=$(git config --bool hooks.allowmodifytag) + +# check for no description +projectdesc=$(sed -e '1q' "$GIT_DIR/description") +case "$projectdesc" in +"Unnamed repository"* | "") + echo "*** Project description file hasn't been set" >&2 + exit 1 + ;; +esac + +# --- Check types +# if $newrev is 0000...0000, it's a commit to delete a ref. +zero="0000000000000000000000000000000000000000" +if [ "$newrev" = "$zero" ]; then + newrev_type=delete +else + newrev_type=$(git cat-file -t $newrev) +fi + +case "$refname","$newrev_type" in + refs/tags/*,commit) + # un-annotated tag + short_refname=${refname##refs/tags/} + if [ "$allowunannotated" != "true" ]; then + echo "*** The un-annotated tag, $short_refname, is not allowed in this repository" >&2 + echo "*** Use 'git tag [ -a | -s ]' for tags you want to propagate." >&2 + exit 1 + fi + ;; + refs/tags/*,delete) + # delete tag + if [ "$allowdeletetag" != "true" ]; then + echo "*** Deleting a tag is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/tags/*,tag) + # annotated tag + if [ "$allowmodifytag" != "true" ] && git rev-parse $refname > /dev/null 2>&1 + then + echo "*** Tag '$refname' already exists." >&2 + echo "*** Modifying a tag is not allowed in this repository." >&2 + exit 1 + fi + ;; + refs/heads/*,commit) + # branch + if [ "$oldrev" = "$zero" -a "$denycreatebranch" = "true" ]; then + echo "*** Creating a branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/heads/*,delete) + # delete branch + if [ "$allowdeletebranch" != "true" ]; then + echo "*** Deleting a branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/remotes/*,commit) + # tracking branch + ;; + refs/remotes/*,delete) + # delete tracking branch + if [ "$allowdeletebranch" != "true" ]; then + echo "*** Deleting a tracking branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + *) + # Anything else (is there anything else?) + echo "*** Update hook: unknown type of update to ref $refname of type $newrev_type" >&2 + exit 1 + ;; +esac + +# --- Finished +exit 0 diff --git a/config/module/test-fixtures/basic-git/DOTgit/index b/config/module/test-fixtures/basic-git/DOTgit/index new file mode 100644 index 0000000000000000000000000000000000000000..071f9d4396b0a2668a6251fb2c4e562646072487 GIT binary patch literal 104 zcmZ?q402{*U|<4b#t`xQ4nUd#Ml&)nurP!iUctc7xCAKu6(}VF#QGKn$%PT$OBV6n vIj~>Gmwm(j_9M0o?74}Vd3q&jKm~eSTpuh%?bn^LR?io^x_jLU`_j_@uMZss literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-git/DOTgit/info/exclude b/config/module/test-fixtures/basic-git/DOTgit/info/exclude new file mode 100644 index 000000000..a5196d1be --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/info/exclude @@ -0,0 +1,6 @@ +# git ls-files --others --exclude-from=.git/info/exclude +# Lines that start with '#' are comments. +# For a project mostly in C, the following would be a good set of +# exclude patterns (uncomment them if you want to use them): +# *.[oa] +# *~ diff --git a/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD b/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD new file mode 100644 index 000000000..ad39b8a0f --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD @@ -0,0 +1 @@ +0000000000000000000000000000000000000000 497bc37401eb3c9b11865b1768725b64066eccee Mitchell Hashimoto 1410850637 -0700 commit (initial): A commit diff --git a/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master b/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master new file mode 100644 index 000000000..ad39b8a0f --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master @@ -0,0 +1 @@ +0000000000000000000000000000000000000000 497bc37401eb3c9b11865b1768725b64066eccee Mitchell Hashimoto 1410850637 -0700 commit (initial): A commit diff --git a/config/module/test-fixtures/basic-git/DOTgit/objects/38/30637158f774a20edcc0bf1c4d07b0bf87c43d b/config/module/test-fixtures/basic-git/DOTgit/objects/38/30637158f774a20edcc0bf1c4d07b0bf87c43d new file mode 100644 index 0000000000000000000000000000000000000000..ef8ebf7282975b2da2da3cfe6cef1477fcca395c GIT binary patch literal 59 zcmV-B0L1@z0ZYosPf{>3XHZt~NX^N~=ix_U-g&xG EByA >/)}kT….ѸSl HjqH %D \ No newline at end of file diff --git a/config/module/test-fixtures/basic-git/DOTgit/objects/96/43088174e25a9bd91c27970a580af0085c9f32 b/config/module/test-fixtures/basic-git/DOTgit/objects/96/43088174e25a9bd91c27970a580af0085c9f32 new file mode 100644 index 0000000000000000000000000000000000000000..387943288da570c5373bc4f0cb7b8005536653e7 GIT binary patch literal 52 zcmV-40L%Y)0V^p=O;s>9WiT`_Ff%bx$W6@5(<@11urNq2jQC!%i0{sU{W8An8}_#! Ku>}Anzz?PeB^DI` literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-git/DOTgit/refs/heads/master b/config/module/test-fixtures/basic-git/DOTgit/refs/heads/master new file mode 100644 index 000000000..4f8bf4274 --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/refs/heads/master @@ -0,0 +1 @@ +497bc37401eb3c9b11865b1768725b64066eccee diff --git a/config/module/test-fixtures/basic-git/main.tf b/config/module/test-fixtures/basic-git/main.tf new file mode 100644 index 000000000..383063715 --- /dev/null +++ b/config/module/test-fixtures/basic-git/main.tf @@ -0,0 +1,5 @@ +# Hello + +module "foo" { + source = "./foo" +} From 7e94f7d4a912cf1ce35c1822e0c65a16bac1ef50 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Sep 2014 09:30:31 -0700 Subject: [PATCH 31/41] config/module: Mercurial support --- config/module/get.go | 1 + config/module/get_hg.go | 51 ++++++++++++++++++ config/module/get_hg_test.go | 41 ++++++++++++++ .../test-fixtures/basic-hg/.hg/00changelog.i | Bin 0 -> 57 bytes .../basic-hg/.hg/cache/branch2-served | 2 + .../test-fixtures/basic-hg/.hg/dirstate | Bin 0 -> 64 bytes .../basic-hg/.hg/last-message.txt | 2 + .../test-fixtures/basic-hg/.hg/requires | 4 ++ .../basic-hg/.hg/store/00changelog.i | Bin 0 -> 176 bytes .../basic-hg/.hg/store/00manifest.i | Bin 0 -> 114 bytes .../basic-hg/.hg/store/data/main.tf.i | Bin 0 -> 112 bytes .../test-fixtures/basic-hg/.hg/store/fncache | 1 + .../basic-hg/.hg/store/phaseroots | 1 + .../test-fixtures/basic-hg/.hg/store/undo | Bin 0 -> 58 bytes .../basic-hg/.hg/store/undo.phaseroots | 0 .../test-fixtures/basic-hg/.hg/undo.bookmarks | 0 .../test-fixtures/basic-hg/.hg/undo.branch | 1 + .../test-fixtures/basic-hg/.hg/undo.desc | 2 + .../test-fixtures/basic-hg/.hg/undo.dirstate | Bin 0 -> 64 bytes config/module/test-fixtures/basic-hg/main.tf | 5 ++ 20 files changed, 111 insertions(+) create mode 100644 config/module/get_hg.go create mode 100644 config/module/get_hg_test.go create mode 100644 config/module/test-fixtures/basic-hg/.hg/00changelog.i create mode 100644 config/module/test-fixtures/basic-hg/.hg/cache/branch2-served create mode 100644 config/module/test-fixtures/basic-hg/.hg/dirstate create mode 100644 config/module/test-fixtures/basic-hg/.hg/last-message.txt create mode 100644 config/module/test-fixtures/basic-hg/.hg/requires create mode 100644 config/module/test-fixtures/basic-hg/.hg/store/00changelog.i create mode 100644 config/module/test-fixtures/basic-hg/.hg/store/00manifest.i create mode 100644 config/module/test-fixtures/basic-hg/.hg/store/data/main.tf.i create mode 100644 config/module/test-fixtures/basic-hg/.hg/store/fncache create mode 100644 config/module/test-fixtures/basic-hg/.hg/store/phaseroots create mode 100644 config/module/test-fixtures/basic-hg/.hg/store/undo create mode 100644 config/module/test-fixtures/basic-hg/.hg/store/undo.phaseroots create mode 100644 config/module/test-fixtures/basic-hg/.hg/undo.bookmarks create mode 100644 config/module/test-fixtures/basic-hg/.hg/undo.branch create mode 100644 config/module/test-fixtures/basic-hg/.hg/undo.desc create mode 100644 config/module/test-fixtures/basic-hg/.hg/undo.dirstate create mode 100644 config/module/test-fixtures/basic-hg/main.tf diff --git a/config/module/get.go b/config/module/get.go index 9a459760c..5131d7384 100644 --- a/config/module/get.go +++ b/config/module/get.go @@ -33,6 +33,7 @@ func init() { Getters = map[string]Getter{ "file": new(FileGetter), "git": new(GitGetter), + "hg": new(HgGetter), } } diff --git a/config/module/get_hg.go b/config/module/get_hg.go new file mode 100644 index 000000000..2bad0143a --- /dev/null +++ b/config/module/get_hg.go @@ -0,0 +1,51 @@ +package module + +import ( + "fmt" + "net/url" + "os" + "os/exec" +) + +// HgGetter is a Getter implementation that will download a module from +// a Mercurial repository. +type HgGetter struct{} + +func (g *HgGetter) Get(dst string, u *url.URL) error { + if _, err := exec.LookPath("hg"); err != nil { + return fmt.Errorf("hg must be available and on the PATH") + } + + _, err := os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return err + } + if err != nil { + if err := g.clone(dst, u); err != nil { + return err + } + } + + if err:= g.pull(dst, u); err != nil { + return err + } + + return g.update(dst, u) +} + +func (g *HgGetter) clone(dst string, u *url.URL) error { + cmd := exec.Command("hg", "clone", "-U", u.String(), dst) + return getRunCommand(cmd) +} + +func (g *HgGetter) pull(dst string, u *url.URL) error { + cmd := exec.Command("hg", "pull") + cmd.Dir = dst + return getRunCommand(cmd) +} + +func (g *HgGetter) update(dst string, u *url.URL) error { + cmd := exec.Command("hg", "update") + cmd.Dir = dst + return getRunCommand(cmd) +} diff --git a/config/module/get_hg_test.go b/config/module/get_hg_test.go new file mode 100644 index 000000000..71631a715 --- /dev/null +++ b/config/module/get_hg_test.go @@ -0,0 +1,41 @@ +package module + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +var testHasHg bool + +func init() { + if _, err := exec.LookPath("hg"); err == nil { + testHasHg = true + } +} + +func TestHgGetter_impl(t *testing.T) { + var _ Getter = new(HgGetter) +} + +func TestHgGetter(t *testing.T) { + if !testHasHg { + t.Log("hg not found, skipping") + t.Skip() + } + + g := new(HgGetter) + dst := tempDir(t) + + // With a dir that doesn't exist + if err := g.Get(dst, testModuleURL("basic-hg")); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath := filepath.Join(dst, "main.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} diff --git a/config/module/test-fixtures/basic-hg/.hg/00changelog.i b/config/module/test-fixtures/basic-hg/.hg/00changelog.i new file mode 100644 index 0000000000000000000000000000000000000000..d3a8311050e54c57c5be7cfe169e60a95768812c GIT binary patch literal 57 zcmWN_K?=Yi3=^>}vhYRG!~=>ejxFxg7V^;JV-L`jXd&PhOer=+rTh@j}%%hBW{9?W+v?SN{IR J{@{bE4*+3SNfH16 literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-hg/.hg/store/00manifest.i b/config/module/test-fixtures/basic-hg/.hg/store/00manifest.i new file mode 100644 index 0000000000000000000000000000000000000000..25ad8dc2a4d87b479772db8075e653f37c0a1e30 GIT binary patch literal 114 zcmZQzWME`~03#q}2xT+;hXRIf+oy9R`GtjZS|7jmaf#kx&QzG literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-hg/.hg/store/data/main.tf.i b/config/module/test-fixtures/basic-hg/.hg/store/data/main.tf.i new file mode 100644 index 0000000000000000000000000000000000000000..f45ddc33f19db6ef2a8f723ce4e36e10db12a436 GIT binary patch literal 112 zcmZQzWME`~00SVU4`nm_hk{)i%ZqQMPS2Zr>zb=wCx2OxuW=Ym4TvsPR`5v8$;s#9 n%FRzH%}G^IO3TkzQmE!q0D|KD(xT*41zQCrJ$;ZcS1lI+ym=or literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-hg/.hg/store/fncache b/config/module/test-fixtures/basic-hg/.hg/store/fncache new file mode 100644 index 000000000..af601aa05 --- /dev/null +++ b/config/module/test-fixtures/basic-hg/.hg/store/fncache @@ -0,0 +1 @@ +data/main.tf.i diff --git a/config/module/test-fixtures/basic-hg/.hg/store/phaseroots b/config/module/test-fixtures/basic-hg/.hg/store/phaseroots new file mode 100644 index 000000000..a08565294 --- /dev/null +++ b/config/module/test-fixtures/basic-hg/.hg/store/phaseroots @@ -0,0 +1 @@ +1 dcaed7754d58264cb9a5916215a5442377307bd1 diff --git a/config/module/test-fixtures/basic-hg/.hg/store/undo b/config/module/test-fixtures/basic-hg/.hg/store/undo new file mode 100644 index 0000000000000000000000000000000000000000..9dd8c80976c979b46d2068ac7111685daae78167 GIT binary patch literal 58 zcmYdEEJ@VQP0Y;GD@oJKWH8_|Fvv~J%S=lxE`f5BGZORCQ*-ju!Thwmlt D9$*ox literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-hg/.hg/store/undo.phaseroots b/config/module/test-fixtures/basic-hg/.hg/store/undo.phaseroots new file mode 100644 index 000000000..e69de29bb diff --git a/config/module/test-fixtures/basic-hg/.hg/undo.bookmarks b/config/module/test-fixtures/basic-hg/.hg/undo.bookmarks new file mode 100644 index 000000000..e69de29bb diff --git a/config/module/test-fixtures/basic-hg/.hg/undo.branch b/config/module/test-fixtures/basic-hg/.hg/undo.branch new file mode 100644 index 000000000..331d858ce --- /dev/null +++ b/config/module/test-fixtures/basic-hg/.hg/undo.branch @@ -0,0 +1 @@ +default \ No newline at end of file diff --git a/config/module/test-fixtures/basic-hg/.hg/undo.desc b/config/module/test-fixtures/basic-hg/.hg/undo.desc new file mode 100644 index 000000000..37970a278 --- /dev/null +++ b/config/module/test-fixtures/basic-hg/.hg/undo.desc @@ -0,0 +1,2 @@ +0 +commit diff --git a/config/module/test-fixtures/basic-hg/.hg/undo.dirstate b/config/module/test-fixtures/basic-hg/.hg/undo.dirstate new file mode 100644 index 0000000000000000000000000000000000000000..b48d181be69e29d1966256b75b6883b252674ee0 GIT binary patch literal 64 acmZQzAPyvgl>LVSAd@{eF*8rEBn<$M@(T_C literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-hg/main.tf b/config/module/test-fixtures/basic-hg/main.tf new file mode 100644 index 000000000..383063715 --- /dev/null +++ b/config/module/test-fixtures/basic-hg/main.tf @@ -0,0 +1,5 @@ +# Hello + +module "foo" { + source = "./foo" +} From dcb900470c989038d77637e051d2c412e53643c9 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Sep 2014 09:55:51 -0700 Subject: [PATCH 32/41] config/module: git supports tags --- config/module/get_git.go | 27 +++++++++++++- config/module/get_git_test.go | 35 ++++++++++++++++++ .../basic-git/DOTgit/COMMIT_EDITMSG | 7 +--- .../test-fixtures/basic-git/DOTgit/logs/HEAD | 2 + .../basic-git/DOTgit/logs/refs/heads/master | 2 + .../1f/31e97f053caeb5d6b7bffa3faf82941c99efa2 | Bin 0 -> 167 bytes .../24/3f0fc5c4e586d1a3daa54c981b6f34e9ab1085 | Bin 0 -> 164 bytes .../b7/757b6a3696ad036e9aa2f5b4856d09e7f17993 | Bin 0 -> 82 bytes .../e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391 | Bin 0 -> 15 bytes .../basic-git/DOTgit/refs/heads/master | 2 +- .../basic-git/DOTgit/refs/tags/v1.0 | 1 + 11 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 config/module/test-fixtures/basic-git/DOTgit/objects/1f/31e97f053caeb5d6b7bffa3faf82941c99efa2 create mode 100644 config/module/test-fixtures/basic-git/DOTgit/objects/24/3f0fc5c4e586d1a3daa54c981b6f34e9ab1085 create mode 100644 config/module/test-fixtures/basic-git/DOTgit/objects/b7/757b6a3696ad036e9aa2f5b4856d09e7f17993 create mode 100644 config/module/test-fixtures/basic-git/DOTgit/objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391 create mode 100644 config/module/test-fixtures/basic-git/DOTgit/refs/tags/v1.0 diff --git a/config/module/get_git.go b/config/module/get_git.go index 132bd8e11..0507bf344 100644 --- a/config/module/get_git.go +++ b/config/module/get_git.go @@ -16,15 +16,38 @@ func (g *GitGetter) Get(dst string, u *url.URL) error { return fmt.Errorf("git must be available and on the PATH") } + // Extract some query parameters we use + q := u.Query() + tag := q.Get("tag") + q.Del("tag") + u.RawQuery = q.Encode() + + // First: clone or update the repository _, err := os.Stat(dst) if err != nil && !os.IsNotExist(err) { return err } if err == nil { - return g.update(dst, u) + err = g.update(dst, u) + } else { + err = g.clone(dst, u) + } + if err != nil { + return err } - return g.clone(dst, u) + // Next: check out the proper tag/branch if it is specified, and checkout + if tag == "" { + return nil + } + + return g.checkout(dst, tag) +} + +func (g *GitGetter) checkout(dst string, ref string) error { + cmd := exec.Command("git", "checkout", ref) + cmd.Dir = dst + return getRunCommand(cmd) } func (g *GitGetter) clone(dst string, u *url.URL) error { diff --git a/config/module/get_git_test.go b/config/module/get_git_test.go index 33a0a98a6..4c81bbbe1 100644 --- a/config/module/get_git_test.go +++ b/config/module/get_git_test.go @@ -49,3 +49,38 @@ func TestGitGetter(t *testing.T) { t.Fatalf("err: %s", err) } } + +func TestGitGetter_tag(t *testing.T) { + if !testHasGit { + t.Log("git not found, skipping") + t.Skip() + } + + g := new(GitGetter) + dst := tempDir(t) + + // Git doesn't allow nested ".git" directories so we do some hackiness + // here to get around that... + moduleDir := filepath.Join(fixtureDir, "basic-git") + oldName := filepath.Join(moduleDir, "DOTgit") + newName := filepath.Join(moduleDir, ".git") + if err := os.Rename(oldName, newName); err != nil { + t.Fatalf("err: %s", err) + } + defer os.Rename(newName, oldName) + + url := testModuleURL("basic-git") + q := url.Query() + q.Add("tag", "v1.0") + url.RawQuery = q.Encode() + + if err := g.Get(dst, url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath := filepath.Join(dst, "main_tag1.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} diff --git a/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG b/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG index fe6bc2b3d..dd932c30f 100644 --- a/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG +++ b/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG @@ -1,10 +1,7 @@ -A commit +remove tag1 # Please enter the commit message for your changes. Lines starting # with '#' will be ignored, and an empty message aborts the commit. # On branch master -# -# Initial commit -# # Changes to be committed: -# new file: main.tf +# deleted: main_tag1.tf # diff --git a/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD b/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD index ad39b8a0f..396932ba1 100644 --- a/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD +++ b/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD @@ -1 +1,3 @@ 0000000000000000000000000000000000000000 497bc37401eb3c9b11865b1768725b64066eccee Mitchell Hashimoto 1410850637 -0700 commit (initial): A commit +497bc37401eb3c9b11865b1768725b64066eccee 243f0fc5c4e586d1a3daa54c981b6f34e9ab1085 Mitchell Hashimoto 1410886526 -0700 commit: tag1 +243f0fc5c4e586d1a3daa54c981b6f34e9ab1085 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 Mitchell Hashimoto 1410886536 -0700 commit: remove tag1 diff --git a/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master b/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master index ad39b8a0f..396932ba1 100644 --- a/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master +++ b/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master @@ -1 +1,3 @@ 0000000000000000000000000000000000000000 497bc37401eb3c9b11865b1768725b64066eccee Mitchell Hashimoto 1410850637 -0700 commit (initial): A commit +497bc37401eb3c9b11865b1768725b64066eccee 243f0fc5c4e586d1a3daa54c981b6f34e9ab1085 Mitchell Hashimoto 1410886526 -0700 commit: tag1 +243f0fc5c4e586d1a3daa54c981b6f34e9ab1085 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 Mitchell Hashimoto 1410886536 -0700 commit: remove tag1 diff --git a/config/module/test-fixtures/basic-git/DOTgit/objects/1f/31e97f053caeb5d6b7bffa3faf82941c99efa2 b/config/module/test-fixtures/basic-git/DOTgit/objects/1f/31e97f053caeb5d6b7bffa3faf82941c99efa2 new file mode 100644 index 0000000000000000000000000000000000000000..5793a840b76235dafcb144ed66d1b8d274ed76bf GIT binary patch literal 167 zcmV;Y09gNc0jde)V4 V5S{)q#u}CSCf;!s>;oX7Q=N++Phyo>KhY@))4ka?%IV3*@oJ1_}!Me>)YD$BHN~~{94vhTh*t2;A_M3v&+5kx(&IOvjz`l>`{yQXvi4V SwJ)0dC8iqRL45!N?@<9%)ls?t literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-git/DOTgit/objects/b7/757b6a3696ad036e9aa2f5b4856d09e7f17993 b/config/module/test-fixtures/basic-git/DOTgit/objects/b7/757b6a3696ad036e9aa2f5b4856d09e7f17993 new file mode 100644 index 0000000000000000000000000000000000000000..10192566594b6bfa1e9d04c558444006bb2f27fe GIT binary patch literal 82 zcmV-Y0ImOc0V^p=O;s>AWiT`_Ff%bx$W6@5(<@11urNq2jQC!%i0{sU{W8An8}_#! ou|-l6Uy_(^2vYZK?xWe8E?#r??$%sa9(Ci;lb+y-0LsiEdb9E;RsaA1 literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-git/DOTgit/objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391 b/config/module/test-fixtures/basic-git/DOTgit/objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391 new file mode 100644 index 0000000000000000000000000000000000000000..711223894375fe1186ac5bfffdc48fb1fa1e65cc GIT binary patch literal 15 Wcmb Date: Tue, 16 Sep 2014 09:59:09 -0700 Subject: [PATCH 33/41] config/module: fix some issues where tag re-pulling didnt' work --- config/module/get_git.go | 18 +++++++++++++++--- config/module/get_git_test.go | 11 +++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/config/module/get_git.go b/config/module/get_git.go index 0507bf344..b28122230 100644 --- a/config/module/get_git.go +++ b/config/module/get_git.go @@ -17,10 +17,17 @@ func (g *GitGetter) Get(dst string, u *url.URL) error { } // Extract some query parameters we use + var tag string q := u.Query() - tag := q.Get("tag") - q.Del("tag") - u.RawQuery = q.Encode() + if len(q) > 0 { + tag = q.Get("tag") + q.Del("tag") + + // Copy the URL + var newU url.URL = *u + u = &newU + u.RawQuery = q.Encode() + } // First: clone or update the repository _, err := os.Stat(dst) @@ -56,6 +63,11 @@ func (g *GitGetter) clone(dst string, u *url.URL) error { } func (g *GitGetter) update(dst string, u *url.URL) error { + // We have to be on a branch to pull + if err := g.checkout(dst, "master"); err != nil { + return err + } + cmd := exec.Command("git", "pull", "--ff-only") cmd.Dir = dst return getRunCommand(cmd) diff --git a/config/module/get_git_test.go b/config/module/get_git_test.go index 4c81bbbe1..598444b6d 100644 --- a/config/module/get_git_test.go +++ b/config/module/get_git_test.go @@ -83,4 +83,15 @@ func TestGitGetter_tag(t *testing.T) { if _, err := os.Stat(mainPath); err != nil { t.Fatalf("err: %s", err) } + + // Get again should work + if err := g.Get(dst, url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath = filepath.Join(dst, "main_tag1.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } } From ac19a488d23ce0937faa5960c332286dade45644 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Sep 2014 10:02:11 -0700 Subject: [PATCH 34/41] config/module: support branches in git --- config/module/get_git.go | 10 ++-- config/module/get_git_test.go | 48 +++++++++++++++++- .../basic-git/DOTgit/COMMIT_EDITMSG | 6 +-- .../test-fixtures/basic-git/DOTgit/logs/HEAD | 3 ++ .../DOTgit/logs/refs/heads/test-branch | 2 + .../40/4618c9d96dfa0a5d365b518e0dfbb5a387c649 | Bin 0 -> 84 bytes .../7b/7614f8759ac8b5e4b02be65ad8e2667be6dd87 | 2 + .../basic-git/DOTgit/refs/heads/test-branch | 1 + 8 files changed, 63 insertions(+), 9 deletions(-) create mode 100644 config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/test-branch create mode 100644 config/module/test-fixtures/basic-git/DOTgit/objects/40/4618c9d96dfa0a5d365b518e0dfbb5a387c649 create mode 100644 config/module/test-fixtures/basic-git/DOTgit/objects/7b/7614f8759ac8b5e4b02be65ad8e2667be6dd87 create mode 100644 config/module/test-fixtures/basic-git/DOTgit/refs/heads/test-branch diff --git a/config/module/get_git.go b/config/module/get_git.go index b28122230..5ab27ba0b 100644 --- a/config/module/get_git.go +++ b/config/module/get_git.go @@ -17,11 +17,11 @@ func (g *GitGetter) Get(dst string, u *url.URL) error { } // Extract some query parameters we use - var tag string + var ref string q := u.Query() if len(q) > 0 { - tag = q.Get("tag") - q.Del("tag") + ref = q.Get("ref") + q.Del("ref") // Copy the URL var newU url.URL = *u @@ -44,11 +44,11 @@ func (g *GitGetter) Get(dst string, u *url.URL) error { } // Next: check out the proper tag/branch if it is specified, and checkout - if tag == "" { + if ref == "" { return nil } - return g.checkout(dst, tag) + return g.checkout(dst, ref) } func (g *GitGetter) checkout(dst string, ref string) error { diff --git a/config/module/get_git_test.go b/config/module/get_git_test.go index 598444b6d..3885ff8e7 100644 --- a/config/module/get_git_test.go +++ b/config/module/get_git_test.go @@ -50,6 +50,52 @@ func TestGitGetter(t *testing.T) { } } +func TestGitGetter_branch(t *testing.T) { + if !testHasGit { + t.Log("git not found, skipping") + t.Skip() + } + + g := new(GitGetter) + dst := tempDir(t) + + // Git doesn't allow nested ".git" directories so we do some hackiness + // here to get around that... + moduleDir := filepath.Join(fixtureDir, "basic-git") + oldName := filepath.Join(moduleDir, "DOTgit") + newName := filepath.Join(moduleDir, ".git") + if err := os.Rename(oldName, newName); err != nil { + t.Fatalf("err: %s", err) + } + defer os.Rename(newName, oldName) + + url := testModuleURL("basic-git") + q := url.Query() + q.Add("ref", "test-branch") + url.RawQuery = q.Encode() + + if err := g.Get(dst, url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath := filepath.Join(dst, "main_branch.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } + + // Get again should work + if err := g.Get(dst, url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath = filepath.Join(dst, "main_branch.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} + func TestGitGetter_tag(t *testing.T) { if !testHasGit { t.Log("git not found, skipping") @@ -71,7 +117,7 @@ func TestGitGetter_tag(t *testing.T) { url := testModuleURL("basic-git") q := url.Query() - q.Add("tag", "v1.0") + q.Add("ref", "v1.0") url.RawQuery = q.Encode() if err := g.Get(dst, url); err != nil { diff --git a/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG b/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG index dd932c30f..d13fed6c9 100644 --- a/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG +++ b/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG @@ -1,7 +1,7 @@ -remove tag1 +Branch # Please enter the commit message for your changes. Lines starting # with '#' will be ignored, and an empty message aborts the commit. -# On branch master +# On branch test-branch # Changes to be committed: -# deleted: main_tag1.tf +# new file: main_branch.tf # diff --git a/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD b/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD index 396932ba1..40709bc8e 100644 --- a/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD +++ b/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD @@ -1,3 +1,6 @@ 0000000000000000000000000000000000000000 497bc37401eb3c9b11865b1768725b64066eccee Mitchell Hashimoto 1410850637 -0700 commit (initial): A commit 497bc37401eb3c9b11865b1768725b64066eccee 243f0fc5c4e586d1a3daa54c981b6f34e9ab1085 Mitchell Hashimoto 1410886526 -0700 commit: tag1 243f0fc5c4e586d1a3daa54c981b6f34e9ab1085 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 Mitchell Hashimoto 1410886536 -0700 commit: remove tag1 +1f31e97f053caeb5d6b7bffa3faf82941c99efa2 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 Mitchell Hashimoto 1410886909 -0700 checkout: moving from master to test-branch +1f31e97f053caeb5d6b7bffa3faf82941c99efa2 7b7614f8759ac8b5e4b02be65ad8e2667be6dd87 Mitchell Hashimoto 1410886913 -0700 commit: Branch +7b7614f8759ac8b5e4b02be65ad8e2667be6dd87 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 Mitchell Hashimoto 1410886916 -0700 checkout: moving from test-branch to master diff --git a/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/test-branch b/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/test-branch new file mode 100644 index 000000000..937067a2a --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/test-branch @@ -0,0 +1,2 @@ +0000000000000000000000000000000000000000 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 Mitchell Hashimoto 1410886909 -0700 branch: Created from HEAD +1f31e97f053caeb5d6b7bffa3faf82941c99efa2 7b7614f8759ac8b5e4b02be65ad8e2667be6dd87 Mitchell Hashimoto 1410886913 -0700 commit: Branch diff --git a/config/module/test-fixtures/basic-git/DOTgit/objects/40/4618c9d96dfa0a5d365b518e0dfbb5a387c649 b/config/module/test-fixtures/basic-git/DOTgit/objects/40/4618c9d96dfa0a5d365b518e0dfbb5a387c649 new file mode 100644 index 0000000000000000000000000000000000000000..434fcab2093d5da8ea874dbbd26efa678cbfef10 GIT binary patch literal 84 zcmV-a0IUCa0V^p=O;s>AXD~D{Ff%bx$W6@5(<@11urNq2jQC!%i0{sU{W8An8}_#! qu|-l6pH!5Xmz)7o`E2f^*_$q2bN24mTvr}-l&`&Qdр.Y:nKR#aT&"(s23(2Ru7xi?򨰬Sj̥{G`k-S \ No newline at end of file diff --git a/config/module/test-fixtures/basic-git/DOTgit/refs/heads/test-branch b/config/module/test-fixtures/basic-git/DOTgit/refs/heads/test-branch new file mode 100644 index 000000000..a5f298b83 --- /dev/null +++ b/config/module/test-fixtures/basic-git/DOTgit/refs/heads/test-branch @@ -0,0 +1 @@ +7b7614f8759ac8b5e4b02be65ad8e2667be6dd87 From feb9a365976cc59dcbf21bb0835f67b8c3ccaaf1 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Sep 2014 10:06:44 -0700 Subject: [PATCH 35/41] config/module: tests to verify that params are preserved on files --- config/module/detect_file_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/module/detect_file_test.go b/config/module/detect_file_test.go index 6ccd49f4f..02a6ccec2 100644 --- a/config/module/detect_file_test.go +++ b/config/module/detect_file_test.go @@ -10,8 +10,10 @@ func TestFileDetector(t *testing.T) { Output string }{ {"./foo", "file:///pwd/foo"}, + {"./foo?foo=bar", "file:///pwd/foo?foo=bar"}, {"foo", "file:///pwd/foo"}, {"/foo", "file:///foo"}, + {"/foo?bar=baz", "file:///foo?bar=baz"}, } pwd := "/pwd" From 9c74d6b5c089636790fcd30302950e12913c1368 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Sep 2014 10:30:55 -0700 Subject: [PATCH 36/41] config/module: hg supports branches/tags/etc. --- config/module/get_hg.go | 24 ++++++++++-- config/module/get_hg_test.go | 36 ++++++++++++++++++ .../module/test-fixtures/basic-hg/.hg/branch | 1 + .../basic-hg/.hg/cache/branch2-served | 3 +- .../test-fixtures/basic-hg/.hg/cache/tags | 2 + .../basic-hg/.hg/last-message.txt | 2 +- .../basic-hg/.hg/store/00changelog.i | Bin 176 -> 355 bytes .../basic-hg/.hg/store/00manifest.i | Bin 114 -> 246 bytes .../basic-hg/.hg/store/data/main__branch.tf.i | Bin 0 -> 64 bytes .../test-fixtures/basic-hg/.hg/store/fncache | 1 + .../test-fixtures/basic-hg/.hg/store/undo | Bin 58 -> 59 bytes .../basic-hg/.hg/store/undo.phaseroots | 1 + .../test-fixtures/basic-hg/.hg/undo.branch | 2 +- .../test-fixtures/basic-hg/.hg/undo.desc | 2 +- .../test-fixtures/basic-hg/.hg/undo.dirstate | Bin 64 -> 95 bytes 15 files changed, 67 insertions(+), 7 deletions(-) create mode 100644 config/module/test-fixtures/basic-hg/.hg/branch create mode 100644 config/module/test-fixtures/basic-hg/.hg/cache/tags create mode 100644 config/module/test-fixtures/basic-hg/.hg/store/data/main__branch.tf.i diff --git a/config/module/get_hg.go b/config/module/get_hg.go index 2bad0143a..19e4abd5a 100644 --- a/config/module/get_hg.go +++ b/config/module/get_hg.go @@ -16,6 +16,19 @@ func (g *HgGetter) Get(dst string, u *url.URL) error { return fmt.Errorf("hg must be available and on the PATH") } + // Extract some query parameters we use + var rev string + q := u.Query() + if len(q) > 0 { + rev = q.Get("rev") + q.Del("rev") + + // Copy the URL + var newU url.URL = *u + u = &newU + u.RawQuery = q.Encode() + } + _, err := os.Stat(dst) if err != nil && !os.IsNotExist(err) { return err @@ -30,7 +43,7 @@ func (g *HgGetter) Get(dst string, u *url.URL) error { return err } - return g.update(dst, u) + return g.update(dst, u, rev) } func (g *HgGetter) clone(dst string, u *url.URL) error { @@ -44,8 +57,13 @@ func (g *HgGetter) pull(dst string, u *url.URL) error { return getRunCommand(cmd) } -func (g *HgGetter) update(dst string, u *url.URL) error { - cmd := exec.Command("hg", "update") +func (g *HgGetter) update(dst string, u *url.URL, rev string) error { + args := []string{"update"} + if rev != "" { + args = append(args, rev) + } + + cmd := exec.Command("hg", args...) cmd.Dir = dst return getRunCommand(cmd) } diff --git a/config/module/get_hg_test.go b/config/module/get_hg_test.go index 71631a715..02260bd29 100644 --- a/config/module/get_hg_test.go +++ b/config/module/get_hg_test.go @@ -39,3 +39,39 @@ func TestHgGetter(t *testing.T) { t.Fatalf("err: %s", err) } } + +func TestHgGetter_branch(t *testing.T) { + if !testHasHg { + t.Log("hg not found, skipping") + t.Skip() + } + + g := new(HgGetter) + dst := tempDir(t) + + url := testModuleURL("basic-hg") + q := url.Query() + q.Add("rev", "test-branch") + url.RawQuery = q.Encode() + + if err := g.Get(dst, url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath := filepath.Join(dst, "main_branch.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } + + // Get again should work + if err := g.Get(dst, url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath = filepath.Join(dst, "main_branch.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} diff --git a/config/module/test-fixtures/basic-hg/.hg/branch b/config/module/test-fixtures/basic-hg/.hg/branch new file mode 100644 index 000000000..4ad96d515 --- /dev/null +++ b/config/module/test-fixtures/basic-hg/.hg/branch @@ -0,0 +1 @@ +default diff --git a/config/module/test-fixtures/basic-hg/.hg/cache/branch2-served b/config/module/test-fixtures/basic-hg/.hg/cache/branch2-served index 1bae0f6bc..f2a9aae94 100644 --- a/config/module/test-fixtures/basic-hg/.hg/cache/branch2-served +++ b/config/module/test-fixtures/basic-hg/.hg/cache/branch2-served @@ -1,2 +1,3 @@ -dcaed7754d58264cb9a5916215a5442377307bd1 0 +c65e998d747ffbb1fe3b1c067a50664bb3fb5da4 1 dcaed7754d58264cb9a5916215a5442377307bd1 o default +c65e998d747ffbb1fe3b1c067a50664bb3fb5da4 o test-branch diff --git a/config/module/test-fixtures/basic-hg/.hg/cache/tags b/config/module/test-fixtures/basic-hg/.hg/cache/tags new file mode 100644 index 000000000..b30a3de43 --- /dev/null +++ b/config/module/test-fixtures/basic-hg/.hg/cache/tags @@ -0,0 +1,2 @@ +1 c65e998d747ffbb1fe3b1c067a50664bb3fb5da4 + diff --git a/config/module/test-fixtures/basic-hg/.hg/last-message.txt b/config/module/test-fixtures/basic-hg/.hg/last-message.txt index e8f25bcfc..a24e1a3f2 100644 --- a/config/module/test-fixtures/basic-hg/.hg/last-message.txt +++ b/config/module/test-fixtures/basic-hg/.hg/last-message.txt @@ -1,2 +1,2 @@ -Commit +Branch diff --git a/config/module/test-fixtures/basic-hg/.hg/store/00changelog.i b/config/module/test-fixtures/basic-hg/.hg/store/00changelog.i index 64e5cb6dc9b81f41f4470c00d3b17c0424a73be2..b3dc2666acac6fdf0f5d9b83446f8a8ef851eb26 100644 GIT binary patch delta 187 zcmdnM_?T(JhI$4NC;*eiKyoUW#lXM_qW=TIvACJNCH22I{)3ie=nnL2Hx8|Hkx>uc^erTd2TW=^xNd)v&F;6(C|5v zRv+6VAq6ut6GM{?8%#DBn3{(g7?~Ow1bAq9o$)@mLCaHD?_%qQokhaNpC2ja?44vX U;aiZxI<_P;^$i*frM&Wy0CMs?FaQ7m delta 6 NcmaFNw1IKL1^@~50@nZl diff --git a/config/module/test-fixtures/basic-hg/.hg/store/00manifest.i b/config/module/test-fixtures/basic-hg/.hg/store/00manifest.i index 25ad8dc2a4d87b479772db8075e653f37c0a1e30..e35c6bf121b04e9a520ae6fcb95a06e381132f86 100644 GIT binary patch delta 139 zcmXT=#yFvdkE@Zt7$^dAC^4TCv3)ss=_G z0vRA|k(-#A7oSv=n3tTPSCYn%WMPn!YMNqbY+-I{VQONSY?@|ZYGh`bm||#Rl9*(f OmS|zdnqrue#03DHo+Fb0 delta 5 McmeyySTvys00+MUYybcN diff --git a/config/module/test-fixtures/basic-hg/.hg/store/data/main__branch.tf.i b/config/module/test-fixtures/basic-hg/.hg/store/data/main__branch.tf.i new file mode 100644 index 0000000000000000000000000000000000000000..a6bdf46f1091be151835fda73ea8ccb9f8c70c21 GIT binary patch literal 64 ocmZQzWME{#1dRWoUQPrLGn6+WD*P;%#K*02?S1MgRZ+ literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-hg/.hg/store/fncache b/config/module/test-fixtures/basic-hg/.hg/store/fncache index af601aa05..a1babe068 100644 --- a/config/module/test-fixtures/basic-hg/.hg/store/fncache +++ b/config/module/test-fixtures/basic-hg/.hg/store/fncache @@ -1 +1,2 @@ data/main.tf.i +data/main_branch.tf.i diff --git a/config/module/test-fixtures/basic-hg/.hg/store/undo b/config/module/test-fixtures/basic-hg/.hg/store/undo index 9dd8c80976c979b46d2068ac7111685daae78167..cf2be297d7083584b62cd4d06b4f2bb835f193ab 100644 GIT binary patch literal 59 zcmYdEEJ@VQP0Y-TPbx~xOU}?MNz=<@FyJyU$W6@4OiL{;0dfruO@IQ)8Hsu6sX6)S JAQ5vjE&xq(5=sC7 delta 44 ycmcDv;!H^_Nz~6x%*>l8ASz?PWnhq;k(igBnvLVSAd@{eF*8rEBn<$M@(T_C From 2a655bc7d96002f7060542bab49961eeeb3b4ecb Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Sep 2014 10:52:08 -0700 Subject: [PATCH 37/41] config/module: detect GitHub URLs --- config/module/detect.go | 1 + config/module/detect_github.go | 63 +++++++++++++++++++++++++++++ config/module/detect_github_test.go | 47 +++++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 config/module/detect_github.go create mode 100644 config/module/detect_github_test.go diff --git a/config/module/detect.go b/config/module/detect.go index 633bda246..36a34632f 100644 --- a/config/module/detect.go +++ b/config/module/detect.go @@ -20,6 +20,7 @@ var Detectors []Detector func init() { Detectors = []Detector{ + new(GitHubDetector), new(FileDetector), } } diff --git a/config/module/detect_github.go b/config/module/detect_github.go new file mode 100644 index 000000000..e4854cf42 --- /dev/null +++ b/config/module/detect_github.go @@ -0,0 +1,63 @@ +package module + +import ( + "fmt" + "net/url" + "strings" +) + +// GitHubDetector implements Detector to detect GitHub URLs and turn +// them into URLs that the Git Getter can understand. +type GitHubDetector struct{} + +func (d *GitHubDetector) Detect(src, _ string) (string, bool, error) { + if len(src) == 0 { + return "", false, nil + } + + if strings.HasPrefix(src, "github.com/") { + return d.detectHTTP(src) + } else if strings.HasPrefix(src, "git@github.com:") { + return d.detectSSH(src) + } + + return "", false, nil +} + +func (d *GitHubDetector) detectHTTP(src string) (string, bool, error) { + urlStr := fmt.Sprintf("https://%s", src) + url, err := url.Parse(urlStr) + if err != nil { + return "", true, fmt.Errorf("error parsing GitHub URL: %s", err) + } + + if !strings.HasSuffix(url.Path, ".git") { + url.Path += ".git" + } + + return "git::" + url.String(), true, nil +} + +func (d *GitHubDetector) detectSSH(src string) (string, bool, error) { + idx := strings.Index(src, ":") + qidx := strings.Index(src, "?") + if qidx == -1 { + qidx = len(src) + } + + var u url.URL + u.Scheme = "ssh" + u.User = url.User("git") + u.Host = "github.com" + u.Path = src[idx+1:qidx] + if qidx < len(src) { + q, err := url.ParseQuery(src[qidx+1:]) + if err != nil { + return "", true, fmt.Errorf("error parsing GitHub SSH URL: %s", err) + } + + u.RawQuery = q.Encode() + } + + return "git::"+u.String(), true, nil +} diff --git a/config/module/detect_github_test.go b/config/module/detect_github_test.go new file mode 100644 index 000000000..37fac84c8 --- /dev/null +++ b/config/module/detect_github_test.go @@ -0,0 +1,47 @@ +package module + +import ( + "testing" +) + +func TestGitHubDetector(t *testing.T) { + cases := []struct { + Input string + Output string + }{ + // HTTP + {"github.com/hashicorp/foo", "git::https://github.com/hashicorp/foo.git"}, + {"github.com/hashicorp/foo.git", "git::https://github.com/hashicorp/foo.git"}, + { + "github.com/hashicorp/foo?foo=bar", + "git::https://github.com/hashicorp/foo.git?foo=bar", + }, + { + "github.com/hashicorp/foo.git?foo=bar", + "git::https://github.com/hashicorp/foo.git?foo=bar", + }, + + // SSH + {"git@github.com:hashicorp/foo.git", "git::ssh://git@github.com/hashicorp/foo.git"}, + { + "git@github.com:hashicorp/foo.git?foo=bar", + "git::ssh://git@github.com/hashicorp/foo.git?foo=bar", + }, + } + + pwd := "/pwd" + f := new(GitHubDetector) + for i, tc := range cases { + output, ok, err := f.Detect(tc.Input, pwd) + if err != nil { + t.Fatalf("err: %s", err) + } + if !ok { + t.Fatal("not ok") + } + + if output != tc.Output { + t.Fatalf("%d: bad: %#v", i, output) + } + } +} From 5480eb4e41c45f34dd63dfa4e82f7963cf97bd8a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Sep 2014 10:54:23 -0700 Subject: [PATCH 38/41] config/module: detect preserves forces --- config/module/detect.go | 9 ++++++++- config/module/detect_test.go | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/config/module/detect.go b/config/module/detect.go index 36a34632f..0021c0a9f 100644 --- a/config/module/detect.go +++ b/config/module/detect.go @@ -48,9 +48,16 @@ func Detect(src string, pwd string) (string, error) { continue } - // Preserve the forced getter if it exists + var detectForce string + detectForce, result = getForcedGetter(result) + + // Preserve the forced getter if it exists. We try to use the + // original set force first, followed by any force set by the + // detector. if getForce != "" { result = fmt.Sprintf("%s::%s", getForce, result) + } else if detectForce != "" { + result = fmt.Sprintf("%s::%s", detectForce, result) } return result, nil diff --git a/config/module/detect_test.go b/config/module/detect_test.go index 5fdf5dc74..8f62f6618 100644 --- a/config/module/detect_test.go +++ b/config/module/detect_test.go @@ -13,6 +13,7 @@ func TestDetect(t *testing.T) { }{ {"./foo", "/foo", "file:///foo/foo", false}, {"git::./foo", "/foo", "git::file:///foo/foo", false}, + {"git::github.com/hashicorp/foo", "", "git::https://github.com/hashicorp/foo.git", false}, } for i, tc := range cases { From 27564fff2ba31ace70d91917fc4cedb3d258348b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Sep 2014 11:55:14 -0700 Subject: [PATCH 39/41] config/module: detect BitBucket URLs --- config/module/detect.go | 1 + config/module/detect_bitbucket.go | 66 ++++++++++++++++++++++++++ config/module/detect_bitbucket_test.go | 50 +++++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 config/module/detect_bitbucket.go create mode 100644 config/module/detect_bitbucket_test.go diff --git a/config/module/detect.go b/config/module/detect.go index 0021c0a9f..99b2af0a7 100644 --- a/config/module/detect.go +++ b/config/module/detect.go @@ -21,6 +21,7 @@ var Detectors []Detector func init() { Detectors = []Detector{ new(GitHubDetector), + new(BitBucketDetector), new(FileDetector), } } diff --git a/config/module/detect_bitbucket.go b/config/module/detect_bitbucket.go new file mode 100644 index 000000000..657637c09 --- /dev/null +++ b/config/module/detect_bitbucket.go @@ -0,0 +1,66 @@ +package module + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" +) + +// BitBucketDetector implements Detector to detect BitBucket URLs and turn +// them into URLs that the Git or Hg Getter can understand. +type BitBucketDetector struct{} + +func (d *BitBucketDetector) Detect(src, _ string) (string, bool, error) { + if len(src) == 0 { + return "", false, nil + } + + if strings.HasPrefix(src, "bitbucket.org/") { + return d.detectHTTP(src) + } + + return "", false, nil +} + +func (d *BitBucketDetector) detectHTTP(src string) (string, bool, error) { + u, err := url.Parse("https://" + src) + if err != nil { + return "", true, fmt.Errorf("error parsing BitBucket URL: %s", err) + } + + // We need to get info on this BitBucket repository to determine whether + // it is Git or Hg. + var info struct { + SCM string `json:"scm"` + } + infoUrl := "https://api.bitbucket.org/1.0/repositories" + u.Path + resp, err := http.Get(infoUrl) + if err != nil { + return "", true, fmt.Errorf("error looking up BitBucket URL: %s", err) + } + if resp.StatusCode == 403 { + // A private repo + return "", true, fmt.Errorf( + "shorthand BitBucket URL can't be used for private repos, " + + "please use a full URL") + } + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&info); err != nil { + return "", true, fmt.Errorf("error looking up BitBucket URL: %s", err) + } + + switch info.SCM { + case "git": + if !strings.HasSuffix(u.Path, ".git") { + u.Path += ".git" + } + + return "git::" + u.String(), true, nil + case "hg": + return "hg::" + u.String(), true, nil + default: + return "", true, fmt.Errorf("unknown BitBucket SCM type: %s", info.SCM) + } +} diff --git a/config/module/detect_bitbucket_test.go b/config/module/detect_bitbucket_test.go new file mode 100644 index 000000000..91dad2e59 --- /dev/null +++ b/config/module/detect_bitbucket_test.go @@ -0,0 +1,50 @@ +package module + +import ( + "net/http" + "testing" +) + +const testBBUrl = "https://bitbucket.org/hashicorp/tf-test-git" + +func TestBitBucketDetector(t *testing.T) { + if _, err := http.Get(testBBUrl); err != nil { + t.Log("internet may not be working, skipping BB tests") + t.Skip() + } + + cases := []struct { + Input string + Output string + }{ + // HTTP + { + "bitbucket.org/hashicorp/tf-test-git", + "git::https://bitbucket.org/hashicorp/tf-test-git.git", + }, + { + "bitbucket.org/hashicorp/tf-test-git.git", + "git::https://bitbucket.org/hashicorp/tf-test-git.git", + }, + { + "bitbucket.org/hashicorp/tf-test-hg", + "hg::https://bitbucket.org/hashicorp/tf-test-hg", + }, + } + + pwd := "/pwd" + f := new(BitBucketDetector) + for i, tc := range cases { + output, ok, err := f.Detect(tc.Input, pwd) + if err != nil { + t.Fatalf("err: %s", err) + } + if !ok { + t.Fatal("not ok") + } + + if output != tc.Output { + t.Fatalf("%d: bad: %#v", i, output) + } + } +} From fde151978ef5123aa2422fb862f7cf5bf6b13c60 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Sep 2014 12:02:35 -0700 Subject: [PATCH 40/41] config/module: parallelize some things --- Makefile | 2 +- config/module/detect_bitbucket_test.go | 2 ++ config/module/get_hg_test.go | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ec8a82500..7520a245a 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ dev: config/y.go @TF_DEV=1 sh -c "$(CURDIR)/scripts/build.sh" test: config/y.go - TF_ACC= go test $(TEST) $(TESTARGS) -timeout=10s + TF_ACC= go test $(TEST) $(TESTARGS) -timeout=10s -parallel=4 testacc: config/y.go @if [ "$(TEST)" = "./..." ]; then \ diff --git a/config/module/detect_bitbucket_test.go b/config/module/detect_bitbucket_test.go index 91dad2e59..92ce633b1 100644 --- a/config/module/detect_bitbucket_test.go +++ b/config/module/detect_bitbucket_test.go @@ -8,6 +8,8 @@ import ( const testBBUrl = "https://bitbucket.org/hashicorp/tf-test-git" func TestBitBucketDetector(t *testing.T) { + t.Parallel() + if _, err := http.Get(testBBUrl); err != nil { t.Log("internet may not be working, skipping BB tests") t.Skip() diff --git a/config/module/get_hg_test.go b/config/module/get_hg_test.go index 02260bd29..d7125bde2 100644 --- a/config/module/get_hg_test.go +++ b/config/module/get_hg_test.go @@ -20,6 +20,8 @@ func TestHgGetter_impl(t *testing.T) { } func TestHgGetter(t *testing.T) { + t.Parallel() + if !testHasHg { t.Log("hg not found, skipping") t.Skip() @@ -41,6 +43,8 @@ func TestHgGetter(t *testing.T) { } func TestHgGetter_branch(t *testing.T) { + t.Parallel() + if !testHasHg { t.Log("hg not found, skipping") t.Skip() From 9a626b3e8c1a38d25957ae39d0a1e6739cf21ce7 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 16 Sep 2014 13:44:12 -0700 Subject: [PATCH 41/41] config/module: support HTTP protocol --- config/module/get.go | 10 ++- config/module/get_http.go | 128 +++++++++++++++++++++++++++++++++ config/module/get_http_test.go | 126 ++++++++++++++++++++++++++++++++ 3 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 config/module/get_http.go create mode 100644 config/module/get_http_test.go diff --git a/config/module/get.go b/config/module/get.go index 5131d7384..89a143604 100644 --- a/config/module/get.go +++ b/config/module/get.go @@ -30,10 +30,14 @@ var Getters map[string]Getter var forcedRegexp = regexp.MustCompile(`^([A-Za-z]+)::(.+)$`) func init() { + httpGetter := new(HttpGetter) + Getters = map[string]Getter{ - "file": new(FileGetter), - "git": new(GitGetter), - "hg": new(HgGetter), + "file": new(FileGetter), + "git": new(GitGetter), + "hg": new(HgGetter), + "http": httpGetter, + "https": httpGetter, } } diff --git a/config/module/get_http.go b/config/module/get_http.go new file mode 100644 index 000000000..7aa7d9abf --- /dev/null +++ b/config/module/get_http.go @@ -0,0 +1,128 @@ +package module + +import ( + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" + "strings" +) + +// HttpGetter is a Getter implementation that will download a module from +// an HTTP endpoint. The protocol for downloading a module from an HTTP +// endpoing is as follows: +// +// An HTTP GET request is made to the URL with the additional GET parameter +// "terraform-get=1". This lets you handle that scenario specially if you +// wish. The response must be a 2xx. +// +// First, a header is looked for "X-Terraform-Get" which should contain +// a source URL to download. +// +// If the header is not present, then a meta tag is searched for named +// "terraform-get" and the content should be a source URL. +// +// The source URL, whether from the header or meta tag, must be a fully +// formed URL. The shorthand syntax of "github.com/foo/bar" or relative +// paths are not allowed. +type HttpGetter struct{} + +func (g *HttpGetter) Get(dst string, u *url.URL) error { + // Copy the URL so we can modify it + var newU url.URL = *u + u = &newU + + // Add terraform-get to the parameter. + q := u.Query() + q.Add("terraform-get", "1") + u.RawQuery = q.Encode() + + // Get the URL + resp, err := http.Get(u.String()) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("bad response code: %d", resp.StatusCode) + } + + // Extract the source URL + var source string + if v := resp.Header.Get("X-Terraform-Get"); v != "" { + source = v + } else { + source, err = g.parseMeta(resp.Body) + if err != nil { + return err + } + } + if source == "" { + return fmt.Errorf("no source URL was returned") + } + + // Get it! + return Get(dst, source) +} + +// parseMeta looks for the first meta tag in the given reader that +// will give us the source URL. +func (g *HttpGetter) parseMeta(r io.Reader) (string, error) { + d := xml.NewDecoder(r) + d.CharsetReader = charsetReader + d.Strict = false + var err error + var t xml.Token + for { + t, err = d.Token() + if err != nil { + if err == io.EOF { + err = nil + } + return "", err + } + if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") { + return "", nil + } + if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") { + return "", nil + } + e, ok := t.(xml.StartElement) + if !ok || !strings.EqualFold(e.Name.Local, "meta") { + continue + } + if attrValue(e.Attr, "name") != "terraform-get" { + continue + } + if f := attrValue(e.Attr, "content"); f != "" { + return f, nil + } + } +} + +// attrValue returns the attribute value for the case-insensitive key +// `name', or the empty string if nothing is found. +func attrValue(attrs []xml.Attr, name string) string { + for _, a := range attrs { + if strings.EqualFold(a.Name.Local, name) { + return a.Value + } + } + return "" +} + +// charsetReader returns a reader for the given charset. Currently +// it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful +// error which is printed by go get, so the user can find why the package +// wasn't downloaded if the encoding is not supported. Note that, in +// order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters +// greater than 0x7f are not rejected). +func charsetReader(charset string, input io.Reader) (io.Reader, error) { + switch strings.ToLower(charset) { + case "ascii": + return input, nil + default: + return nil, fmt.Errorf("can't decode XML document using charset %q", charset) + } +} diff --git a/config/module/get_http_test.go b/config/module/get_http_test.go new file mode 100644 index 000000000..8fa61d2e4 --- /dev/null +++ b/config/module/get_http_test.go @@ -0,0 +1,126 @@ +package module + +import ( + "fmt" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "testing" +) + +func TestHttpGetter_impl(t *testing.T) { + var _ Getter = new(HttpGetter) +} + +func TestHttpGetter_header(t *testing.T) { + ln := testHttpServer(t) + defer ln.Close() + + g := new(HttpGetter) + dst := tempDir(t) + + var u url.URL + u.Scheme = "http" + u.Host = ln.Addr().String() + u.Path = "/header" + + // Get it! + if err := g.Get(dst, &u); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath := filepath.Join(dst, "main.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestHttpGetter_meta(t *testing.T) { + ln := testHttpServer(t) + defer ln.Close() + + g := new(HttpGetter) + dst := tempDir(t) + + var u url.URL + u.Scheme = "http" + u.Host = ln.Addr().String() + u.Path = "/meta" + + // Get it! + if err := g.Get(dst, &u); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath := filepath.Join(dst, "main.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestHttpGetter_none(t *testing.T) { + ln := testHttpServer(t) + defer ln.Close() + + g := new(HttpGetter) + dst := tempDir(t) + + var u url.URL + u.Scheme = "http" + u.Host = ln.Addr().String() + u.Path = "/none" + + // Get it! + if err := g.Get(dst, &u); err == nil { + t.Fatal("should error") + } +} + +func testHttpServer(t *testing.T) net.Listener { + ln, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatalf("err: %s", err) + } + + mux := http.NewServeMux() + mux.HandleFunc("/header", testHttpHandlerHeader) + mux.HandleFunc("/meta", testHttpHandlerMeta) + + var server http.Server + server.Handler = mux + go server.Serve(ln) + + return ln +} + +func testHttpHandlerHeader(w http.ResponseWriter, r *http.Request) { + w.Header().Add("X-Terraform-Get", testModuleURL("basic").String()) + w.WriteHeader(200) +} + +func testHttpHandlerMeta(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(fmt.Sprintf(testHttpMetaStr, testModuleURL("basic").String()))) +} + +func testHttpHandlerNone(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(testHttpNoneStr)) +} + +const testHttpMetaStr = ` + + + + + +` + +const testHttpNoneStr = ` + + + + +`