From 0e0e3d73af094b0935994adca175cd47d8661957 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Sat, 7 May 2016 21:55:32 -0700 Subject: [PATCH] core: New ResourceProvider methods for data resources This is a breaking change to the ResourceProvider interface that adds the new operations relating to data sources. DataSources, ValidateDataSource, ReadDataDiff and ReadDataApply are the data source equivalents of Resources, Validate, Diff and Apply (respectively) for managed resources. The diff/apply model seems at first glance a rather strange workflow for read-only resources, but implementing data resources in this way allows them to fit cleanly into the standard plan/apply lifecycle in cases where the configuration contains computed arguments and thus the read must be deferred until apply time. Along with breaking the interface, we also fix up the plugin client/server and helper/schema implementations of it, which are all of the callers used when provider plugins use helper/schema. This would be a breaking change for any provider plugin that directly implements the provider interface, but no known plugins do this and it is not recommended. At the helper/schema layer the implementer sees ReadDataApply as a "Read", as opposed to "Create" or "Update" as in the managed resource Apply implementation. The planning mechanics are handled entirely within helper/schema, so that complexity is hidden from the provider implementation itself. --- helper/schema/provider.go | 78 ++++++++++++- helper/schema/provider_test.go | 32 ++++++ helper/schema/resource.go | 27 +++++ plugin/resource_provider.go | 140 +++++++++++++++++++++++ plugin/resource_provider_test.go | 109 ++++++++++++++++++ terraform/resource_provider.go | 46 +++++++- terraform/resource_provider_mock.go | 166 ++++++++++++++++++++-------- 7 files changed, 551 insertions(+), 47 deletions(-) diff --git a/helper/schema/provider.go b/helper/schema/provider.go index a3edf82f7..7b3371c8a 100644 --- a/helper/schema/provider.go +++ b/helper/schema/provider.go @@ -33,6 +33,14 @@ type Provider struct { // Diff, etc. to the proper resource. ResourcesMap map[string]*Resource + // DataSourcesMap is the collection of available data sources that + // this provider implements, with a Resource instance defining + // the schema and Read operation of each. + // + // Resource instances for data sources must have a Read function + // and must *not* implement Create, Update or Delete. + DataSourcesMap map[string]*Resource + // ConfigureFunc is a function for configuring the provider. If the // provider doesn't need to be configured, this can be omitted. // @@ -68,7 +76,19 @@ func (p *Provider) InternalValidate() error { for k, r := range p.ResourcesMap { if err := r.InternalValidate(nil); err != nil { - return fmt.Errorf("%s: %s", k, err) + return fmt.Errorf("resource %s: %s", k, err) + } + } + + for k, r := range p.DataSourcesMap { + if err := r.InternalValidate(nil); err != nil { + return fmt.Errorf("data source %s: %s", k, err) + } + + if r.Create != nil || r.Update != nil || r.Delete != nil { + return fmt.Errorf( + "data source %s: must not have Create, Update or Delete", k, + ) } } @@ -262,3 +282,59 @@ func (p *Provider) ImportState( return states, nil } + +// ValidateDataSource implementation of terraform.ResourceProvider interface. +func (p *Provider) ValidateDataSource( + t string, c *terraform.ResourceConfig) ([]string, []error) { + r, ok := p.DataSourcesMap[t] + if !ok { + return nil, []error{fmt.Errorf( + "Provider doesn't support data source: %s", t)} + } + + return r.Validate(c) +} + +// ReadDataDiff implementation of terraform.ResourceProvider interface. +func (p *Provider) ReadDataDiff( + info *terraform.InstanceInfo, + c *terraform.ResourceConfig) (*terraform.InstanceDiff, error) { + + r, ok := p.DataSourcesMap[info.Type] + if !ok { + return nil, fmt.Errorf("unknown data source: %s", info.Type) + } + + return r.Diff(nil, c) +} + +// RefreshData implementation of terraform.ResourceProvider interface. +func (p *Provider) ReadDataApply( + info *terraform.InstanceInfo, + d *terraform.InstanceDiff) (*terraform.InstanceState, error) { + + r, ok := p.DataSourcesMap[info.Type] + if !ok { + return nil, fmt.Errorf("unknown data source: %s", info.Type) + } + + return r.ReadDataApply(d, p.meta) +} + +// DataSources implementation of terraform.ResourceProvider interface. +func (p *Provider) DataSources() []terraform.DataSource { + keys := make([]string, 0, len(p.DataSourcesMap)) + for k, _ := range p.DataSourcesMap { + keys = append(keys, k) + } + sort.Strings(keys) + + result := make([]terraform.DataSource, 0, len(keys)) + for _, k := range keys { + result = append(result, terraform.DataSource{ + Name: k, + }) + } + + return result +} diff --git a/helper/schema/provider_test.go b/helper/schema/provider_test.go index 3c8a93940..aa8a787bf 100644 --- a/helper/schema/provider_test.go +++ b/helper/schema/provider_test.go @@ -132,6 +132,38 @@ func TestProviderResources(t *testing.T) { } } +func TestProviderDataSources(t *testing.T) { + cases := []struct { + P *Provider + Result []terraform.DataSource + }{ + { + P: &Provider{}, + Result: []terraform.DataSource{}, + }, + + { + P: &Provider{ + DataSourcesMap: map[string]*Resource{ + "foo": nil, + "bar": nil, + }, + }, + Result: []terraform.DataSource{ + terraform.DataSource{Name: "bar"}, + terraform.DataSource{Name: "foo"}, + }, + }, + } + + for i, tc := range cases { + actual := tc.P.DataSources() + if !reflect.DeepEqual(actual, tc.Result) { + t.Fatalf("%d: got %#v; want %#v", i, actual, tc.Result) + } + } +} + func TestProviderValidate(t *testing.T) { cases := []struct { P *Provider diff --git a/helper/schema/resource.go b/helper/schema/resource.go index 5e917d144..9d2ad8007 100644 --- a/helper/schema/resource.go +++ b/helper/schema/resource.go @@ -175,6 +175,33 @@ func (r *Resource) Validate(c *terraform.ResourceConfig) ([]string, []error) { return schemaMap(r.Schema).Validate(c) } +// ReadDataApply loads the data for a data source, given a diff that +// describes the configuration arguments and desired computed attributes. +func (r *Resource) ReadDataApply( + d *terraform.InstanceDiff, + meta interface{}, +) (*terraform.InstanceState, error) { + + // Data sources are always built completely from scratch + // on each read, so the source state is always nil. + data, err := schemaMap(r.Schema).Data(nil, d) + if err != nil { + return nil, err + } + + err = r.Read(data, meta) + state := data.State() + if state != nil && state.ID == "" { + // Data sources can set an ID if they want, but they aren't + // required to; we'll provide a placeholder if they don't, + // to preserve the invariant that all resources have non-empty + // ids. + state.ID = "-" + } + + return r.recordCurrentSchemaVersion(state), err +} + // Refresh refreshes the state of the resource. func (r *Resource) Refresh( s *terraform.InstanceState, diff --git a/plugin/resource_provider.go b/plugin/resource_provider.go index d7512674a..b711864a1 100644 --- a/plugin/resource_provider.go +++ b/plugin/resource_provider.go @@ -156,6 +156,30 @@ func (p *ResourceProvider) Diff( return resp.Diff, err } +func (p *ResourceProvider) ValidateDataSource( + t string, c *terraform.ResourceConfig) ([]string, []error) { + var resp ResourceProviderValidateResourceResponse + args := ResourceProviderValidateResourceArgs{ + Config: c, + Type: t, + } + + err := p.Client.Call("Plugin.ValidateDataSource", &args, &resp) + if err != nil { + return nil, []error{err} + } + + var errs []error + if len(resp.Errors) > 0 { + errs = make([]error, len(resp.Errors)) + for i, err := range resp.Errors { + errs[i] = err + } + } + + return resp.Warnings, errs +} + func (p *ResourceProvider) Refresh( info *terraform.InstanceInfo, s *terraform.InstanceState) (*terraform.InstanceState, error) { @@ -208,6 +232,58 @@ func (p *ResourceProvider) Resources() []terraform.ResourceType { return result } +func (p *ResourceProvider) ReadDataDiff( + info *terraform.InstanceInfo, + c *terraform.ResourceConfig) (*terraform.InstanceDiff, error) { + var resp ResourceProviderReadDataDiffResponse + args := &ResourceProviderReadDataDiffArgs{ + Info: info, + Config: c, + } + + err := p.Client.Call("Plugin.ReadDataDiff", args, &resp) + if err != nil { + return nil, err + } + if resp.Error != nil { + err = resp.Error + } + + return resp.Diff, err +} + +func (p *ResourceProvider) ReadDataApply( + info *terraform.InstanceInfo, + d *terraform.InstanceDiff) (*terraform.InstanceState, error) { + var resp ResourceProviderReadDataApplyResponse + args := &ResourceProviderReadDataApplyArgs{ + Info: info, + Diff: d, + } + + err := p.Client.Call("Plugin.ReadDataApply", args, &resp) + if err != nil { + return nil, err + } + if resp.Error != nil { + err = resp.Error + } + + return resp.State, err +} + +func (p *ResourceProvider) DataSources() []terraform.DataSource { + var result []terraform.DataSource + + err := p.Client.Call("Plugin.DataSources", new(interface{}), &result) + if err != nil { + // TODO: panic, log, what? + return nil + } + + return result +} + func (p *ResourceProvider) Close() error { return p.Client.Close() } @@ -275,6 +351,26 @@ type ResourceProviderImportStateResponse struct { Error *plugin.BasicError } +type ResourceProviderReadDataApplyArgs struct { + Info *terraform.InstanceInfo + Diff *terraform.InstanceDiff +} + +type ResourceProviderReadDataApplyResponse struct { + State *terraform.InstanceState + Error *plugin.BasicError +} + +type ResourceProviderReadDataDiffArgs struct { + Info *terraform.InstanceInfo + Config *terraform.ResourceConfig +} + +type ResourceProviderReadDataDiffResponse struct { + Diff *terraform.InstanceDiff + Error *plugin.BasicError +} + type ResourceProviderValidateArgs struct { Config *terraform.ResourceConfig } @@ -408,3 +504,47 @@ func (s *ResourceProviderServer) Resources( *result = s.Provider.Resources() return nil } + +func (s *ResourceProviderServer) ValidateDataSource( + args *ResourceProviderValidateResourceArgs, + reply *ResourceProviderValidateResourceResponse) error { + warns, errs := s.Provider.ValidateDataSource(args.Type, args.Config) + berrs := make([]*plugin.BasicError, len(errs)) + for i, err := range errs { + berrs[i] = plugin.NewBasicError(err) + } + *reply = ResourceProviderValidateResourceResponse{ + Warnings: warns, + Errors: berrs, + } + return nil +} + +func (s *ResourceProviderServer) ReadDataDiff( + args *ResourceProviderReadDataDiffArgs, + result *ResourceProviderReadDataDiffResponse) error { + diff, err := s.Provider.ReadDataDiff(args.Info, args.Config) + *result = ResourceProviderReadDataDiffResponse{ + Diff: diff, + Error: plugin.NewBasicError(err), + } + return nil +} + +func (s *ResourceProviderServer) ReadDataApply( + args *ResourceProviderReadDataApplyArgs, + result *ResourceProviderReadDataApplyResponse) error { + newState, err := s.Provider.ReadDataApply(args.Info, args.Diff) + *result = ResourceProviderReadDataApplyResponse{ + State: newState, + Error: plugin.NewBasicError(err), + } + return nil +} + +func (s *ResourceProviderServer) DataSources( + nothing interface{}, + result *[]terraform.DataSource) error { + *result = s.Provider.DataSources() + return nil +} diff --git a/plugin/resource_provider_test.go b/plugin/resource_provider_test.go index 200f9e566..c20662b62 100644 --- a/plugin/resource_provider_test.go +++ b/plugin/resource_provider_test.go @@ -389,6 +389,77 @@ func TestResourceProvider_resources(t *testing.T) { } } +func TestResourceProvider_readdataapply(t *testing.T) { + p := new(terraform.MockResourceProvider) + + // Create a mock provider + client, _ := plugin.TestPluginRPCConn(t, pluginMap(&ServeOpts{ + ProviderFunc: testProviderFixed(p), + })) + defer client.Close() + + // Request the provider + raw, err := client.Dispense(ProviderPluginName) + if err != nil { + t.Fatalf("err: %s", err) + } + provider := raw.(terraform.ResourceProvider) + + p.ReadDataApplyReturn = &terraform.InstanceState{ + ID: "bob", + } + + // ReadDataApply + info := &terraform.InstanceInfo{} + diff := &terraform.InstanceDiff{} + newState, err := provider.ReadDataApply(info, diff) + if !p.ReadDataApplyCalled { + t.Fatal("ReadDataApply should be called") + } + if !reflect.DeepEqual(p.ReadDataApplyDiff, diff) { + t.Fatalf("bad: %#v", p.ReadDataApplyDiff) + } + if err != nil { + t.Fatalf("bad: %#v", err) + } + if !reflect.DeepEqual(p.ReadDataApplyReturn, newState) { + t.Fatalf("bad: %#v", newState) + } +} + +func TestResourceProvider_datasources(t *testing.T) { + p := new(terraform.MockResourceProvider) + + // Create a mock provider + client, _ := plugin.TestPluginRPCConn(t, pluginMap(&ServeOpts{ + ProviderFunc: testProviderFixed(p), + })) + defer client.Close() + + // Request the provider + raw, err := client.Dispense(ProviderPluginName) + if err != nil { + t.Fatalf("err: %s", err) + } + provider := raw.(terraform.ResourceProvider) + + expected := []terraform.DataSource{ + {"foo"}, + {"bar"}, + } + + p.DataSourcesReturn = expected + + // DataSources + result := provider.DataSources() + if !p.DataSourcesCalled { + t.Fatal("DataSources should be called") + } + if !reflect.DeepEqual(result, expected) { + t.Fatalf("bad: %#v", result) + } +} + func TestResourceProvider_validate(t *testing.T) { p := new(terraform.MockResourceProvider) @@ -628,6 +699,44 @@ func TestResourceProvider_validateResource_warns(t *testing.T) { } } +func TestResourceProvider_validateDataSource(t *testing.T) { + p := new(terraform.MockResourceProvider) + + // Create a mock provider + client, _ := plugin.TestPluginRPCConn(t, pluginMap(&ServeOpts{ + ProviderFunc: testProviderFixed(p), + })) + defer client.Close() + + // Request the provider + raw, err := client.Dispense(ProviderPluginName) + if err != nil { + t.Fatalf("err: %s", err) + } + provider := raw.(terraform.ResourceProvider) + + // Configure + config := &terraform.ResourceConfig{ + Raw: map[string]interface{}{"foo": "bar"}, + } + w, e := provider.ValidateDataSource("foo", config) + if !p.ValidateDataSourceCalled { + t.Fatal("configure should be called") + } + if p.ValidateDataSourceType != "foo" { + t.Fatalf("bad: %#v", p.ValidateDataSourceType) + } + if !reflect.DeepEqual(p.ValidateDataSourceConfig, config) { + t.Fatalf("bad: %#v", p.ValidateDataSourceConfig) + } + if w != nil { + t.Fatalf("bad: %#v", w) + } + if e != nil { + t.Fatalf("bad: %#v", e) + } +} + func TestResourceProvider_close(t *testing.T) { p := new(terraform.MockResourceProvider) diff --git a/terraform/resource_provider.go b/terraform/resource_provider.go index 9957f6896..37cd1d5c3 100644 --- a/terraform/resource_provider.go +++ b/terraform/resource_provider.go @@ -97,6 +97,35 @@ type ResourceProvider interface { // Each rule is represented by a separate resource in Terraform, // therefore multiple states are returned. ImportState(*InstanceInfo, string) ([]*InstanceState, error) + + /********************************************************************* + * Functions related to data resources + *********************************************************************/ + + // ValidateDataSource is called once at the beginning with the raw + // configuration (no interpolation done) and can return a list of warnings + // and/or errors. + // + // This is called once per data source instance. + // + // This should not assume any of the values in the resource configuration + // are valid since it is possible they have to be interpolated still. + // The primary use case of this call is to check that the required keys + // are set and that the general structure is correct. + ValidateDataSource(string, *ResourceConfig) ([]string, []error) + + // DataSources returns all of the available data sources that this + // provider implements. + DataSources() []DataSource + + // ReadDataDiff produces a diff that represents the state that will + // be produced when the given data source is read using a later call + // to ReadDataApply. + ReadDataDiff(*InstanceInfo, *ResourceConfig) (*InstanceDiff, error) + + // ReadDataApply initializes a data instance using the configuration + // in a diff produced by ReadDataDiff. + ReadDataApply(*InstanceInfo, *InstanceDiff) (*InstanceState, error) } // ResourceProviderCloser is an interface that providers that can close @@ -111,6 +140,11 @@ type ResourceType struct { Importable bool // Whether this resource supports importing } +// DataSource is a data source that a resource provider implements. +type DataSource struct { + Name string +} + // ResourceProviderFactory is a function type that creates a new instance // of a resource provider. type ResourceProviderFactory func() (ResourceProvider, error) @@ -123,7 +157,7 @@ func ResourceProviderFactoryFixed(p ResourceProvider) ResourceProviderFactory { } } -func ProviderSatisfies(p ResourceProvider, n string) bool { +func ProviderHasResource(p ResourceProvider, n string) bool { for _, rt := range p.Resources() { if rt.Name == n { return true @@ -132,3 +166,13 @@ func ProviderSatisfies(p ResourceProvider, n string) bool { return false } + +func ProviderHasDataSource(p ResourceProvider, n string) bool { + for _, rt := range p.DataSources() { + if rt.Name == n { + return true + } + } + + return false +} diff --git a/terraform/resource_provider_mock.go b/terraform/resource_provider_mock.go index f3bfbe3dd..8389fd0ae 100644 --- a/terraform/resource_provider_mock.go +++ b/terraform/resource_provider_mock.go @@ -10,51 +10,71 @@ type MockResourceProvider struct { // Anything you want, in case you need to store extra data with the mock. Meta interface{} - CloseCalled bool - CloseError error - InputCalled bool - InputInput UIInput - InputConfig *ResourceConfig - InputReturnConfig *ResourceConfig - InputReturnError error - InputFn func(UIInput, *ResourceConfig) (*ResourceConfig, error) - ApplyCalled bool - ApplyInfo *InstanceInfo - ApplyState *InstanceState - ApplyDiff *InstanceDiff - ApplyFn func(*InstanceInfo, *InstanceState, *InstanceDiff) (*InstanceState, error) - ApplyReturn *InstanceState - ApplyReturnError error - ConfigureCalled bool - ConfigureConfig *ResourceConfig - ConfigureFn func(*ResourceConfig) error - ConfigureReturnError error - DiffCalled bool - DiffInfo *InstanceInfo - DiffState *InstanceState - DiffDesired *ResourceConfig - DiffFn func(*InstanceInfo, *InstanceState, *ResourceConfig) (*InstanceDiff, error) - DiffReturn *InstanceDiff - DiffReturnError error - RefreshCalled bool - RefreshInfo *InstanceInfo - RefreshState *InstanceState - RefreshFn func(*InstanceInfo, *InstanceState) (*InstanceState, error) - RefreshReturn *InstanceState - RefreshReturnError error - ResourcesCalled bool - ResourcesReturn []ResourceType - ValidateCalled bool - ValidateConfig *ResourceConfig - ValidateFn func(*ResourceConfig) ([]string, []error) - ValidateReturnWarns []string - ValidateReturnErrors []error - ValidateResourceFn func(string, *ResourceConfig) ([]string, []error) - ValidateResourceCalled bool - ValidateResourceType string - ValidateResourceConfig *ResourceConfig - ValidateResourceReturnWarns []string - ValidateResourceReturnErrors []error + CloseCalled bool + CloseError error + InputCalled bool + InputInput UIInput + InputConfig *ResourceConfig + InputReturnConfig *ResourceConfig + InputReturnError error + InputFn func(UIInput, *ResourceConfig) (*ResourceConfig, error) + ApplyCalled bool + ApplyInfo *InstanceInfo + ApplyState *InstanceState + ApplyDiff *InstanceDiff + ApplyFn func(*InstanceInfo, *InstanceState, *InstanceDiff) (*InstanceState, error) + ApplyReturn *InstanceState + ApplyReturnError error + ConfigureCalled bool + ConfigureConfig *ResourceConfig + ConfigureFn func(*ResourceConfig) error + ConfigureReturnError error + DiffCalled bool + DiffInfo *InstanceInfo + DiffState *InstanceState + DiffDesired *ResourceConfig + DiffFn func(*InstanceInfo, *InstanceState, *ResourceConfig) (*InstanceDiff, error) + DiffReturn *InstanceDiff + DiffReturnError error + RefreshCalled bool + RefreshInfo *InstanceInfo + RefreshState *InstanceState + RefreshFn func(*InstanceInfo, *InstanceState) (*InstanceState, error) + RefreshReturn *InstanceState + RefreshReturnError error + ResourcesCalled bool + ResourcesReturn []ResourceType + ReadDataApplyCalled bool + ReadDataApplyInfo *InstanceInfo + ReadDataApplyDiff *InstanceDiff + ReadDataApplyFn func(*InstanceInfo, *InstanceDiff) (*InstanceState, error) + ReadDataApplyReturn *InstanceState + ReadDataApplyReturnError error + ReadDataDiffCalled bool + ReadDataDiffInfo *InstanceInfo + ReadDataDiffDesired *ResourceConfig + ReadDataDiffFn func(*InstanceInfo, *ResourceConfig) (*InstanceDiff, error) + ReadDataDiffReturn *InstanceDiff + ReadDataDiffReturnError error + DataSourcesCalled bool + DataSourcesReturn []DataSource + ValidateCalled bool + ValidateConfig *ResourceConfig + ValidateFn func(*ResourceConfig) ([]string, []error) + ValidateReturnWarns []string + ValidateReturnErrors []error + ValidateResourceFn func(string, *ResourceConfig) ([]string, []error) + ValidateResourceCalled bool + ValidateResourceType string + ValidateResourceConfig *ResourceConfig + ValidateResourceReturnWarns []string + ValidateResourceReturnErrors []error + ValidateDataSourceFn func(string, *ResourceConfig) ([]string, []error) + ValidateDataSourceCalled bool + ValidateDataSourceType string + ValidateDataSourceConfig *ResourceConfig + ValidateDataSourceReturnWarns []string + ValidateDataSourceReturnErrors []error ImportStateCalled bool ImportStateInfo *InstanceInfo @@ -196,3 +216,59 @@ func (p *MockResourceProvider) ImportState(info *InstanceInfo, id string) ([]*In return p.ImportStateReturn, p.ImportStateReturnError } + +func (p *MockResourceProvider) ValidateDataSource(t string, c *ResourceConfig) ([]string, []error) { + p.Lock() + defer p.Unlock() + + p.ValidateDataSourceCalled = true + p.ValidateDataSourceType = t + p.ValidateDataSourceConfig = c + + if p.ValidateDataSourceFn != nil { + return p.ValidateDataSourceFn(t, c) + } + + return p.ValidateDataSourceReturnWarns, p.ValidateDataSourceReturnErrors +} + +func (p *MockResourceProvider) ReadDataDiff( + info *InstanceInfo, + desired *ResourceConfig) (*InstanceDiff, error) { + p.Lock() + defer p.Unlock() + + p.ReadDataDiffCalled = true + p.ReadDataDiffInfo = info + p.ReadDataDiffDesired = desired + if p.ReadDataDiffFn != nil { + return p.ReadDataDiffFn(info, desired) + } + + return p.ReadDataDiffReturn, p.ReadDataDiffReturnError +} + +func (p *MockResourceProvider) ReadDataApply( + info *InstanceInfo, + d *InstanceDiff) (*InstanceState, error) { + p.Lock() + defer p.Unlock() + + p.ReadDataApplyCalled = true + p.ReadDataApplyInfo = info + p.ReadDataApplyDiff = d + + if p.ReadDataApplyFn != nil { + return p.ReadDataApplyFn(info, d) + } + + return p.ReadDataApplyReturn, p.ReadDataApplyReturnError +} + +func (p *MockResourceProvider) DataSources() []DataSource { + p.Lock() + defer p.Unlock() + + p.DataSourcesCalled = true + return p.DataSourcesReturn +}