core: New ResourceProvider methods for data resources

This is a breaking change to the ResourceProvider interface that adds the
new operations relating to data sources.

DataSources, ValidateDataSource, ReadDataDiff and ReadDataApply are the
data source equivalents of Resources, Validate, Diff and Apply (respectively)
for managed resources.

The diff/apply model seems at first glance a rather strange workflow for
read-only resources, but implementing data resources in this way allows them
to fit cleanly into the standard plan/apply lifecycle in cases where the
configuration contains computed arguments and thus the read must be deferred
until apply time.

Along with breaking the interface, we also fix up the plugin client/server
and helper/schema implementations of it, which are all of the callers
used when provider plugins use helper/schema. This would be a breaking
change for any provider plugin that directly implements the provider
interface, but no known plugins do this and it is not recommended.

At the helper/schema layer the implementer sees ReadDataApply as a "Read",
as opposed to "Create" or "Update" as in the managed resource Apply
implementation. The planning mechanics are handled entirely within
helper/schema, so that complexity is hidden from the provider implementation
itself.
This commit is contained in:
Martin Atkins 2016-05-07 21:55:32 -07:00
parent 718cdda77b
commit 0e0e3d73af
7 changed files with 551 additions and 47 deletions

View File

@ -33,6 +33,14 @@ type Provider struct {
// Diff, etc. to the proper resource.
ResourcesMap map[string]*Resource
// DataSourcesMap is the collection of available data sources that
// this provider implements, with a Resource instance defining
// the schema and Read operation of each.
//
// Resource instances for data sources must have a Read function
// and must *not* implement Create, Update or Delete.
DataSourcesMap map[string]*Resource
// ConfigureFunc is a function for configuring the provider. If the
// provider doesn't need to be configured, this can be omitted.
//
@ -68,7 +76,19 @@ func (p *Provider) InternalValidate() error {
for k, r := range p.ResourcesMap {
if err := r.InternalValidate(nil); err != nil {
return fmt.Errorf("%s: %s", k, err)
return fmt.Errorf("resource %s: %s", k, err)
}
}
for k, r := range p.DataSourcesMap {
if err := r.InternalValidate(nil); err != nil {
return fmt.Errorf("data source %s: %s", k, err)
}
if r.Create != nil || r.Update != nil || r.Delete != nil {
return fmt.Errorf(
"data source %s: must not have Create, Update or Delete", k,
)
}
}
@ -262,3 +282,59 @@ func (p *Provider) ImportState(
return states, nil
}
// ValidateDataSource implementation of terraform.ResourceProvider interface.
func (p *Provider) ValidateDataSource(
t string, c *terraform.ResourceConfig) ([]string, []error) {
r, ok := p.DataSourcesMap[t]
if !ok {
return nil, []error{fmt.Errorf(
"Provider doesn't support data source: %s", t)}
}
return r.Validate(c)
}
// ReadDataDiff implementation of terraform.ResourceProvider interface.
func (p *Provider) ReadDataDiff(
info *terraform.InstanceInfo,
c *terraform.ResourceConfig) (*terraform.InstanceDiff, error) {
r, ok := p.DataSourcesMap[info.Type]
if !ok {
return nil, fmt.Errorf("unknown data source: %s", info.Type)
}
return r.Diff(nil, c)
}
// RefreshData implementation of terraform.ResourceProvider interface.
func (p *Provider) ReadDataApply(
info *terraform.InstanceInfo,
d *terraform.InstanceDiff) (*terraform.InstanceState, error) {
r, ok := p.DataSourcesMap[info.Type]
if !ok {
return nil, fmt.Errorf("unknown data source: %s", info.Type)
}
return r.ReadDataApply(d, p.meta)
}
// DataSources implementation of terraform.ResourceProvider interface.
func (p *Provider) DataSources() []terraform.DataSource {
keys := make([]string, 0, len(p.DataSourcesMap))
for k, _ := range p.DataSourcesMap {
keys = append(keys, k)
}
sort.Strings(keys)
result := make([]terraform.DataSource, 0, len(keys))
for _, k := range keys {
result = append(result, terraform.DataSource{
Name: k,
})
}
return result
}

View File

@ -132,6 +132,38 @@ func TestProviderResources(t *testing.T) {
}
}
func TestProviderDataSources(t *testing.T) {
cases := []struct {
P *Provider
Result []terraform.DataSource
}{
{
P: &Provider{},
Result: []terraform.DataSource{},
},
{
P: &Provider{
DataSourcesMap: map[string]*Resource{
"foo": nil,
"bar": nil,
},
},
Result: []terraform.DataSource{
terraform.DataSource{Name: "bar"},
terraform.DataSource{Name: "foo"},
},
},
}
for i, tc := range cases {
actual := tc.P.DataSources()
if !reflect.DeepEqual(actual, tc.Result) {
t.Fatalf("%d: got %#v; want %#v", i, actual, tc.Result)
}
}
}
func TestProviderValidate(t *testing.T) {
cases := []struct {
P *Provider

View File

@ -175,6 +175,33 @@ func (r *Resource) Validate(c *terraform.ResourceConfig) ([]string, []error) {
return schemaMap(r.Schema).Validate(c)
}
// ReadDataApply loads the data for a data source, given a diff that
// describes the configuration arguments and desired computed attributes.
func (r *Resource) ReadDataApply(
d *terraform.InstanceDiff,
meta interface{},
) (*terraform.InstanceState, error) {
// Data sources are always built completely from scratch
// on each read, so the source state is always nil.
data, err := schemaMap(r.Schema).Data(nil, d)
if err != nil {
return nil, err
}
err = r.Read(data, meta)
state := data.State()
if state != nil && state.ID == "" {
// Data sources can set an ID if they want, but they aren't
// required to; we'll provide a placeholder if they don't,
// to preserve the invariant that all resources have non-empty
// ids.
state.ID = "-"
}
return r.recordCurrentSchemaVersion(state), err
}
// Refresh refreshes the state of the resource.
func (r *Resource) Refresh(
s *terraform.InstanceState,

View File

@ -156,6 +156,30 @@ func (p *ResourceProvider) Diff(
return resp.Diff, err
}
func (p *ResourceProvider) ValidateDataSource(
t string, c *terraform.ResourceConfig) ([]string, []error) {
var resp ResourceProviderValidateResourceResponse
args := ResourceProviderValidateResourceArgs{
Config: c,
Type: t,
}
err := p.Client.Call("Plugin.ValidateDataSource", &args, &resp)
if err != nil {
return nil, []error{err}
}
var errs []error
if len(resp.Errors) > 0 {
errs = make([]error, len(resp.Errors))
for i, err := range resp.Errors {
errs[i] = err
}
}
return resp.Warnings, errs
}
func (p *ResourceProvider) Refresh(
info *terraform.InstanceInfo,
s *terraform.InstanceState) (*terraform.InstanceState, error) {
@ -208,6 +232,58 @@ func (p *ResourceProvider) Resources() []terraform.ResourceType {
return result
}
func (p *ResourceProvider) ReadDataDiff(
info *terraform.InstanceInfo,
c *terraform.ResourceConfig) (*terraform.InstanceDiff, error) {
var resp ResourceProviderReadDataDiffResponse
args := &ResourceProviderReadDataDiffArgs{
Info: info,
Config: c,
}
err := p.Client.Call("Plugin.ReadDataDiff", args, &resp)
if err != nil {
return nil, err
}
if resp.Error != nil {
err = resp.Error
}
return resp.Diff, err
}
func (p *ResourceProvider) ReadDataApply(
info *terraform.InstanceInfo,
d *terraform.InstanceDiff) (*terraform.InstanceState, error) {
var resp ResourceProviderReadDataApplyResponse
args := &ResourceProviderReadDataApplyArgs{
Info: info,
Diff: d,
}
err := p.Client.Call("Plugin.ReadDataApply", args, &resp)
if err != nil {
return nil, err
}
if resp.Error != nil {
err = resp.Error
}
return resp.State, err
}
func (p *ResourceProvider) DataSources() []terraform.DataSource {
var result []terraform.DataSource
err := p.Client.Call("Plugin.DataSources", new(interface{}), &result)
if err != nil {
// TODO: panic, log, what?
return nil
}
return result
}
func (p *ResourceProvider) Close() error {
return p.Client.Close()
}
@ -275,6 +351,26 @@ type ResourceProviderImportStateResponse struct {
Error *plugin.BasicError
}
type ResourceProviderReadDataApplyArgs struct {
Info *terraform.InstanceInfo
Diff *terraform.InstanceDiff
}
type ResourceProviderReadDataApplyResponse struct {
State *terraform.InstanceState
Error *plugin.BasicError
}
type ResourceProviderReadDataDiffArgs struct {
Info *terraform.InstanceInfo
Config *terraform.ResourceConfig
}
type ResourceProviderReadDataDiffResponse struct {
Diff *terraform.InstanceDiff
Error *plugin.BasicError
}
type ResourceProviderValidateArgs struct {
Config *terraform.ResourceConfig
}
@ -408,3 +504,47 @@ func (s *ResourceProviderServer) Resources(
*result = s.Provider.Resources()
return nil
}
func (s *ResourceProviderServer) ValidateDataSource(
args *ResourceProviderValidateResourceArgs,
reply *ResourceProviderValidateResourceResponse) error {
warns, errs := s.Provider.ValidateDataSource(args.Type, args.Config)
berrs := make([]*plugin.BasicError, len(errs))
for i, err := range errs {
berrs[i] = plugin.NewBasicError(err)
}
*reply = ResourceProviderValidateResourceResponse{
Warnings: warns,
Errors: berrs,
}
return nil
}
func (s *ResourceProviderServer) ReadDataDiff(
args *ResourceProviderReadDataDiffArgs,
result *ResourceProviderReadDataDiffResponse) error {
diff, err := s.Provider.ReadDataDiff(args.Info, args.Config)
*result = ResourceProviderReadDataDiffResponse{
Diff: diff,
Error: plugin.NewBasicError(err),
}
return nil
}
func (s *ResourceProviderServer) ReadDataApply(
args *ResourceProviderReadDataApplyArgs,
result *ResourceProviderReadDataApplyResponse) error {
newState, err := s.Provider.ReadDataApply(args.Info, args.Diff)
*result = ResourceProviderReadDataApplyResponse{
State: newState,
Error: plugin.NewBasicError(err),
}
return nil
}
func (s *ResourceProviderServer) DataSources(
nothing interface{},
result *[]terraform.DataSource) error {
*result = s.Provider.DataSources()
return nil
}

View File

@ -389,6 +389,77 @@ func TestResourceProvider_resources(t *testing.T) {
}
}
func TestResourceProvider_readdataapply(t *testing.T) {
p := new(terraform.MockResourceProvider)
// Create a mock provider
client, _ := plugin.TestPluginRPCConn(t, pluginMap(&ServeOpts{
ProviderFunc: testProviderFixed(p),
}))
defer client.Close()
// Request the provider
raw, err := client.Dispense(ProviderPluginName)
if err != nil {
t.Fatalf("err: %s", err)
}
provider := raw.(terraform.ResourceProvider)
p.ReadDataApplyReturn = &terraform.InstanceState{
ID: "bob",
}
// ReadDataApply
info := &terraform.InstanceInfo{}
diff := &terraform.InstanceDiff{}
newState, err := provider.ReadDataApply(info, diff)
if !p.ReadDataApplyCalled {
t.Fatal("ReadDataApply should be called")
}
if !reflect.DeepEqual(p.ReadDataApplyDiff, diff) {
t.Fatalf("bad: %#v", p.ReadDataApplyDiff)
}
if err != nil {
t.Fatalf("bad: %#v", err)
}
if !reflect.DeepEqual(p.ReadDataApplyReturn, newState) {
t.Fatalf("bad: %#v", newState)
}
}
func TestResourceProvider_datasources(t *testing.T) {
p := new(terraform.MockResourceProvider)
// Create a mock provider
client, _ := plugin.TestPluginRPCConn(t, pluginMap(&ServeOpts{
ProviderFunc: testProviderFixed(p),
}))
defer client.Close()
// Request the provider
raw, err := client.Dispense(ProviderPluginName)
if err != nil {
t.Fatalf("err: %s", err)
}
provider := raw.(terraform.ResourceProvider)
expected := []terraform.DataSource{
{"foo"},
{"bar"},
}
p.DataSourcesReturn = expected
// DataSources
result := provider.DataSources()
if !p.DataSourcesCalled {
t.Fatal("DataSources should be called")
}
if !reflect.DeepEqual(result, expected) {
t.Fatalf("bad: %#v", result)
}
}
func TestResourceProvider_validate(t *testing.T) {
p := new(terraform.MockResourceProvider)
@ -628,6 +699,44 @@ func TestResourceProvider_validateResource_warns(t *testing.T) {
}
}
func TestResourceProvider_validateDataSource(t *testing.T) {
p := new(terraform.MockResourceProvider)
// Create a mock provider
client, _ := plugin.TestPluginRPCConn(t, pluginMap(&ServeOpts{
ProviderFunc: testProviderFixed(p),
}))
defer client.Close()
// Request the provider
raw, err := client.Dispense(ProviderPluginName)
if err != nil {
t.Fatalf("err: %s", err)
}
provider := raw.(terraform.ResourceProvider)
// Configure
config := &terraform.ResourceConfig{
Raw: map[string]interface{}{"foo": "bar"},
}
w, e := provider.ValidateDataSource("foo", config)
if !p.ValidateDataSourceCalled {
t.Fatal("configure should be called")
}
if p.ValidateDataSourceType != "foo" {
t.Fatalf("bad: %#v", p.ValidateDataSourceType)
}
if !reflect.DeepEqual(p.ValidateDataSourceConfig, config) {
t.Fatalf("bad: %#v", p.ValidateDataSourceConfig)
}
if w != nil {
t.Fatalf("bad: %#v", w)
}
if e != nil {
t.Fatalf("bad: %#v", e)
}
}
func TestResourceProvider_close(t *testing.T) {
p := new(terraform.MockResourceProvider)

View File

@ -97,6 +97,35 @@ type ResourceProvider interface {
// Each rule is represented by a separate resource in Terraform,
// therefore multiple states are returned.
ImportState(*InstanceInfo, string) ([]*InstanceState, error)
/*********************************************************************
* Functions related to data resources
*********************************************************************/
// ValidateDataSource is called once at the beginning with the raw
// configuration (no interpolation done) and can return a list of warnings
// and/or errors.
//
// This is called once per data source instance.
//
// This should not assume any of the values in the resource configuration
// are valid since it is possible they have to be interpolated still.
// The primary use case of this call is to check that the required keys
// are set and that the general structure is correct.
ValidateDataSource(string, *ResourceConfig) ([]string, []error)
// DataSources returns all of the available data sources that this
// provider implements.
DataSources() []DataSource
// ReadDataDiff produces a diff that represents the state that will
// be produced when the given data source is read using a later call
// to ReadDataApply.
ReadDataDiff(*InstanceInfo, *ResourceConfig) (*InstanceDiff, error)
// ReadDataApply initializes a data instance using the configuration
// in a diff produced by ReadDataDiff.
ReadDataApply(*InstanceInfo, *InstanceDiff) (*InstanceState, error)
}
// ResourceProviderCloser is an interface that providers that can close
@ -111,6 +140,11 @@ type ResourceType struct {
Importable bool // Whether this resource supports importing
}
// DataSource is a data source that a resource provider implements.
type DataSource struct {
Name string
}
// ResourceProviderFactory is a function type that creates a new instance
// of a resource provider.
type ResourceProviderFactory func() (ResourceProvider, error)
@ -123,7 +157,7 @@ func ResourceProviderFactoryFixed(p ResourceProvider) ResourceProviderFactory {
}
}
func ProviderSatisfies(p ResourceProvider, n string) bool {
func ProviderHasResource(p ResourceProvider, n string) bool {
for _, rt := range p.Resources() {
if rt.Name == n {
return true
@ -132,3 +166,13 @@ func ProviderSatisfies(p ResourceProvider, n string) bool {
return false
}
func ProviderHasDataSource(p ResourceProvider, n string) bool {
for _, rt := range p.DataSources() {
if rt.Name == n {
return true
}
}
return false
}

View File

@ -44,6 +44,20 @@ type MockResourceProvider struct {
RefreshReturnError error
ResourcesCalled bool
ResourcesReturn []ResourceType
ReadDataApplyCalled bool
ReadDataApplyInfo *InstanceInfo
ReadDataApplyDiff *InstanceDiff
ReadDataApplyFn func(*InstanceInfo, *InstanceDiff) (*InstanceState, error)
ReadDataApplyReturn *InstanceState
ReadDataApplyReturnError error
ReadDataDiffCalled bool
ReadDataDiffInfo *InstanceInfo
ReadDataDiffDesired *ResourceConfig
ReadDataDiffFn func(*InstanceInfo, *ResourceConfig) (*InstanceDiff, error)
ReadDataDiffReturn *InstanceDiff
ReadDataDiffReturnError error
DataSourcesCalled bool
DataSourcesReturn []DataSource
ValidateCalled bool
ValidateConfig *ResourceConfig
ValidateFn func(*ResourceConfig) ([]string, []error)
@ -55,6 +69,12 @@ type MockResourceProvider struct {
ValidateResourceConfig *ResourceConfig
ValidateResourceReturnWarns []string
ValidateResourceReturnErrors []error
ValidateDataSourceFn func(string, *ResourceConfig) ([]string, []error)
ValidateDataSourceCalled bool
ValidateDataSourceType string
ValidateDataSourceConfig *ResourceConfig
ValidateDataSourceReturnWarns []string
ValidateDataSourceReturnErrors []error
ImportStateCalled bool
ImportStateInfo *InstanceInfo
@ -196,3 +216,59 @@ func (p *MockResourceProvider) ImportState(info *InstanceInfo, id string) ([]*In
return p.ImportStateReturn, p.ImportStateReturnError
}
func (p *MockResourceProvider) ValidateDataSource(t string, c *ResourceConfig) ([]string, []error) {
p.Lock()
defer p.Unlock()
p.ValidateDataSourceCalled = true
p.ValidateDataSourceType = t
p.ValidateDataSourceConfig = c
if p.ValidateDataSourceFn != nil {
return p.ValidateDataSourceFn(t, c)
}
return p.ValidateDataSourceReturnWarns, p.ValidateDataSourceReturnErrors
}
func (p *MockResourceProvider) ReadDataDiff(
info *InstanceInfo,
desired *ResourceConfig) (*InstanceDiff, error) {
p.Lock()
defer p.Unlock()
p.ReadDataDiffCalled = true
p.ReadDataDiffInfo = info
p.ReadDataDiffDesired = desired
if p.ReadDataDiffFn != nil {
return p.ReadDataDiffFn(info, desired)
}
return p.ReadDataDiffReturn, p.ReadDataDiffReturnError
}
func (p *MockResourceProvider) ReadDataApply(
info *InstanceInfo,
d *InstanceDiff) (*InstanceState, error) {
p.Lock()
defer p.Unlock()
p.ReadDataApplyCalled = true
p.ReadDataApplyInfo = info
p.ReadDataApplyDiff = d
if p.ReadDataApplyFn != nil {
return p.ReadDataApplyFn(info, d)
}
return p.ReadDataApplyReturn, p.ReadDataApplyReturnError
}
func (p *MockResourceProvider) DataSources() []DataSource {
p.Lock()
defer p.Unlock()
p.DataSourcesCalled = true
return p.DataSourcesReturn
}