terraform/helper/schema/resource_data.go

503 lines
13 KiB
Go
Raw Normal View History

2014-08-13 23:23:22 +02:00
package schema
import (
"log"
2014-08-18 19:00:41 +02:00
"reflect"
"strings"
2014-08-19 00:41:12 +02:00
"sync"
"time"
"github.com/hashicorp/terraform/terraform"
)
2014-08-27 06:52:09 +02:00
// ResourceData is used to query and set the attributes of a resource.
//
// ResourceData is the primary argument received for CRUD operations on
// a resource as well as configuration of a provider. It is a powerful
// structure that can be used to not only query data, but check for changes,
// define partial state updates, etc.
//
// The most relevant methods to take a look at are Get, Set, and Partial.
type ResourceData struct {
// Settable (internally)
schema map[string]*Schema
config *terraform.ResourceConfig
state *terraform.InstanceState
diff *terraform.InstanceDiff
meta map[string]interface{}
timeouts *ResourceTimeout
2014-08-27 06:52:09 +02:00
// Don't set
2015-01-09 03:02:19 +01:00
multiReader *MultiLevelFieldReader
setWriter *MapFieldWriter
2015-01-09 03:02:19 +01:00
newState *terraform.InstanceState
partial bool
partialMap map[string]struct{}
once sync.Once
isNew bool
2014-08-27 06:52:09 +02:00
}
2014-08-22 08:03:04 +02:00
// getResult is the internal structure that is generated when a Get
// is called that contains some extra data that might be used.
type getResult struct {
Value interface{}
ValueProcessed interface{}
Computed bool
Exists bool
Schema *Schema
2014-08-22 08:03:04 +02:00
}
// UnsafeSetFieldRaw allows setting arbitrary values in state to arbitrary
// values, bypassing schema. This MUST NOT be used in normal circumstances -
// it exists only to support the remote_state data source.
func (d *ResourceData) UnsafeSetFieldRaw(key string, value string) {
d.once.Do(d.init)
d.setWriter.unsafeWriteField(key, value)
}
2014-08-22 08:03:04 +02:00
// Get returns the data for the given key, or nil if the key doesn't exist
// in the schema.
2014-08-16 02:46:05 +02:00
//
2014-08-22 08:03:04 +02:00
// If the key does exist in the schema but doesn't exist in the configuration,
// then the default value for that type will be returned. For strings, this is
// "", for numbers it is 0, etc.
//
2014-08-27 06:52:09 +02:00
// If you want to test if something is set at all in the configuration,
// use GetOk.
2014-08-15 04:55:47 +02:00
func (d *ResourceData) Get(key string) interface{} {
2014-08-22 08:03:04 +02:00
v, _ := d.GetOk(key)
return v
2014-08-18 18:58:44 +02:00
}
// GetChange returns the old and new value for a given key.
//
2014-08-27 06:52:09 +02:00
// HasChange should be used to check if a change exists. It is possible
// that both the old and new value are the same if the old value was not
// set and the new value is. This is common, for example, for boolean
// fields which have a zero value of false.
2014-08-18 18:58:44 +02:00
func (d *ResourceData) GetChange(key string) (interface{}, interface{}) {
o, n := d.getChange(key, getSourceState, getSourceDiff)
2014-08-22 08:03:04 +02:00
return o.Value, n.Value
}
// GetOk returns the data for the given key and whether or not the key
// has been set to a non-zero value at some point.
2014-08-27 06:52:09 +02:00
//
// The first result will not necessarilly be nil if the value doesn't exist.
// The second result should be checked to determine this information.
2014-08-22 08:03:04 +02:00
func (d *ResourceData) GetOk(key string) (interface{}, bool) {
r := d.getRaw(key, getSourceSet)
exists := r.Exists && !r.Computed
if exists {
// If it exists, we also want to verify it is not the zero-value.
value := r.Value
zero := r.Schema.Type.Zero()
if eq, ok := value.(Equal); ok {
exists = !eq.Equal(zero)
} else {
exists = !reflect.DeepEqual(value, zero)
}
}
return r.Value, exists
}
2014-08-27 05:19:44 +02:00
func (d *ResourceData) getRaw(key string, level getSource) getResult {
2014-08-22 08:03:04 +02:00
var parts []string
if key != "" {
parts = strings.Split(key, ".")
}
return d.get(parts, level)
}
2014-08-18 19:00:41 +02:00
// HasChange returns whether or not the given key has been changed.
func (d *ResourceData) HasChange(key string) bool {
o, n := d.GetChange(key)
// If the type implements the Equal interface, then call that
// instead of just doing a reflect.DeepEqual. An example where this is
// needed is *Set
if eq, ok := o.(Equal); ok {
return !eq.Equal(n)
}
2014-08-18 19:00:41 +02:00
return !reflect.DeepEqual(o, n)
}
2014-08-27 05:19:44 +02:00
// Partial turns partial state mode on/off.
//
// When partial state mode is enabled, then only key prefixes specified
// by SetPartial will be in the final state. This allows providers to return
// partial states for partially applied resources (when errors occur).
func (d *ResourceData) Partial(on bool) {
d.partial = on
if on {
if d.partialMap == nil {
d.partialMap = make(map[string]struct{})
}
2014-08-27 05:19:44 +02:00
} else {
d.partialMap = nil
}
}
2014-08-17 00:02:51 +02:00
// Set sets the value for the given key.
//
// If the key is invalid or the value is not a correct type, an error
// will be returned.
func (d *ResourceData) Set(key string, value interface{}) error {
2015-01-09 03:02:19 +01:00
d.once.Do(d.init)
// If the value is a pointer to a non-struct, get its value and
// use that. This allows Set to take a pointer to primitives to
// simplify the interface.
reflectVal := reflect.ValueOf(value)
if reflectVal.Kind() == reflect.Ptr {
if reflectVal.IsNil() {
// If the pointer is nil, then the value is just nil
value = nil
} else {
// Otherwise, we dereference the pointer as long as its not
// a pointer to a struct, since struct pointers are allowed.
reflectVal = reflect.Indirect(reflectVal)
if reflectVal.Kind() != reflect.Struct {
value = reflectVal.Interface()
}
}
}
return d.setWriter.WriteField(strings.Split(key, "."), value)
2014-08-17 00:02:51 +02:00
}
// SetPartial adds the key to the final state output while
// in partial state mode. The key must be a root key in the schema (i.e.
// it cannot be "list.0").
2014-08-27 05:19:44 +02:00
//
// If partial state mode is disabled, then this has no effect. Additionally,
// whenever partial state mode is toggled, the partial data is cleared.
func (d *ResourceData) SetPartial(k string) {
if d.partial {
d.partialMap[k] = struct{}{}
}
}
func (d *ResourceData) MarkNewResource() {
d.isNew = true
}
func (d *ResourceData) IsNewResource() bool {
return d.isNew
}
// Id returns the ID of the resource.
func (d *ResourceData) Id() string {
var result string
if d.state != nil {
result = d.state.ID
}
if d.newState != nil {
result = d.newState.ID
}
return result
}
2014-08-22 07:15:47 +02:00
// ConnInfo returns the connection info for this resource.
func (d *ResourceData) ConnInfo() map[string]string {
if d.newState != nil {
return d.newState.Ephemeral.ConnInfo
2014-08-22 07:15:47 +02:00
}
if d.state != nil {
return d.state.Ephemeral.ConnInfo
2014-08-19 00:41:12 +02:00
}
return nil
}
// SetId sets the ID of the resource. If the value is blank, then the
// resource is destroyed.
func (d *ResourceData) SetId(v string) {
2014-08-19 00:41:12 +02:00
d.once.Do(d.init)
d.newState.ID = v
}
2014-08-22 07:15:47 +02:00
// SetConnInfo sets the connection info for a resource.
func (d *ResourceData) SetConnInfo(v map[string]string) {
d.once.Do(d.init)
d.newState.Ephemeral.ConnInfo = v
2014-08-19 00:41:12 +02:00
}
// SetType sets the ephemeral type for the data. This is only required
// for importing.
func (d *ResourceData) SetType(t string) {
d.once.Do(d.init)
d.newState.Ephemeral.Type = t
}
// State returns the new InstanceState after the diff and any Set
// calls.
func (d *ResourceData) State() *terraform.InstanceState {
var result terraform.InstanceState
result.ID = d.Id()
result.Meta = d.meta
// If we have no ID, then this resource doesn't exist and we just
// return nil.
if result.ID == "" {
return nil
}
if d.timeouts != nil {
if err := d.timeouts.StateEncode(&result); err != nil {
log.Printf("[ERR] Error encoding Timeout meta to Instance State: %s", err)
}
}
// Look for a magic key in the schema that determines we skip the
// integrity check of fields existing in the schema, allowing dynamic
// keys to be created.
hasDynamicAttributes := false
for k, _ := range d.schema {
if k == "__has_dynamic_attributes" {
hasDynamicAttributes = true
log.Printf("[INFO] Resource %s has dynamic attributes", result.ID)
}
}
// In order to build the final state attributes, we read the full
// attribute set as a map[string]interface{}, write it to a MapFieldWriter,
// and then use that map.
rawMap := make(map[string]interface{})
Change Set internals and make (extreme) performance improvements Changing the Set internals makes a lot of sense as it saves doing conversions in multiple places and gives a central place to alter the key when a item is computed. This will have no side effects other then that the ordering is now based on strings instead on integers, so the order will be different. This will however have no effect on existing configs as these will use the individual codes/keys and not the ordering to determine if there is a diff or not. Lastly (but I think also most importantly) there is a fix in this PR that makes diffing sets extremely more performand. Before a full diff required reading the complete Set for every single parameter/attribute you wanted to diff, while now it only gets that specific parameter. We have a use case where we have a Set that has 18 parameters and the set consist of about 600 items (don't ask :wink:). So when doing a diff it would take 100% CPU of all cores and stay that way for almost an hour before being able to complete the diff. Debugging this we learned that for retrieving every single parameter it made over 52.000 calls to `func (c *ResourceConfig) get(..)`. In this function a slice is created and used only for the duration of the call, so the time needed to create all needed slices and on the other hand the time the garbage collector needed to clean them up again caused the system to cripple itself. Next to that there are also some expensive reflect calls in this function which also claimed a fair amount of CPU time. After this fix the number of calls needed to get a single parameter dropped from 52.000+ to only 2! :smiley:
2015-11-18 11:24:04 +01:00
for k := range d.schema {
source := getSourceSet
if d.partial {
source = getSourceState
if _, ok := d.partialMap[k]; ok {
source = getSourceSet
}
}
raw := d.get([]string{k}, source)
if raw.Exists && !raw.Computed {
rawMap[k] = raw.Value
if raw.ValueProcessed != nil {
rawMap[k] = raw.ValueProcessed
}
}
}
mapW := &MapFieldWriter{Schema: d.schema}
if err := mapW.WriteField(nil, rawMap); err != nil {
return nil
}
result.Attributes = mapW.Map()
if hasDynamicAttributes {
// If we have dynamic attributes, just copy the attributes map
// one for one into the result attributes.
for k, v := range d.setWriter.Map() {
// Don't clobber schema values. This limits usage of dynamic
// attributes to names which _do not_ conflict with schema
// keys!
if _, ok := result.Attributes[k]; !ok {
result.Attributes[k] = v
}
}
}
if d.newState != nil {
result.Ephemeral = d.newState.Ephemeral
}
2014-08-18 04:45:26 +02:00
// TODO: This is hacky and we can remove this when we have a proper
// state writer. We should instead have a proper StateFieldWriter
// and use that.
for k, schema := range d.schema {
if schema.Type != TypeMap {
continue
}
if result.Attributes[k] == "" {
delete(result.Attributes, k)
}
}
if v := d.Id(); v != "" {
result.Attributes["id"] = d.Id()
}
if d.state != nil {
result.Tainted = d.state.Tainted
}
return &result
}
// Timeout returns the data for the given timeout key
// Returns a duration of 20 minutes for any key not found, or not found and no default.
func (d *ResourceData) Timeout(key string) time.Duration {
key = strings.ToLower(key)
var timeout *time.Duration
switch key {
case TimeoutCreate:
timeout = d.timeouts.Create
case TimeoutRead:
timeout = d.timeouts.Read
case TimeoutUpdate:
timeout = d.timeouts.Update
case TimeoutDelete:
timeout = d.timeouts.Delete
}
if timeout != nil {
return *timeout
}
if d.timeouts.Default != nil {
return *d.timeouts.Default
}
// Return system default of 20 minutes
return 20 * time.Minute
}
2014-08-19 00:41:12 +02:00
func (d *ResourceData) init() {
2015-01-09 03:02:19 +01:00
// Initialize the field that will store our new state
var copyState terraform.InstanceState
2014-08-19 00:41:12 +02:00
if d.state != nil {
copyState = *d.state.DeepCopy()
2014-08-19 00:41:12 +02:00
}
d.newState = &copyState
2015-01-09 03:02:19 +01:00
// Initialize the map for storing set data
d.setWriter = &MapFieldWriter{Schema: d.schema}
2015-01-09 03:02:19 +01:00
// Initialize the reader for getting data from the
// underlying sources (config, diff, etc.)
readers := make(map[string]FieldReader)
var stateAttributes map[string]string
if d.state != nil {
stateAttributes = d.state.Attributes
readers["state"] = &MapFieldReader{
Schema: d.schema,
Map: BasicMapReader(stateAttributes),
}
}
if d.config != nil {
readers["config"] = &ConfigFieldReader{
Schema: d.schema,
Config: d.config,
}
}
if d.diff != nil {
readers["diff"] = &DiffFieldReader{
Schema: d.schema,
Diff: d.diff,
Source: &MultiLevelFieldReader{
Levels: []string{"state", "config"},
Readers: readers,
},
}
}
readers["set"] = &MapFieldReader{
Schema: d.schema,
Map: BasicMapReader(d.setWriter.Map()),
2015-01-09 03:02:19 +01:00
}
d.multiReader = &MultiLevelFieldReader{
Levels: []string{
"state",
"config",
"diff",
"set",
},
Readers: readers,
}
2014-08-19 00:41:12 +02:00
}
func (d *ResourceData) diffChange(
k string) (interface{}, interface{}, bool, bool) {
// Get the change between the state and the config.
o, n := d.getChange(k, getSourceState, getSourceConfig|getSourceExact)
2014-08-22 08:03:04 +02:00
if !o.Exists {
o.Value = nil
}
if !n.Exists {
n.Value = nil
}
// Return the old, new, and whether there is a change
return o.Value, n.Value, !reflect.DeepEqual(o.Value, n.Value), n.Computed
}
func (d *ResourceData) getChange(
Change Set internals and make (extreme) performance improvements Changing the Set internals makes a lot of sense as it saves doing conversions in multiple places and gives a central place to alter the key when a item is computed. This will have no side effects other then that the ordering is now based on strings instead on integers, so the order will be different. This will however have no effect on existing configs as these will use the individual codes/keys and not the ordering to determine if there is a diff or not. Lastly (but I think also most importantly) there is a fix in this PR that makes diffing sets extremely more performand. Before a full diff required reading the complete Set for every single parameter/attribute you wanted to diff, while now it only gets that specific parameter. We have a use case where we have a Set that has 18 parameters and the set consist of about 600 items (don't ask :wink:). So when doing a diff it would take 100% CPU of all cores and stay that way for almost an hour before being able to complete the diff. Debugging this we learned that for retrieving every single parameter it made over 52.000 calls to `func (c *ResourceConfig) get(..)`. In this function a slice is created and used only for the duration of the call, so the time needed to create all needed slices and on the other hand the time the garbage collector needed to clean them up again caused the system to cripple itself. Next to that there are also some expensive reflect calls in this function which also claimed a fair amount of CPU time. After this fix the number of calls needed to get a single parameter dropped from 52.000+ to only 2! :smiley:
2015-11-18 11:24:04 +01:00
k string,
oldLevel getSource,
2014-08-22 08:03:04 +02:00
newLevel getSource) (getResult, getResult) {
2014-08-21 06:02:42 +02:00
var parts, parts2 []string
Change Set internals and make (extreme) performance improvements Changing the Set internals makes a lot of sense as it saves doing conversions in multiple places and gives a central place to alter the key when a item is computed. This will have no side effects other then that the ordering is now based on strings instead on integers, so the order will be different. This will however have no effect on existing configs as these will use the individual codes/keys and not the ordering to determine if there is a diff or not. Lastly (but I think also most importantly) there is a fix in this PR that makes diffing sets extremely more performand. Before a full diff required reading the complete Set for every single parameter/attribute you wanted to diff, while now it only gets that specific parameter. We have a use case where we have a Set that has 18 parameters and the set consist of about 600 items (don't ask :wink:). So when doing a diff it would take 100% CPU of all cores and stay that way for almost an hour before being able to complete the diff. Debugging this we learned that for retrieving every single parameter it made over 52.000 calls to `func (c *ResourceConfig) get(..)`. In this function a slice is created and used only for the duration of the call, so the time needed to create all needed slices and on the other hand the time the garbage collector needed to clean them up again caused the system to cripple itself. Next to that there are also some expensive reflect calls in this function which also claimed a fair amount of CPU time. After this fix the number of calls needed to get a single parameter dropped from 52.000+ to only 2! :smiley:
2015-11-18 11:24:04 +01:00
if k != "" {
parts = strings.Split(k, ".")
parts2 = strings.Split(k, ".")
}
o := d.get(parts, oldLevel)
n := d.get(parts2, newLevel)
return o, n
}
func (d *ResourceData) get(addr []string, source getSource) getResult {
2015-01-09 03:02:19 +01:00
d.once.Do(d.init)
level := "set"
flags := source & ^getSourceLevelMask
exact := flags&getSourceExact != 0
source = source & getSourceLevelMask
if source >= getSourceSet {
level = "set"
} else if source >= getSourceDiff {
2015-01-09 03:02:19 +01:00
level = "diff"
} else if source >= getSourceConfig {
level = "config"
} else {
level = "state"
}
var result FieldReadResult
var err error
if exact {
result, err = d.multiReader.ReadFieldExact(addr, level)
} else {
result, err = d.multiReader.ReadFieldMerge(addr, level)
}
if err != nil {
panic(err)
}
// If the result doesn't exist, then we set the value to the zero value
var schema *Schema
if schemaL := addrToSchema(addr, d.schema); len(schemaL) > 0 {
schema = schemaL[len(schemaL)-1]
}
if result.Value == nil && schema != nil {
result.Value = result.ValueOrZero(schema)
2015-01-09 03:02:19 +01:00
}
// Transform the FieldReadResult into a getResult. It might be worth
// merging these two structures one day.
return getResult{
Value: result.Value,
ValueProcessed: result.ValueProcessed,
Computed: result.Computed,
Exists: result.Exists,
Schema: schema,
}
}