From 706ccb7dfedb9807fd65643a6e4eefd6a8b4f8f0 Mon Sep 17 00:00:00 2001 From: James Nugent Date: Tue, 7 Jun 2016 21:12:55 +0200 Subject: [PATCH] core: Introduce state v3 and upgrade process This commit makes the current Terraform state version 3 (previously 2), and a migration process as part of reading v2 state. For the most part this is unnecessary: helper/schema will deal with upgrading state for providers written with that framework. However, for providers which implemented the resource model directly, this gives a best-efforts attempt at lossless upgrade. The heuristics used to change the count of a map from the .# key to the .% key are as follows: - if the flat map contains any non-numeric keys, we treat it as a map - if the map is empty it must be computed or optional, so we remove it from state There is a known edge condition: maps with all-numeric keys are indistinguishable from sets without access to the schema. They will need manual conversion or may result in spurious diffs. --- state/remote/atlas_test.go | 4 +- terraform/state.go | 108 +++++++++++++---- terraform/state_test.go | 8 +- terraform/state_upgrade_v1_to_v2.go | 178 +++++++++++++++++++++++++++ terraform/state_upgrade_v2_to_v3.go | 142 ++++++++++++++++++++++ terraform/state_v1.go | 180 +--------------------------- 6 files changed, 413 insertions(+), 207 deletions(-) create mode 100644 terraform/state_upgrade_v1_to_v2.go create mode 100644 terraform/state_upgrade_v2_to_v3.go diff --git a/state/remote/atlas_test.go b/state/remote/atlas_test.go index 701fbcbd8..1d73540a4 100644 --- a/state/remote/atlas_test.go +++ b/state/remote/atlas_test.go @@ -250,7 +250,7 @@ func (f *fakeAtlas) handler(resp http.ResponseWriter, req *http.Request) { // loads the state. var testStateModuleOrderChange = []byte( `{ - "version": 2, + "version": 3, "serial": 1, "modules": [ { @@ -289,7 +289,7 @@ var testStateModuleOrderChange = []byte( var testStateSimple = []byte( `{ - "version": 2, + "version": 3, "serial": 1, "modules": [ { diff --git a/terraform/state.go b/terraform/state.go index 63dbe95ef..c41ef15dd 100644 --- a/terraform/state.go +++ b/terraform/state.go @@ -19,7 +19,7 @@ import ( const ( // StateVersion is the current version for our state file - StateVersion = 2 + StateVersion = 3 ) // rootModulePath is the path of the root module @@ -1379,24 +1379,32 @@ type jsonStateVersionIdentifier struct { Version int `json:"version"` } +// Check if this is a V0 format - the magic bytes at the start of the file +// should be "tfstate" if so. We no longer support upgrading this type of +// state but return an error message explaining to a user how they can +// upgrade via the 0.6.x series. +func testForV0State(buf *bufio.Reader) error { + start, err := buf.Peek(len("tfstate")) + if err != nil { + return fmt.Errorf("Failed to check for magic bytes: %v", err) + } + if string(start) == "tfstate" { + return fmt.Errorf("Terraform 0.7 no longer supports upgrading the binary state\n" + + "format which was used prior to Terraform 0.3. Please upgrade\n" + + "this state file using Terraform 0.6.16 prior to using it with\n" + + "Terraform 0.7.") + } + + return nil +} + // ReadState reads a state structure out of a reader in the format that // was written by WriteState. func ReadState(src io.Reader) (*State, error) { buf := bufio.NewReader(src) - // Check if this is a V0 format - the magic bytes at the start of the file - // should be "tfstate" if so. We no longer support upgrading this type of - // state but display an error message explaining to a user how they can - // upgrade via the 0.6.x series. - start, err := buf.Peek(len("tfstate")) - if err != nil { - return nil, fmt.Errorf("Failed to check for magic bytes: %v", err) - } - if string(start) == "tfstate" { - return nil, fmt.Errorf("Terraform 0.7 no longer supports upgrading the binary state\n" + - "format which was used prior to Terraform 0.3. Please upgrade\n" + - "this state file using Terraform 0.6.16 prior to using it with\n" + - "Terraform 0.7.") + if err := testForV0State(buf); err != nil { + return nil, err } // If we are JSON we buffer the whole thing in memory so we can read it twice. @@ -1415,17 +1423,39 @@ func ReadState(src io.Reader) (*State, error) { case 0: return nil, fmt.Errorf("State version 0 is not supported as JSON.") case 1: - old, err := ReadStateV1(jsonBytes) + v1State, err := ReadStateV1(jsonBytes) if err != nil { return nil, err } - return old.upgrade() + + v2State, err := upgradeStateV1ToV2(v1State) + if err != nil { + return nil, err + } + + v3State, err := upgradeStateV2ToV3(v2State) + if err != nil { + return nil, err + } + + return v3State, nil case 2: - state, err := ReadStateV2(jsonBytes) + v2State, err := ReadStateV2(jsonBytes) if err != nil { return nil, err } - return state, nil + v3State, err := upgradeStateV2ToV3(v2State) + if err != nil { + return nil, err + } + + return v3State, nil + case 3: + v3State, err := ReadStateV3(jsonBytes) + if err != nil { + return nil, err + } + return v3State, nil default: return nil, fmt.Errorf("State version %d not supported, please update.", versionIdentifier.Version) @@ -1433,20 +1463,52 @@ func ReadState(src io.Reader) (*State, error) { } func ReadStateV1(jsonBytes []byte) (*stateV1, error) { - state := &stateV1{} + v1State := &stateV1{} + if err := json.Unmarshal(jsonBytes, v1State); err != nil { + return nil, fmt.Errorf("Decoding state file failed: %v", err) + } + + if v1State.Version != 1 { + return nil, fmt.Errorf("Decoded state version did not match the decoder selection: "+ + "read %d, expected 1", v1State.Version) + } + + return v1State, nil +} + +func ReadStateV2(jsonBytes []byte) (*State, error) { + state := &State{} if err := json.Unmarshal(jsonBytes, state); err != nil { return nil, fmt.Errorf("Decoding state file failed: %v", err) } - if state.Version != 1 { - return nil, fmt.Errorf("Decoded state version did not match the decoder selection: "+ - "read %d, expected 1", state.Version) + // Check the version, this to ensure we don't read a future + // version that we don't understand + if state.Version > StateVersion { + return nil, fmt.Errorf("State version %d not supported, please update.", + state.Version) } + // Make sure the version is semantic + if state.TFVersion != "" { + if _, err := version.NewVersion(state.TFVersion); err != nil { + return nil, fmt.Errorf( + "State contains invalid version: %s\n\n"+ + "Terraform validates the version format prior to writing it. This\n"+ + "means that this is invalid of the state becoming corrupted through\n"+ + "some external means. Please manually modify the Terraform version\n"+ + "field to be a proper semantic version.", + state.TFVersion) + } + } + + // Sort it + state.sort() + return state, nil } -func ReadStateV2(jsonBytes []byte) (*State, error) { +func ReadStateV3(jsonBytes []byte) (*State, error) { state := &State{} if err := json.Unmarshal(jsonBytes, state); err != nil { return nil, fmt.Errorf("Decoding state file failed: %v", err) diff --git a/terraform/state_test.go b/terraform/state_test.go index f265074aa..afab8daec 100644 --- a/terraform/state_test.go +++ b/terraform/state_test.go @@ -1128,7 +1128,7 @@ func TestInstanceState_MergeDiff_nilDiff(t *testing.T) { } } -func TestReadUpgradeStateV1toV2(t *testing.T) { +func TestReadUpgradeStateV1toV3(t *testing.T) { // ReadState should transparently detect the old version but will upgrade // it on Write. actual, err := ReadState(strings.NewReader(testV1State)) @@ -1141,7 +1141,7 @@ func TestReadUpgradeStateV1toV2(t *testing.T) { t.Fatalf("err: %s", err) } - if actual.Version != 2 { + if actual.Version != 3 { t.Fatalf("bad: State version not incremented; is %d", actual.Version) } @@ -1155,7 +1155,7 @@ func TestReadUpgradeStateV1toV2(t *testing.T) { } } -func TestReadUpgradeStateV1toV2_outputs(t *testing.T) { +func TestReadUpgradeStateV1toV3_outputs(t *testing.T) { // ReadState should transparently detect the old version but will upgrade // it on Write. actual, err := ReadState(strings.NewReader(testV1StateWithOutputs)) @@ -1168,7 +1168,7 @@ func TestReadUpgradeStateV1toV2_outputs(t *testing.T) { t.Fatalf("err: %s", err) } - if actual.Version != 2 { + if actual.Version != 3 { t.Fatalf("bad: State version not incremented; is %d", actual.Version) } diff --git a/terraform/state_upgrade_v1_to_v2.go b/terraform/state_upgrade_v1_to_v2.go new file mode 100644 index 000000000..98d58d47d --- /dev/null +++ b/terraform/state_upgrade_v1_to_v2.go @@ -0,0 +1,178 @@ +package terraform + +import ( + "fmt" + "github.com/mitchellh/copystructure" +) + +// upgradeStateV1ToV2 is used to upgrade a V1 state representation +// into a V2 state representation +func upgradeStateV1ToV2(old *stateV1) (*State, error) { + if old == nil { + return nil, nil + } + + remote, err := old.Remote.upgradeToV2() + if err != nil { + return nil, fmt.Errorf("Error upgrading State V1: %v", err) + } + + modules := make([]*ModuleState, len(old.Modules)) + for i, module := range old.Modules { + upgraded, err := module.upgradeToV2() + if err != nil { + return nil, fmt.Errorf("Error upgrading State V1: %v", err) + } + modules[i] = upgraded + } + if len(modules) == 0 { + modules = nil + } + + newState := &State{ + Version: 2, + Serial: old.Serial, + Remote: remote, + Modules: modules, + } + + newState.sort() + + return newState, nil +} + +func (old *remoteStateV1) upgradeToV2() (*RemoteState, error) { + if old == nil { + return nil, nil + } + + config, err := copystructure.Copy(old.Config) + if err != nil { + return nil, fmt.Errorf("Error upgrading RemoteState V1: %v", err) + } + + return &RemoteState{ + Type: old.Type, + Config: config.(map[string]string), + }, nil +} + +func (old *moduleStateV1) upgradeToV2() (*ModuleState, error) { + if old == nil { + return nil, nil + } + + path, err := copystructure.Copy(old.Path) + if err != nil { + return nil, fmt.Errorf("Error upgrading ModuleState V1: %v", err) + } + + // Outputs needs upgrading to use the new structure + outputs := make(map[string]*OutputState) + for key, output := range old.Outputs { + outputs[key] = &OutputState{ + Type: "string", + Value: output, + Sensitive: false, + } + } + if len(outputs) == 0 { + outputs = nil + } + + resources := make(map[string]*ResourceState) + for key, oldResource := range old.Resources { + upgraded, err := oldResource.upgradeToV2() + if err != nil { + return nil, fmt.Errorf("Error upgrading ModuleState V1: %v", err) + } + resources[key] = upgraded + } + if len(resources) == 0 { + resources = nil + } + + dependencies, err := copystructure.Copy(old.Dependencies) + if err != nil { + return nil, fmt.Errorf("Error upgrading ModuleState V1: %v", err) + } + + return &ModuleState{ + Path: path.([]string), + Outputs: outputs, + Resources: resources, + Dependencies: dependencies.([]string), + }, nil +} + +func (old *resourceStateV1) upgradeToV2() (*ResourceState, error) { + if old == nil { + return nil, nil + } + + dependencies, err := copystructure.Copy(old.Dependencies) + if err != nil { + return nil, fmt.Errorf("Error upgrading ResourceState V1: %v", err) + } + + primary, err := old.Primary.upgradeToV2() + if err != nil { + return nil, fmt.Errorf("Error upgrading ResourceState V1: %v", err) + } + + deposed := make([]*InstanceState, len(old.Deposed)) + for i, v := range old.Deposed { + upgraded, err := v.upgradeToV2() + if err != nil { + return nil, fmt.Errorf("Error upgrading ResourceState V1: %v", err) + } + deposed[i] = upgraded + } + if len(deposed) == 0 { + deposed = nil + } + + return &ResourceState{ + Type: old.Type, + Dependencies: dependencies.([]string), + Primary: primary, + Deposed: deposed, + Provider: old.Provider, + }, nil +} + +func (old *instanceStateV1) upgradeToV2() (*InstanceState, error) { + if old == nil { + return nil, nil + } + + attributes, err := copystructure.Copy(old.Attributes) + if err != nil { + return nil, fmt.Errorf("Error upgrading InstanceState V1: %v", err) + } + ephemeral, err := old.Ephemeral.upgradeToV2() + if err != nil { + return nil, fmt.Errorf("Error upgrading InstanceState V1: %v", err) + } + meta, err := copystructure.Copy(old.Meta) + if err != nil { + return nil, fmt.Errorf("Error upgrading InstanceState V1: %v", err) + } + + return &InstanceState{ + ID: old.ID, + Attributes: attributes.(map[string]string), + Ephemeral: *ephemeral, + Meta: meta.(map[string]string), + }, nil +} + +func (old *ephemeralStateV1) upgradeToV2() (*EphemeralState, error) { + connInfo, err := copystructure.Copy(old.ConnInfo) + if err != nil { + return nil, fmt.Errorf("Error upgrading EphemeralState V1: %v", err) + } + return &EphemeralState{ + ConnInfo: connInfo.(map[string]string), + }, nil +} diff --git a/terraform/state_upgrade_v2_to_v3.go b/terraform/state_upgrade_v2_to_v3.go new file mode 100644 index 000000000..1fc458d15 --- /dev/null +++ b/terraform/state_upgrade_v2_to_v3.go @@ -0,0 +1,142 @@ +package terraform + +import ( + "fmt" + "log" + "regexp" + "sort" + "strconv" + "strings" +) + +// The upgrade process from V2 to V3 state does not affect the structure, +// so we do not need to redeclare all of the structs involved - we just +// take a deep copy of the old structure and assert the version number is +// as we expect. +func upgradeStateV2ToV3(old *State) (*State, error) { + new := old.DeepCopy() + + // Ensure the copied version is v2 before attempting to upgrade + if new.Version != 2 { + return nil, fmt.Errorf("Cannot appply v2->v3 state upgrade to " + + "a state which is not version 2.") + } + + // Set the new version number + new.Version = 3 + + // Change the counts for things which look like maps to use the % + // syntax. Remove counts for empty collections - they will be added + // back in later. + for _, module := range new.Modules { + for _, resource := range module.Resources { + // Upgrade Primary + if resource.Primary != nil { + upgradeAttributesV2ToV3(resource.Primary) + } + + // Upgrade Deposed + if resource.Deposed != nil { + for _, deposed := range resource.Deposed { + upgradeAttributesV2ToV3(deposed) + } + } + } + } + + return new, nil +} + +func upgradeAttributesV2ToV3(instanceState *InstanceState) error { + collectionKeyRegexp := regexp.MustCompile(`^(.*\.)#$`) + collectionSubkeyRegexp := regexp.MustCompile(`^([^\.]+)\..*`) + + // Identify the key prefix of anything which is a collection + var collectionKeyPrefixes []string + for key := range instanceState.Attributes { + if submatches := collectionKeyRegexp.FindAllStringSubmatch(key, -1); len(submatches) > 0 { + collectionKeyPrefixes = append(collectionKeyPrefixes, submatches[0][1]) + } + } + sort.Strings(collectionKeyPrefixes) + + log.Printf("[STATE UPGRADE] Detected the following collections in state: %v", collectionKeyPrefixes) + + // This could be rolled into fewer loops, but it is somewhat clearer this way, and will not + // run very often. + for _, prefix := range collectionKeyPrefixes { + // First get the actual keys that belong to this prefix + var potentialKeysMatching []string + for key := range instanceState.Attributes { + if strings.HasPrefix(key, prefix) { + potentialKeysMatching = append(potentialKeysMatching, strings.TrimPrefix(key, prefix)) + } + } + sort.Strings(potentialKeysMatching) + + var actualKeysMatching []string + for _, key := range potentialKeysMatching { + if submatches := collectionSubkeyRegexp.FindAllStringSubmatch(key, -1); len(submatches) > 0 { + actualKeysMatching = append(actualKeysMatching, submatches[0][1]) + } else { + if key != "#" { + actualKeysMatching = append(actualKeysMatching, key) + } + } + } + actualKeysMatching = uniqueSortedStrings(actualKeysMatching) + + // Now inspect the keys in order to determine whether this is most likely to be + // a map, list or set. There is room for error here, so we log in each case. If + // there is no method of telling, we remove the key from the InstanceState in + // order that it will be recreated. Again, this could be rolled into fewer loops + // but we prefer clarity. + + oldCountKey := fmt.Sprintf("%s#", prefix) + + // First, detect "obvious" maps - which have non-numeric keys (mostly). + hasNonNumericKeys := false + for _, key := range actualKeysMatching { + if _, err := strconv.Atoi(key); err != nil { + hasNonNumericKeys = true + } + } + if hasNonNumericKeys { + newCountKey := fmt.Sprintf("%s%%", prefix) + + instanceState.Attributes[newCountKey] = instanceState.Attributes[oldCountKey] + delete(instanceState.Attributes, oldCountKey) + log.Printf("[STATE UPGRADE] Detected %s as a map. Replaced count = %s", + strings.TrimSuffix(prefix, "."), instanceState.Attributes[newCountKey]) + } + + // Now detect empty collections and remove them from state. + if len(actualKeysMatching) == 0 { + delete(instanceState.Attributes, oldCountKey) + log.Printf("[STATE UPGRADE] Detected %s as an empty collection. Removed from state.", + strings.TrimSuffix(prefix, ".")) + } + } + + return nil +} + +// uniqueSortedStrings removes duplicates from a slice of strings and returns +// a sorted slice of the unique strings. +func uniqueSortedStrings(input []string) []string { + uniquemap := make(map[string]struct{}) + for _, str := range input { + uniquemap[str] = struct{}{} + } + + output := make([]string, len(uniquemap)) + + i := 0 + for key := range uniquemap { + output[i] = key + i = i + 1 + } + + sort.Strings(output) + return output +} diff --git a/terraform/state_v1.go b/terraform/state_v1.go index 72611e11d..68cffb41b 100644 --- a/terraform/state_v1.go +++ b/terraform/state_v1.go @@ -1,17 +1,13 @@ package terraform -import ( - "fmt" - - "github.com/mitchellh/copystructure" -) - // stateV1 keeps track of a snapshot state-of-the-world that Terraform // can use to keep track of what real world resources it is actually // managing. // // stateV1 is _only used for the purposes of backwards compatibility // and is no longer used in Terraform. +// +// For the upgrade process, see state_upgrade_v1_to_v2.go type stateV1 struct { // Version is the protocol version. "1" for a StateV1. Version int `json:"version"` @@ -29,42 +25,6 @@ type stateV1 struct { Modules []*moduleStateV1 `json:"modules"` } -// upgrade is used to upgrade a V1 state representation -// into a State (current) representation. -func (old *stateV1) upgrade() (*State, error) { - if old == nil { - return nil, nil - } - - remote, err := old.Remote.upgrade() - if err != nil { - return nil, fmt.Errorf("Error upgrading State V1: %v", err) - } - - modules := make([]*ModuleState, len(old.Modules)) - for i, module := range old.Modules { - upgraded, err := module.upgrade() - if err != nil { - return nil, fmt.Errorf("Error upgrading State V1: %v", err) - } - modules[i] = upgraded - } - if len(modules) == 0 { - modules = nil - } - - newState := &State{ - Version: old.Version, - Serial: old.Serial, - Remote: remote, - Modules: modules, - } - - newState.sort() - - return newState, nil -} - type remoteStateV1 struct { // Type controls the client we use for the remote state Type string `json:"type"` @@ -74,22 +34,6 @@ type remoteStateV1 struct { Config map[string]string `json:"config"` } -func (old *remoteStateV1) upgrade() (*RemoteState, error) { - if old == nil { - return nil, nil - } - - config, err := copystructure.Copy(old.Config) - if err != nil { - return nil, fmt.Errorf("Error upgrading RemoteState V1: %v", err) - } - - return &RemoteState{ - Type: old.Type, - Config: config.(map[string]string), - }, nil -} - type moduleStateV1 struct { // Path is the import path from the root module. Modules imports are // always disjoint, so the path represents amodule tree @@ -121,54 +65,6 @@ type moduleStateV1 struct { Dependencies []string `json:"depends_on,omitempty"` } -func (old *moduleStateV1) upgrade() (*ModuleState, error) { - if old == nil { - return nil, nil - } - - path, err := copystructure.Copy(old.Path) - if err != nil { - return nil, fmt.Errorf("Error upgrading ModuleState V1: %v", err) - } - - // Outputs needs upgrading to use the new structure - outputs := make(map[string]*OutputState) - for key, output := range old.Outputs { - outputs[key] = &OutputState{ - Type: "string", - Value: output, - Sensitive: false, - } - } - if len(outputs) == 0 { - outputs = nil - } - - resources := make(map[string]*ResourceState) - for key, oldResource := range old.Resources { - upgraded, err := oldResource.upgrade() - if err != nil { - return nil, fmt.Errorf("Error upgrading ModuleState V1: %v", err) - } - resources[key] = upgraded - } - if len(resources) == 0 { - resources = nil - } - - dependencies, err := copystructure.Copy(old.Dependencies) - if err != nil { - return nil, fmt.Errorf("Error upgrading ModuleState V1: %v", err) - } - - return &ModuleState{ - Path: path.([]string), - Outputs: outputs, - Resources: resources, - Dependencies: dependencies.([]string), - }, nil -} - type resourceStateV1 struct { // This is filled in and managed by Terraform, and is the resource // type itself such as "mycloud_instance". If a resource provider sets @@ -220,42 +116,6 @@ type resourceStateV1 struct { Provider string `json:"provider,omitempty"` } -func (old *resourceStateV1) upgrade() (*ResourceState, error) { - if old == nil { - return nil, nil - } - - dependencies, err := copystructure.Copy(old.Dependencies) - if err != nil { - return nil, fmt.Errorf("Error upgrading ResourceState V1: %v", err) - } - - primary, err := old.Primary.upgrade() - if err != nil { - return nil, fmt.Errorf("Error upgrading ResourceState V1: %v", err) - } - - deposed := make([]*InstanceState, len(old.Deposed)) - for i, v := range old.Deposed { - upgraded, err := v.upgrade() - if err != nil { - return nil, fmt.Errorf("Error upgrading ResourceState V1: %v", err) - } - deposed[i] = upgraded - } - if len(deposed) == 0 { - deposed = nil - } - - return &ResourceState{ - Type: old.Type, - Dependencies: dependencies.([]string), - Primary: primary, - Deposed: deposed, - Provider: old.Provider, - }, nil -} - type instanceStateV1 struct { // A unique ID for this resource. This is opaque to Terraform // and is only meant as a lookup mechanism for the providers. @@ -277,45 +137,9 @@ type instanceStateV1 struct { Meta map[string]string `json:"meta,omitempty"` } -func (old *instanceStateV1) upgrade() (*InstanceState, error) { - if old == nil { - return nil, nil - } - - attributes, err := copystructure.Copy(old.Attributes) - if err != nil { - return nil, fmt.Errorf("Error upgrading InstanceState V1: %v", err) - } - ephemeral, err := old.Ephemeral.upgrade() - if err != nil { - return nil, fmt.Errorf("Error upgrading InstanceState V1: %v", err) - } - meta, err := copystructure.Copy(old.Meta) - if err != nil { - return nil, fmt.Errorf("Error upgrading InstanceState V1: %v", err) - } - - return &InstanceState{ - ID: old.ID, - Attributes: attributes.(map[string]string), - Ephemeral: *ephemeral, - Meta: meta.(map[string]string), - }, nil -} - type ephemeralStateV1 struct { // ConnInfo is used for the providers to export information which is // used to connect to the resource for provisioning. For example, // this could contain SSH or WinRM credentials. ConnInfo map[string]string `json:"-"` } - -func (old *ephemeralStateV1) upgrade() (*EphemeralState, error) { - connInfo, err := copystructure.Copy(old.ConnInfo) - if err != nil { - return nil, fmt.Errorf("Error upgrading EphemeralState V1: %v", err) - } - return &EphemeralState{ - ConnInfo: connInfo.(map[string]string), - }, nil -}