core: Add terraform_version to state

This adds a field terraform_version to the state that represents the
Terraform version that wrote that state. If Terraform encounters a state
written by a future version, it will error. You must use at least the
version that wrote that state.

Internally we have fields to override this behavior (StateFutureAllowed),
but I chose not to expose them as CLI flags, since the user can just
modify the state directly. This is tricky, but should be tricky to
represent the horrible disaster that can happen by enabling it.

We didn't have to bump the state format version since the absense of the
field means it was written by version "0.0.0" which will always be
older. In effect though this change will always apply to version 2 of
the state since it appears in 0.7 which bumped the version for other
purposes.

command/apply_test.go

@@ -889,15 +889,6 @@ func TestApply_stateNoExist(t *testing.T) {
 func TestApply_sensitiveOutput(t *testing.T) {
 	statePath := testTempFile(t)
 
-	p := testProvider()
-	ui := new(cli.MockUi)
-	c := &ApplyCommand{
-		Meta: Meta{
-			ContextOpts: testCtxConfig(p),
-			Ui:          ui,
-		},
-	}
-
 	args := []string{
 		"-state", statePath,
 		testFixturePath("apply-sensitive-output"),
@@ -916,6 +907,70 @@ func TestApply_sensitiveOutput(t *testing.T) {
 	}
 }
 
+func TestApply_stateFuture(t *testing.T) {
+	originalState := testState()
+	originalState.TFVersion = "99.99.99"
+	statePath := testStateFile(t, originalState)
+
+	p := testProvider()
+	ui := new(cli.MockUi)
+	c := &ApplyCommand{
+		Meta: Meta{
+			ContextOpts: testCtxConfig(p),
+			Ui:          ui,
+		},
+	}
+
+	args := []string{
+		"-state", statePath,
+		testFixturePath("apply"),
+	}
+	if code := c.Run(args); code == 0 {
+		t.Fatal("should fail")
+	}
+
+	f, err := os.Open(statePath)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	newState, err := terraform.ReadState(f)
+	f.Close()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if !newState.Equal(originalState) {
+		t.Fatalf("bad: %#v", newState)
+	}
+	if newState.TFVersion != originalState.TFVersion {
+		t.Fatalf("bad: %#v", newState)
+	}
+}
+
+func TestApply_statePast(t *testing.T) {
+	originalState := testState()
+	originalState.TFVersion = "0.1.0"
+	statePath := testStateFile(t, originalState)
+
+	p := testProvider()
+	ui := new(cli.MockUi)
+	c := &ApplyCommand{
+		Meta: Meta{
+			ContextOpts: testCtxConfig(p),
+			Ui:          ui,
+		},
+	}
+
+	args := []string{
+		"-state", statePath,
+		testFixturePath("apply"),
+	}
+	if code := c.Run(args); code != 0 {
+		t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String())
+	}
+}
+
 func TestApply_vars(t *testing.T) {
 	statePath := testTempFile(t)
 

command/meta.go

@@ -126,7 +126,8 @@ func (m *Meta) Context(copts contextOpts) (*terraform.Context, bool, error) {
 						"variable values, create a new plan file.")
 			}
 
-			return plan.Context(opts), true, nil
+			ctx, err := plan.Context(opts)
+			return ctx, true, err
 		}
 	}
 
@@ -158,8 +159,8 @@ func (m *Meta) Context(copts contextOpts) (*terraform.Context, bool, error) {
 	opts.Module = mod
 	opts.Parallelism = copts.Parallelism
 	opts.State = state.State()
-	ctx := terraform.NewContext(opts)
+	ctx, err := terraform.NewContext(opts)
-	return ctx, false, nil
+	return ctx, false, err
 }
 
 // DataDir returns the directory where local data will be stored.

command/plan_test.go

@@ -345,6 +345,70 @@ func TestPlan_stateDefault(t *testing.T) {
 	}
 }
 
+func TestPlan_stateFuture(t *testing.T) {
+	originalState := testState()
+	originalState.TFVersion = "99.99.99"
+	statePath := testStateFile(t, originalState)
+
+	p := testProvider()
+	ui := new(cli.MockUi)
+	c := &PlanCommand{
+		Meta: Meta{
+			ContextOpts: testCtxConfig(p),
+			Ui:          ui,
+		},
+	}
+
+	args := []string{
+		"-state", statePath,
+		testFixturePath("plan"),
+	}
+	if code := c.Run(args); code == 0 {
+		t.Fatal("should fail")
+	}
+
+	f, err := os.Open(statePath)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	newState, err := terraform.ReadState(f)
+	f.Close()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if !newState.Equal(originalState) {
+		t.Fatalf("bad: %#v", newState)
+	}
+	if newState.TFVersion != originalState.TFVersion {
+		t.Fatalf("bad: %#v", newState)
+	}
+}
+
+func TestPlan_statePast(t *testing.T) {
+	originalState := testState()
+	originalState.TFVersion = "0.1.0"
+	statePath := testStateFile(t, originalState)
+
+	p := testProvider()
+	ui := new(cli.MockUi)
+	c := &PlanCommand{
+		Meta: Meta{
+			ContextOpts: testCtxConfig(p),
+			Ui:          ui,
+		},
+	}
+
+	args := []string{
+		"-state", statePath,
+		testFixturePath("plan"),
+	}
+	if code := c.Run(args); code != 0 {
+		t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String())
+	}
+}
+
 func TestPlan_vars(t *testing.T) {
 	p := testProvider()
 	ui := new(cli.MockUi)

command/refresh_test.go

@@ -221,6 +221,109 @@ func TestRefresh_defaultState(t *testing.T) {
 	}
 }
 
+func TestRefresh_futureState(t *testing.T) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if err := os.Chdir(testFixturePath("refresh")); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer os.Chdir(cwd)
+
+	state := testState()
+	state.TFVersion = "99.99.99"
+	statePath := testStateFile(t, state)
+
+	p := testProvider()
+	ui := new(cli.MockUi)
+	c := &RefreshCommand{
+		Meta: Meta{
+			ContextOpts: testCtxConfig(p),
+			Ui:          ui,
+		},
+	}
+
+	args := []string{
+		"-state", statePath,
+	}
+	if code := c.Run(args); code == 0 {
+		t.Fatal("should fail")
+	}
+
+	if p.RefreshCalled {
+		t.Fatal("refresh should not be called")
+	}
+
+	f, err := os.Open(statePath)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	newState, err := terraform.ReadState(f)
+	f.Close()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	actual := strings.TrimSpace(newState.String())
+	expected := strings.TrimSpace(state.String())
+	if actual != expected {
+		t.Fatalf("bad:\n\n%s", actual)
+	}
+}
+
+func TestRefresh_pastState(t *testing.T) {
+	state := testState()
+	state.TFVersion = "0.1.0"
+	statePath := testStateFile(t, state)
+
+	p := testProvider()
+	ui := new(cli.MockUi)
+	c := &RefreshCommand{
+		Meta: Meta{
+			ContextOpts: testCtxConfig(p),
+			Ui:          ui,
+		},
+	}
+
+	p.RefreshFn = nil
+	p.RefreshReturn = &terraform.InstanceState{ID: "yes"}
+
+	args := []string{
+		"-state", statePath,
+		testFixturePath("refresh"),
+	}
+	if code := c.Run(args); code != 0 {
+		t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String())
+	}
+
+	if !p.RefreshCalled {
+		t.Fatal("refresh should be called")
+	}
+
+	f, err := os.Open(statePath)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	newState, err := terraform.ReadState(f)
+	f.Close()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	actual := strings.TrimSpace(newState.String())
+	expected := strings.TrimSpace(testRefreshStr)
+	if actual != expected {
+		t.Fatalf("bad:\n\n%s", actual)
+	}
+
+	if newState.TFVersion != terraform.Version {
+		t.Fatalf("bad:\n\n%s", newState.TFVersion)
+	}
+}
+
 func TestRefresh_outPath(t *testing.T) {
 	state := testState()
 	statePath := testStateFile(t, state)

helper/resource/testing.go

@@ -284,7 +284,10 @@ func testIDOnlyRefresh(c TestCase, opts terraform.ContextOpts, step TestStep, r
 	// Initialize the context
 	opts.Module = mod
 	opts.State = state
-	ctx := terraform.NewContext(&opts)
+	ctx, err := terraform.NewContext(&opts)
+	if err != nil {
+		return err
+	}
 	if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 {
 		if len(es) > 0 {
 			estrs := make([]string, len(es))
@@ -362,7 +365,10 @@ func testStep(
 	opts.Module = mod
 	opts.State = state
 	opts.Destroy = step.Destroy
-	ctx := terraform.NewContext(&opts)
+	ctx, err := terraform.NewContext(&opts)
+	if err != nil {
+		return state, fmt.Errorf("Error initializing context: %s", err)
+	}
 	if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 {
 		if len(es) > 0 {
 			estrs := make([]string, len(es))

terraform/context.go

@@ -35,16 +35,17 @@ const (
 // ContextOpts are the user-configurable options to create a context with
 // NewContext.
 type ContextOpts struct {
-	Destroy      bool
+	Destroy            bool
-	Diff         *Diff
+	Diff               *Diff
-	Hooks        []Hook
+	Hooks              []Hook
-	Module       *module.Tree
+	Module             *module.Tree
-	Parallelism  int
+	Parallelism        int
-	State        *State
+	State              *State
-	Providers    map[string]ResourceProviderFactory
+	StateFutureAllowed bool
-	Provisioners map[string]ResourceProvisionerFactory
+	Providers          map[string]ResourceProviderFactory
-	Targets      []string
+	Provisioners       map[string]ResourceProvisionerFactory
-	Variables    map[string]string
+	Targets            []string
+	Variables          map[string]string
 
 	UIInput UIInput
 }
@@ -78,7 +79,7 @@ type Context struct {
 // Once a Context is creator, the pointer values within ContextOpts
 // should not be mutated in any way, since the pointers are copied, not
 // the values themselves.
-func NewContext(opts *ContextOpts) *Context {
+func NewContext(opts *ContextOpts) (*Context, error) {
 	// Copy all the hooks and add our stop hook. We don't append directly
 	// to the Config so that we're not modifying that in-place.
 	sh := new(stopHook)
@@ -92,6 +93,22 @@ func NewContext(opts *ContextOpts) *Context {
 		state.init()
 	}
 
+	// If our state is from the future, then error. Callers can avoid
+	// this error by explicitly setting `StateFutureAllowed`.
+	if !opts.StateFutureAllowed && state.FromFutureTerraform() {
+		return nil, fmt.Errorf(
+			"Terraform doesn't allow running any operations against a state\n"+
+				"that was written by a future Terraform version. The state is\n"+
+				"reporting it is written by Terraform '%s'.\n\n"+
+				"Please run at least that version of Terraform to continue.",
+			state.TFVersion)
+	}
+
+	// Explicitly reset our state version to our current version so that
+	// any operations we do will write out that our latest version
+	// has run.
+	state.TFVersion = Version
+
 	// Determine parallelism, default to 10. We do this both to limit
 	// CPU pressure but also to have an extra guard against rate throttling
 	// from providers.
@@ -135,7 +152,7 @@ func NewContext(opts *ContextOpts) *Context {
 		parallelSem:         NewSemaphore(par),
 		providerInputConfig: make(map[string]map[string]interface{}),
 		sh:                  sh,
-	}
+	}, nil
 }
 
 type ContextGraphOpts struct {

terraform/context_apply_test.go

@@ -4115,11 +4115,14 @@ func TestContext2Apply_issue5254(t *testing.T) {
 		t.Fatalf("err: %s", err)
 	}
 
-	ctx = planFromFile.Context(&ContextOpts{
+	ctx, err = planFromFile.Context(&ContextOpts{
 		Providers: map[string]ResourceProviderFactory{
 			"template": testProviderFuncFixed(p),
 		},
 	})
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
 
 	state, err = ctx.Apply()
 	if err != nil {
@@ -4189,12 +4192,15 @@ func TestContext2Apply_targetedWithTaintedInState(t *testing.T) {
 		t.Fatalf("err: %s", err)
 	}
 
-	ctx = planFromFile.Context(&ContextOpts{
+	ctx, err = planFromFile.Context(&ContextOpts{
 		Module: testModule(t, "apply-tainted-targets"),
 		Providers: map[string]ResourceProviderFactory{
 			"aws": testProviderFuncFixed(p),
 		},
 	})
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
 
 	state, err := ctx.Apply()
 	if err != nil {

terraform/context_test.go

@@ -7,8 +7,71 @@ import (
 	"time"
 )
 
+func TestNewContextState(t *testing.T) {
+	cases := map[string]struct {
+		Input *ContextOpts
+		Err   bool
+	}{
+		"empty TFVersion": {
+			&ContextOpts{
+				State: &State{},
+			},
+			false,
+		},
+
+		"past TFVersion": {
+			&ContextOpts{
+				State: &State{TFVersion: "0.1.2"},
+			},
+			false,
+		},
+
+		"equal TFVersion": {
+			&ContextOpts{
+				State: &State{TFVersion: Version},
+			},
+			false,
+		},
+
+		"future TFVersion": {
+			&ContextOpts{
+				State: &State{TFVersion: "99.99.99"},
+			},
+			true,
+		},
+
+		"future TFVersion, allowed": {
+			&ContextOpts{
+				State:              &State{TFVersion: "99.99.99"},
+				StateFutureAllowed: true,
+			},
+			false,
+		},
+	}
+
+	for k, tc := range cases {
+		ctx, err := NewContext(tc.Input)
+		if (err != nil) != tc.Err {
+			t.Fatalf("%s: err: %s", k, err)
+		}
+		if err != nil {
+			continue
+		}
+
+		// Version should always be set to our current
+		if ctx.state.TFVersion != Version {
+			t.Fatalf("%s: state not set to current version", k)
+		}
+	}
+}
+
 func testContext2(t *testing.T, opts *ContextOpts) *Context {
-	return NewContext(opts)
+	ctx, err := NewContext(opts)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	return ctx
 }
 
 func testApplyFn(

terraform/plan.go

@@ -34,7 +34,7 @@ type Plan struct {
 //
 // The following fields in opts are overridden by the plan: Config,
 // Diff, State, Variables.
-func (p *Plan) Context(opts *ContextOpts) *Context {
+func (p *Plan) Context(opts *ContextOpts) (*Context, error) {
 	opts.Diff = p.Diff
 	opts.Module = p.Module
 	opts.State = p.State

terraform/state.go

@@ -12,6 +12,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/hashicorp/go-version"
 	"github.com/hashicorp/terraform/config"
 )
 
@@ -30,6 +31,9 @@ type State struct {
 	// Version is the protocol version. Currently only "1".
 	Version int `json:"version"`
 
+	// TFVersion is the version of Terraform that wrote this state.
+	TFVersion string `json:"terraform_version,omitempty"`
+
 	// Serial is incremented on any operation that modifies
 	// the State file. It is used to detect potentially conflicting
 	// updates.
@@ -362,9 +366,10 @@ func (s *State) DeepCopy() *State {
 		return nil
 	}
 	n := &State{
-		Version: s.Version,
+		Version:   s.Version,
-		Serial:  s.Serial,
+		TFVersion: s.TFVersion,
-		Modules: make([]*ModuleState, 0, len(s.Modules)),
+		Serial:    s.Serial,
+		Modules:   make([]*ModuleState, 0, len(s.Modules)),
 	}
 	for _, mod := range s.Modules {
 		n.Modules = append(n.Modules, mod.deepcopy())
@@ -387,7 +392,7 @@ func (s *State) IncrementSerialMaybe(other *State) {
 	if s.Serial > other.Serial {
 		return
 	}
-	if !s.Equal(other) {
+	if other.TFVersion != s.TFVersion || !s.Equal(other) {
 		if other.Serial > s.Serial {
 			s.Serial = other.Serial
 		}
@@ -396,6 +401,18 @@ func (s *State) IncrementSerialMaybe(other *State) {
 	}
 }
 
+// FromFutureTerraform checks if this state was written by a Terraform
+// version from the future.
+func (s *State) FromFutureTerraform() bool {
+	// No TF version means it is certainly from the past
+	if s.TFVersion == "" {
+		return false
+	}
+
+	v := version.Must(version.NewVersion(s.TFVersion))
+	return SemVersion.LessThan(v)
+}
+
 func (s *State) init() {
 	if s.Version == 0 {
 		s.Version = StateVersion
@@ -1335,6 +1352,19 @@ func ReadState(src io.Reader) (*State, error) {
 			state.Version)
 	}
 
+	// Make sure the version is semantic
+	if state.TFVersion != "" {
+		if _, err := version.NewVersion(state.TFVersion); err != nil {
+			return nil, fmt.Errorf(
+				"State contains invalid version: %s\n\n"+
+					"Terraform validates the version format prior to writing it. This\n"+
+					"means that this is invalid of the state becoming corrupted through\n"+
+					"some external means. Please manually modify the Terraform version\n"+
+					"field to be a proper semantic version.",
+				state.TFVersion)
+		}
+	}
+
 	// Sort it
 	state.sort()
 
@@ -1349,6 +1379,19 @@ func WriteState(d *State, dst io.Writer) error {
 	// Ensure the version is set
 	d.Version = StateVersion
 
+	// If the TFVersion is set, verify it. We used to just set the version
+	// here, but this isn't safe since it changes the MD5 sum on some remote
+	// state storage backends such as Atlas. We now leave it be if needed.
+	if d.TFVersion != "" {
+		if _, err := version.NewVersion(d.TFVersion); err != nil {
+			return fmt.Errorf(
+				"Error writing state, invalid version: %s\n\n"+
+					"The Terraform version when writing the state must be a semantic\n"+
+					"version.",
+				d.TFVersion)
+		}
+	}
+
 	// Encode the data in a human-friendly way
 	data, err := json.MarshalIndent(d, "", "    ")
 	if err != nil {

terraform/state_test.go

@@ -175,6 +175,35 @@ func TestStateModuleOrphans_deepNestedNilConfig(t *testing.T) {
 	}
 }
 
+func TestStateDeepCopy(t *testing.T) {
+	cases := []struct {
+		One, Two *State
+		F        func(*State) interface{}
+	}{
+		// Version
+		{
+			&State{Version: 5},
+			&State{Version: 5},
+			func(s *State) interface{} { return s.Version },
+		},
+
+		// TFVersion
+		{
+			&State{TFVersion: "5"},
+			&State{TFVersion: "5"},
+			func(s *State) interface{} { return s.TFVersion },
+		},
+	}
+
+	for i, tc := range cases {
+		actual := tc.F(tc.One.DeepCopy())
+		expected := tc.F(tc.Two)
+		if !reflect.DeepEqual(actual, expected) {
+			t.Fatalf("Bad: %d\n\n%s\n\n%s", i, actual, expected)
+		}
+	}
+}
+
 func TestStateEqual(t *testing.T) {
 	cases := []struct {
 		Result   bool
@@ -348,6 +377,11 @@ func TestStateIncrementSerialMaybe(t *testing.T) {
 			},
 			5,
 		},
+		"S2 has a different TFVersion": {
+			&State{TFVersion: "0.1"},
+			&State{TFVersion: "0.2"},
+			1,
+		},
 	}
 
 	for name, tc := range cases {
@@ -987,6 +1021,34 @@ func TestStateEmpty(t *testing.T) {
 	}
 }
 
+func TestStateFromFutureTerraform(t *testing.T) {
+	cases := []struct {
+		In     string
+		Result bool
+	}{
+		{
+			"",
+			false,
+		},
+		{
+			"0.1",
+			false,
+		},
+		{
+			"999.15.1",
+			true,
+		},
+	}
+
+	for _, tc := range cases {
+		state := &State{TFVersion: tc.In}
+		actual := state.FromFutureTerraform()
+		if actual != tc.Result {
+			t.Fatalf("%s: bad: %v", tc.In, actual)
+		}
+	}
+}
+
 func TestStateIsRemote(t *testing.T) {
 	cases := []struct {
 		In     *State
@@ -1206,6 +1268,97 @@ func TestReadStateNewVersion(t *testing.T) {
 	}
 }
 
+func TestReadStateTFVersion(t *testing.T) {
+	type tfVersion struct {
+		TFVersion string `json:"terraform_version"`
+	}
+
+	cases := []struct {
+		Written string
+		Read    string
+		Err     bool
+	}{
+		{
+			"0.0.0",
+			"0.0.0",
+			false,
+		},
+		{
+			"",
+			"",
+			false,
+		},
+		{
+			"bad",
+			"",
+			true,
+		},
+	}
+
+	for _, tc := range cases {
+		buf, err := json.Marshal(&tfVersion{tc.Written})
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		s, err := ReadState(bytes.NewReader(buf))
+		if (err != nil) != tc.Err {
+			t.Fatalf("%s: err: %s", tc.Written, err)
+		}
+		if err != nil {
+			continue
+		}
+
+		if s.TFVersion != tc.Read {
+			t.Fatalf("%s: bad: %s", tc.Written, s.TFVersion)
+		}
+	}
+}
+
+func TestWriteStateTFVersion(t *testing.T) {
+	cases := []struct {
+		Write string
+		Read  string
+		Err   bool
+	}{
+		{
+			"0.0.0",
+			"0.0.0",
+			false,
+		},
+		{
+			"",
+			"",
+			false,
+		},
+		{
+			"bad",
+			"",
+			true,
+		},
+	}
+
+	for _, tc := range cases {
+		var buf bytes.Buffer
+		err := WriteState(&State{TFVersion: tc.Write}, &buf)
+		if (err != nil) != tc.Err {
+			t.Fatalf("%s: err: %s", tc.Write, err)
+		}
+		if err != nil {
+			continue
+		}
+
+		s, err := ReadState(&buf)
+		if err != nil {
+			t.Fatalf("%s: err: %s", tc.Write, err)
+		}
+
+		if s.TFVersion != tc.Read {
+			t.Fatalf("%s: bad: %s", tc.Write, s.TFVersion)
+		}
+	}
+}
+
 func TestUpgradeV1State(t *testing.T) {
 	old := &StateV1{
 		Outputs: map[string]string{

terraform/version.go

@@ -1,5 +1,9 @@
 package terraform
 
+import (
+	"github.com/hashicorp/go-version"
+)
+
 // The main version number that is being run at the moment.
 const Version = "0.7.0"
 
@@ -7,3 +11,8 @@ const Version = "0.7.0"
 // then it means that it is a final release. Otherwise, this is a pre-release
 // such as "dev" (in development), "beta", "rc1", etc.
 const VersionPrerelease = "dev"
+
+// SemVersion is an instance of version.Version. This has the secondary
+// benefit of verifying during tests and init time that our version is a
+// proper semantic version, which should always be the case.
+var SemVersion = version.Must(version.NewVersion(Version))