Merge pull request #15683 from hashicorp/jbardin/remote-state-lineage

Remove strict lineage check in remote.State

backend/remote-state/inmem/backend.go

@@ -2,40 +2,207 @@ package inmem
 
 import (
 	"context"
+	"errors"
+	"fmt"
+	"sort"
+	"sync"
+	"time"
 
 	"github.com/hashicorp/terraform/backend"
-	"github.com/hashicorp/terraform/backend/remote-state"
 	"github.com/hashicorp/terraform/helper/schema"
 	"github.com/hashicorp/terraform/state"
 	"github.com/hashicorp/terraform/state/remote"
+	"github.com/hashicorp/terraform/terraform"
 )
 
-// New creates a new backend for Inmem remote state.
-func New() backend.Backend {
-	return &remotestate.Backend{
-		ConfigureFunc: configure,
+// we keep the states and locks in package-level variables, so that they can be
+// accessed from multiple instances of the backend. This better emulates
+// backend instances accessing a single remote data store.
+var (
+	states stateMap
+	locks  lockMap
+)
 
-		// Set the schema
-		Backend: &schema.Backend{
-			Schema: map[string]*schema.Schema{
-				"lock_id": &schema.Schema{
-					Type:        schema.TypeString,
-					Optional:    true,
-					Description: "initializes the state in a locked configuration",
-				},
-			},
-		},
+func init() {
+	Reset()
+}
+
+// Reset clears out all existing state and lock data.
+// This is used to initialize the package during init, as well as between
+// tests.
+func Reset() {
+	states = stateMap{
+		m: map[string]*remote.State{},
+	}
+
+	locks = lockMap{
+		m: map[string]*state.LockInfo{},
 	}
 }
 
-func configure(ctx context.Context) (remote.Client, error) {
+// New creates a new backend for Inmem remote state.
+func New() backend.Backend {
+	// Set the schema
+	s := &schema.Backend{
+		Schema: map[string]*schema.Schema{
+			"lock_id": &schema.Schema{
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "initializes the state in a locked configuration",
+			},
+		},
+	}
+	backend := &Backend{Backend: s}
+	backend.Backend.ConfigureFunc = backend.configure
+	return backend
+}
+
+type Backend struct {
+	*schema.Backend
+}
+
+func (b *Backend) configure(ctx context.Context) error {
+	states.Lock()
+	defer states.Unlock()
+
+	defaultClient := &RemoteClient{
+		Name: backend.DefaultStateName,
+	}
+
+	states.m[backend.DefaultStateName] = &remote.State{
+		Client: defaultClient,
+	}
+
+	// set the default client lock info per the test config
 	data := schema.FromContextBackendConfig(ctx)
 	if v, ok := data.GetOk("lock_id"); ok && v.(string) != "" {
 		info := state.NewLockInfo()
 		info.ID = v.(string)
 		info.Operation = "test"
 		info.Info = "test config"
-		return &RemoteClient{LockInfo: info}, nil
+
+		locks.lock(backend.DefaultStateName, info)
 	}
-	return &RemoteClient{}, nil
+
+	return nil
+}
+
+func (b *Backend) States() ([]string, error) {
+	states.Lock()
+	defer states.Unlock()
+
+	var workspaces []string
+
+	for s := range states.m {
+		workspaces = append(workspaces, s)
+	}
+
+	sort.Strings(workspaces)
+	return workspaces, nil
+}
+
+func (b *Backend) DeleteState(name string) error {
+	states.Lock()
+	defer states.Unlock()
+
+	if name == backend.DefaultStateName || name == "" {
+		return fmt.Errorf("can't delete default state")
+	}
+
+	delete(states.m, name)
+	return nil
+}
+
+func (b *Backend) State(name string) (state.State, error) {
+	states.Lock()
+	defer states.Unlock()
+
+	s := states.m[name]
+	if s == nil {
+		s = &remote.State{
+			Client: &RemoteClient{
+				Name: name,
+			},
+		}
+		states.m[name] = s
+
+		// to most closely replicate other implementations, we are going to
+		// take a lock and create a new state if it doesn't exist.
+		lockInfo := state.NewLockInfo()
+		lockInfo.Operation = "init"
+		lockID, err := s.Lock(lockInfo)
+		if err != nil {
+			return nil, fmt.Errorf("failed to lock inmem state: %s", err)
+		}
+		defer s.Unlock(lockID)
+
+		// If we have no state, we have to create an empty state
+		if v := s.State(); v == nil {
+			if err := s.WriteState(terraform.NewState()); err != nil {
+				return nil, err
+			}
+			if err := s.PersistState(); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return s, nil
+}
+
+type stateMap struct {
+	sync.Mutex
+	m map[string]*remote.State
+}
+
+// Global level locks for inmem backends.
+type lockMap struct {
+	sync.Mutex
+	m map[string]*state.LockInfo
+}
+
+func (l *lockMap) lock(name string, info *state.LockInfo) (string, error) {
+	l.Lock()
+	defer l.Unlock()
+
+	lockInfo := l.m[name]
+	if lockInfo != nil {
+		lockErr := &state.LockError{
+			Info: lockInfo,
+		}
+
+		lockErr.Err = errors.New("state locked")
+		// make a copy of the lock info to avoid any testing shenanigans
+		*lockErr.Info = *lockInfo
+		return "", lockErr
+	}
+
+	info.Created = time.Now().UTC()
+	l.m[name] = info
+
+	return info.ID, nil
+}
+
+func (l *lockMap) unlock(name, id string) error {
+	l.Lock()
+	defer l.Unlock()
+
+	lockInfo := l.m[name]
+
+	if lockInfo == nil {
+		return errors.New("state not locked")
+	}
+
+	lockErr := &state.LockError{
+		Info: &state.LockInfo{},
+	}
+
+	if id != lockInfo.ID {
+		lockErr.Err = errors.New("invalid lock id")
+		*lockErr.Info = *lockInfo
+		return lockErr
+	}
+
+	delete(l.m, name)
+	return nil
 }

backend/remote-state/inmem/backend_test.go

@@ -0,0 +1,90 @@
+package inmem
+
+import (
+	"testing"
+
+	"github.com/hashicorp/terraform/backend"
+	"github.com/hashicorp/terraform/state/remote"
+	"github.com/hashicorp/terraform/terraform"
+)
+
+func TestBackend_impl(t *testing.T) {
+	var _ backend.Backend = new(Backend)
+}
+
+func TestBackendConfig(t *testing.T) {
+	defer Reset()
+	testID := "test_lock_id"
+
+	config := map[string]interface{}{
+		"lock_id": testID,
+	}
+
+	b := backend.TestBackendConfig(t, New(), config).(*Backend)
+
+	s, err := b.State(backend.DefaultStateName)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c := s.(*remote.State).Client.(*RemoteClient)
+	if c.Name != backend.DefaultStateName {
+		t.Fatal("client name is not configured")
+	}
+
+	if err := locks.unlock(backend.DefaultStateName, testID); err != nil {
+		t.Fatalf("default state should have been locked: %s", err)
+	}
+}
+
+func TestBackend(t *testing.T) {
+	defer Reset()
+	b := backend.TestBackendConfig(t, New(), nil).(*Backend)
+	backend.TestBackend(t, b, nil)
+}
+
+func TestBackendLocked(t *testing.T) {
+	defer Reset()
+	b1 := backend.TestBackendConfig(t, New(), nil).(*Backend)
+	b2 := backend.TestBackendConfig(t, New(), nil).(*Backend)
+
+	backend.TestBackend(t, b1, b2)
+}
+
+// use the this backen to test the remote.State implementation
+func TestRemoteState(t *testing.T) {
+	defer Reset()
+	b := backend.TestBackendConfig(t, New(), nil)
+
+	workspace := "workspace"
+
+	// create a new workspace in this backend
+	s, err := b.State(workspace)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// force overwriting the remote state
+	newState := terraform.NewState()
+
+	if err := s.WriteState(newState); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := s.PersistState(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := s.RefreshState(); err != nil {
+		t.Fatal(err)
+	}
+
+	savedState := s.State()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if savedState.Lineage != newState.Lineage {
+		t.Fatal("saved state has incorrect lineage")
+	}
+}

backend/remote-state/inmem/client.go

@@ -2,8 +2,6 @@ package inmem
 
 import (
 	"crypto/md5"
-	"errors"
-	"time"
 
 	"github.com/hashicorp/terraform/state"
 	"github.com/hashicorp/terraform/state/remote"
@@ -13,8 +11,7 @@ import (
 type RemoteClient struct {
 	Data []byte
 	MD5  []byte
-
-	LockInfo *state.LockInfo
+	Name string
 }
 
 func (c *RemoteClient) Get() (*remote.Payload, error) {
@@ -43,37 +40,8 @@ func (c *RemoteClient) Delete() error {
 }
 
 func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) {
-	lockErr := &state.LockError{
-		Info: &state.LockInfo{},
-	}
-
-	if c.LockInfo != nil {
-		lockErr.Err = errors.New("state locked")
-		// make a copy of the lock info to avoid any testing shenanigans
-		*lockErr.Info = *c.LockInfo
-		return "", lockErr
-	}
-
-	info.Created = time.Now().UTC()
-	c.LockInfo = info
-
-	return c.LockInfo.ID, nil
+	return locks.lock(c.Name, info)
 }
-
 func (c *RemoteClient) Unlock(id string) error {
-	if c.LockInfo == nil {
-		return errors.New("state not locked")
-	}
-
-	lockErr := &state.LockError{
-		Info: &state.LockInfo{},
-	}
-	if id != c.LockInfo.ID {
-		lockErr.Err = errors.New("invalid lock id")
-		*lockErr.Info = *c.LockInfo
-		return lockErr
-	}
-
-	c.LockInfo = nil
-	return nil
+	return locks.unlock(c.Name, id)
 }

backend/remote-state/inmem/client_test.go

@@ -4,7 +4,6 @@ import (
 	"testing"
 
 	"github.com/hashicorp/terraform/backend"
-	remotestate "github.com/hashicorp/terraform/backend/remote-state"
 	"github.com/hashicorp/terraform/state/remote"
 )
 
@@ -14,11 +13,19 @@ func TestRemoteClient_impl(t *testing.T) {
 }
 
 func TestRemoteClient(t *testing.T) {
+	defer Reset()
 	b := backend.TestBackendConfig(t, New(), nil)
-	remotestate.TestClient(t, b)
+
+	s, err := b.State(backend.DefaultStateName)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	remote.TestClient(t, s.(*remote.State).Client)
 }
 
 func TestInmemLocks(t *testing.T) {
+	defer Reset()
 	s, err := backend.TestBackendConfig(t, New(), nil).State(backend.DefaultStateName)
 	if err != nil {
 		t.Fatal(err)

command/state_push_test.go

@@ -5,6 +5,8 @@ import (
 	"os"
 	"testing"
 
+	"github.com/hashicorp/terraform/backend"
+	"github.com/hashicorp/terraform/backend/remote-state/inmem"
 	"github.com/hashicorp/terraform/helper/copy"
 	"github.com/hashicorp/terraform/terraform"
 	"github.com/mitchellh/cli"
@@ -190,3 +192,56 @@ func TestStatePush_serialOlder(t *testing.T) {
 		t.Fatalf("bad: %#v", actual)
 	}
 }
+
+func TestStatePush_forceRemoteState(t *testing.T) {
+	td := tempDir(t)
+	copy.CopyDir(testFixturePath("inmem-backend"), td)
+	defer os.RemoveAll(td)
+	defer testChdir(t, td)()
+	defer inmem.Reset()
+
+	s := terraform.NewState()
+	statePath := testStateFile(t, s)
+
+	// init the backend
+	ui := new(cli.MockUi)
+	initCmd := &InitCommand{
+		Meta: Meta{Ui: ui},
+	}
+	if code := initCmd.Run([]string{}); code != 0 {
+		t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
+	}
+
+	// create a new workspace
+	ui = new(cli.MockUi)
+	newCmd := &WorkspaceNewCommand{
+		Meta: Meta{Ui: ui},
+	}
+	if code := newCmd.Run([]string{"test"}); code != 0 {
+		t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter)
+	}
+
+	// put a dummy state in place, so we have something to force
+	b := backend.TestBackendConfig(t, inmem.New(), nil)
+	sMgr, err := b.State("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := sMgr.WriteState(terraform.NewState()); err != nil {
+		t.Fatal(err)
+	}
+	if err := sMgr.PersistState(); err != nil {
+		t.Fatal(err)
+	}
+
+	// push our local state to that new workspace
+	ui = new(cli.MockUi)
+	c := &StatePushCommand{
+		Meta: Meta{Ui: ui},
+	}
+
+	args := []string{"-force", statePath}
+	if code := c.Run(args); code != 0 {
+		t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String())
+	}
+}

command/test-fixtures/inmem-backend/main.tf

@@ -0,0 +1,3 @@
+terraform {
+  backend "inmem" {}
+}

command/unlock_test.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/hashicorp/terraform/backend/remote-state/inmem"
 	"github.com/hashicorp/terraform/helper/copy"
 	"github.com/hashicorp/terraform/terraform"
 	"github.com/mitchellh/cli"
@@ -57,6 +58,7 @@ func TestUnlock_inmemBackend(t *testing.T) {
 	copy.CopyDir(testFixturePath("backend-inmem-locked"), td)
 	defer os.RemoveAll(td)
 	defer testChdir(t, td)()
+	defer inmem.Reset()
 
 	// init backend
 	ui := new(cli.MockUi)

command/workspace_command_test.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/hashicorp/terraform/backend"
 	"github.com/hashicorp/terraform/backend/local"
+	"github.com/hashicorp/terraform/backend/remote-state/inmem"
+	"github.com/hashicorp/terraform/helper/copy"
 	"github.com/hashicorp/terraform/state"
 	"github.com/hashicorp/terraform/terraform"
 	"github.com/mitchellh/cli"
@@ -211,9 +213,19 @@ func TestWorkspace_createInvalid(t *testing.T) {
 
 func TestWorkspace_createWithState(t *testing.T) {
 	td := tempDir(t)
-	os.MkdirAll(td, 0755)
+	copy.CopyDir(testFixturePath("inmem-backend"), td)
 	defer os.RemoveAll(td)
 	defer testChdir(t, td)()
+	defer inmem.Reset()
+
+	// init the backend
+	ui := new(cli.MockUi)
+	initCmd := &InitCommand{
+		Meta: Meta{Ui: ui},
+	}
+	if code := initCmd.Run([]string{}); code != 0 {
+		t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
+	}
 
 	// create a non-empty state
 	originalState := &terraform.State{
@@ -237,8 +249,10 @@ func TestWorkspace_createWithState(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	args := []string{"-state", "test.tfstate", "test"}
-	ui := new(cli.MockUi)
+	workspace := "test_workspace"
+
+	args := []string{"-state", "test.tfstate", workspace}
+	ui = new(cli.MockUi)
 	newCmd := &WorkspaceNewCommand{
 		Meta: Meta{Ui: ui},
 	}
@@ -253,7 +267,14 @@ func TestWorkspace_createWithState(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	newState := envState.State()
+	b := backend.TestBackendConfig(t, inmem.New(), nil)
+	sMgr, err := b.State(workspace)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newState := sMgr.State()
+
 	originalState.Version = newState.Version // the round-trip through the state manager implicitly populates version
 	if !originalState.Equal(newState) {
 		t.Fatalf("states not equal\norig: %s\nnew: %s", originalState, newState)

state/remote/state.go

@@ -2,7 +2,7 @@ package remote
 
 import (
 	"bytes"
-	"fmt"
+	"log"
 	"sync"
 
 	"github.com/hashicorp/terraform/state"
@@ -35,7 +35,10 @@ func (s *State) WriteState(state *terraform.State) error {
 	defer s.mu.Unlock()
 
 	if s.readState != nil && !state.SameLineage(s.readState) {
-		return fmt.Errorf("incompatible state lineage; given %s but want %s", state.Lineage, s.readState.Lineage)
+		// This can't error here, because we need to be able to overwrite the
+		// state in some cases, like `state push -force` or `workspace new
+		// -state=`
+		log.Printf("[WARN] incompatible state lineage; given %s but want %s", state.Lineage, s.readState.Lineage)
 	}
 
 	// We create a deep copy of the state here, because the caller also has