add test for lock error and force-unlock

This adds a general test to verify that a remote state backend returns
the expected error type when it cannot lock a state. It then extracts
the ID reported in the error, and attempts to unlock the state using
that ID, which simulated the force-unlock scenario. This is a separate
test, since not all backends have persistent locks that can be unlocked
later.

We also split out the backend test to be called individually as needed.

backend/testing.go

@@ -5,6 +5,7 @@ import (
 	"sort"
 	"testing"
 
+	uuid "github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/terraform/config"
 	"github.com/hashicorp/terraform/state"
 	"github.com/hashicorp/terraform/terraform"
@@ -43,20 +44,7 @@ func TestBackendConfig(t *testing.T, b Backend, c map[string]interface{}) Backen
 // assumed to already be configured. This will test state functionality.
 // If the backend reports it doesn't support multi-state by returning the
 // error ErrNamedStatesNotSupported, then it will not test that.
-//
-// If you want to test locking, two backends must be given. If b2 is nil,
-// then state locking won't be tested.
-func TestBackend(t *testing.T, b1, b2 Backend) {
-	t.Helper()
-
-	testBackendStates(t, b1)
-
-	if b2 != nil {
-		testBackendStateLock(t, b1, b2)
-	}
-}
-
-func testBackendStates(t *testing.T, b Backend) {
+func TestBackendStates(t *testing.T, b Backend) {
 	t.Helper()
 
 	states, err := b.States()
@@ -236,7 +224,23 @@ func testBackendStates(t *testing.T, b Backend) {
 	}
 }
 
-func testBackendStateLock(t *testing.T, b1, b2 Backend) {
+// TestBackendStateLocks will test the locking functionality of the remote
+// state backend.
+func TestBackendStateLocks(t *testing.T, b1, b2 Backend) {
+	t.Helper()
+	testLocks(t, b1, b2, false)
+}
+
+// TestBackendStateForceUnlock verifies that the lock error is the expected
+// type, and the lock can be unlocked using the ID reported in the error.
+// Remote state backends that support -force-unlock should call this in at
+// least one of the acceptance tests.
+func TestBackendStateForceUnlock(t *testing.T, b1, b2 Backend) {
+	t.Helper()
+	testLocks(t, b1, b2, true)
+}
+
+func testLocks(t *testing.T, b1, b2 Backend, testForceUnlock bool) {
 	t.Helper()
 
 	// Get the default state for each
@@ -286,7 +290,7 @@ func testBackendStateLock(t *testing.T, b1, b2 Backend) {
 	// backend, and as a remote state.
 	_, err = b2.State(DefaultStateName)
 	if err != nil {
-		t.Fatalf("failed to read locked state from another backend instance: %s", err)
+		t.Errorf("failed to read locked state from another backend instance: %s", err)
 	}
 
 	// If the lock ID is blank, assume locking is disabled
@@ -311,11 +315,51 @@ func testBackendStateLock(t *testing.T, b1, b2 Backend) {
 	}
 
 	if lockIDB == lockIDA {
-		t.Fatalf("duplicate lock IDs: %q", lockIDB)
+		t.Errorf("duplicate lock IDs: %q", lockIDB)
 	}
 
 	if err = lockerB.Unlock(lockIDB); err != nil {
 		t.Fatal("error unlocking client B:", err)
 	}
 
+	// test the equivalent of -force-unlock, by using the id from the error
+	// output.
+	if !testForceUnlock {
+		return
+	}
+
+	// get a new ID
+	infoA.ID, err = uuid.GenerateUUID()
+	if err != nil {
+		panic(err)
+	}
+
+	lockIDA, err = lockerA.Lock(infoA)
+	if err != nil {
+		t.Fatal("unable to get re lock A:", err)
+	}
+	unlock := func() {
+		err := lockerA.Unlock(lockIDA)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	_, err = lockerB.Lock(infoB)
+	if err == nil {
+		unlock()
+		t.Fatal("client B obtained lock while held by client A")
+	}
+
+	infoErr, ok := err.(*state.LockError)
+	if !ok {
+		unlock()
+		t.Fatalf("expected type *state.LockError, got : %#v", err)
+	}
+
+	// try to unlock with the second unlocker, using the ID from the error
+	if err := lockerB.Unlock(infoErr.Info.ID); err != nil {
+		unlock()
+		t.Fatalf("could not unlock with the reported ID %q: %s", infoErr.Info.ID, err)
+	}
 }