Add basic local state locking

Add the LockUnlock methods to LocalState and BackupState.

The implementation for LocalState will be platform specific. We will use
OS-native locking on the state files, speficially locking whichever
state file we intend to write to.

state/backup.go

@@ -43,6 +43,21 @@ func (s *BackupState) PersistState() error {
 	return s.Real.PersistState()
 }
 
+// all states get wrapped by BackupState, so it has to be a Locker
+func (s *BackupState) Lock(reason string) error {
+	if s, ok := s.Real.(Locker); ok {
+		return s.Lock(reason)
+	}
+	return nil
+}
+
+func (s *BackupState) Unlock() error {
+	if s, ok := s.Real.(Locker); ok {
+		return s.Unlock()
+	}
+	return nil
+}
+
 func (s *BackupState) backup() error {
 	state := s.Real.State()
 	if state == nil {

state/local.go

@@ -1,12 +1,34 @@
 package state
 
 import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/hashicorp/terraform/terraform"
 )
 
+// lock metadata structure for local locks
+type lockInfo struct {
+	// Path to the state file
+	Path string
+	// The time the lock was taken
+	Time time.Time
+	// The time this lock expires
+	Expires time.Time
+	// The lock reason passed to State.Lock
+	Reason string
+}
+
+// return the lock info formatted in an error
+func (l *lockInfo) Err() error {
+	return fmt.Errorf("state file %q locked. created:%s, expires:%s, reason:%s",
+		l.Path, l.Time, l.Expires, l.Reason)
+}
+
 // LocalState manages a state storage that is local to the filesystem.
 type LocalState struct {
 	// Path is the path to read the state from. PathOut is the path to
@@ -15,6 +37,10 @@ type LocalState struct {
 	Path    string
 	PathOut string
 
+	// the file handles corresponding to Path and PathOut
+	stateFile    *os.File
+	stateFileOut *os.File
+
 	state     *terraform.State
 	readState *terraform.State
 	written   bool
@@ -31,45 +57,105 @@ func (s *LocalState) State() *terraform.State {
 	return s.state.DeepCopy()
 }
 
-// WriteState for LocalState always persists the state as well.
-//
-// StateWriter impl.
-func (s *LocalState) WriteState(state *terraform.State) error {
-	s.state = state
-
-	path := s.PathOut
-	if path == "" {
-		path = s.Path
-	}
-
-	// If we don't have any state, we actually delete the file if it exists
-	if state == nil {
-		err := os.Remove(path)
-		if err != nil && os.IsNotExist(err) {
-			return nil
+// Lock implements a local filesystem state.Locker.
+func (s *LocalState) Lock(reason string) error {
+	if s.stateFileOut == nil {
+		if err := s.createStateFiles(); err != nil {
+			return err
 		}
-
-		return err
 	}
 
-	// Create all the directories
-	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
-		return err
+	if err := s.lock(); err != nil {
+		if info, err := s.lockInfo(); err != nil {
+			return info.Err()
+		}
+		return fmt.Errorf("state file %q locked: %s", s.Path, err)
 	}
 
-	f, err := os.Create(path)
+	return s.writeLockInfo(reason)
+}
+
+func (s *LocalState) Unlock() error {
+	os.Remove(s.lockInfoPath())
+	return s.unlock()
+}
+
+// Open the state file, creating the directories and file as needed.
+func (s *LocalState) createStateFiles() error {
+	f, err := createFileAndDirs(s.Path)
 	if err != nil {
 		return err
 	}
-	defer f.Close()
+
+	s.stateFile = f
+
+	if s.PathOut == "" {
+		s.PathOut = s.Path
+	}
+
+	if s.PathOut == s.Path {
+		s.stateFileOut = s.stateFile
+		return nil
+	}
+
+	f, err = createFileAndDirs(s.PathOut)
+	if err != nil {
+		return err
+	}
+	s.stateFileOut = f
+	return nil
+}
+
+func createFileAndDirs(path string) (*os.File, error) {
+	// Create all the directories
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return nil, err
+	}
+
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666)
+	if err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
+// WriteState for LocalState always persists the state as well.
+// TODO: this should use a more robust method of writing state, by first
+// writing to a temp file on the same filesystem, and renaming the file over
+// the original.
+//
+// StateWriter impl.
+func (s *LocalState) WriteState(state *terraform.State) error {
+	if state == nil {
+		// if we have no state, don't write anything.
+		return nil
+	}
+
+	if s.stateFileOut == nil {
+		if err := s.createStateFiles(); err != nil {
+			return nil
+		}
+	}
+
+	s.state = state
+
+	if _, err := s.stateFileOut.Seek(0, os.SEEK_SET); err != nil {
+		return err
+	}
+
+	if err := s.stateFileOut.Truncate(0); err != nil {
+		return err
+	}
 
 	s.state.IncrementSerialMaybe(s.readState)
 	s.readState = s.state
 
-	if err := terraform.WriteState(s.state, f); err != nil {
+	if err := terraform.WriteState(s.state, s.stateFileOut); err != nil {
 		return err
 	}
 
+	s.stateFileOut.Sync()
 	s.written = true
 	return nil
 }
@@ -83,33 +169,77 @@ func (s *LocalState) PersistState() error {
 
 // StateRefresher impl.
 func (s *LocalState) RefreshState() error {
-	// If we've never loaded before, read from Path, otherwise we
-	// read from PathOut.
-	path := s.Path
-	if s.written && s.PathOut != "" {
-		path = s.PathOut
-	}
-
-	f, err := os.Open(path)
-	if err != nil {
-		// It is okay if the file doesn't exist, we treat that as a nil state
-		if !os.IsNotExist(err) {
+	if s.stateFile == nil {
+		if err := s.createStateFiles(); err != nil {
 			return err
 		}
-
-		f = nil
 	}
 
-	var state *terraform.State
-	if f != nil {
-		defer f.Close()
-		state, err = terraform.ReadState(f)
-		if err != nil {
-			return err
-		}
+	// make sure we're at the start of the file
+	if _, err := s.stateFile.Seek(0, os.SEEK_SET); err != nil {
+		return err
+	}
+
+	state, err := terraform.ReadState(s.stateFile)
+	// if there's no state we just assign the nil return value
+	if err != nil && err != terraform.ErrNoState {
+		return err
 	}
 
 	s.state = state
 	s.readState = state
 	return nil
 }
+
+// return the path for the lockInfo metadata.
+func (s *LocalState) lockInfoPath() string {
+	stateDir, stateName := filepath.Split(s.Path)
+	if stateName == "" {
+		panic("empty state file path")
+	}
+
+	if stateName[0] == '.' {
+		stateName = stateName[1:]
+	}
+
+	return filepath.Join(stateDir, fmt.Sprintf(".%s.lock.info", stateName))
+}
+
+// lockInfo returns the data in a lock info file
+func (s *LocalState) lockInfo() (*lockInfo, error) {
+	path := s.lockInfoPath()
+	infoData, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	info := lockInfo{}
+	err = json.Unmarshal(infoData, &info)
+	if err != nil {
+		return nil, fmt.Errorf("state file %q locked, but could not unmarshal lock info: %s", s.Path, err)
+	}
+	return &info, nil
+}
+
+// write a new lock info file
+func (s *LocalState) writeLockInfo(reason string) error {
+	path := s.lockInfoPath()
+
+	lockInfo := &lockInfo{
+		Path:    s.Path,
+		Time:    time.Now(),
+		Expires: time.Now().Add(time.Hour),
+		Reason:  reason,
+	}
+
+	infoData, err := json.Marshal(lockInfo)
+	if err != nil {
+		panic(fmt.Sprintf("could not marshal lock info: %#v", lockInfo))
+	}
+
+	err = ioutil.WriteFile(path, infoData, 0600)
+	if err != nil {
+		return fmt.Errorf("could not write lock info for %q: %s", s.Path, err)
+	}
+	return nil
+}

state/local_lock_unix.go

@@ -0,0 +1,34 @@
+// +build !windows
+
+package state
+
+import (
+	"os"
+	"syscall"
+)
+
+// use fcntl POSIX locks for the most consistent behavior across platforms, and
+// hopefully some campatibility over NFS and CIFS.
+func (s *LocalState) lock() error {
+	flock := &syscall.Flock_t{
+		Type:   syscall.F_RDLCK | syscall.F_WRLCK,
+		Whence: int16(os.SEEK_SET),
+		Start:  0,
+		Len:    0,
+	}
+
+	fd := s.stateFile.Fd()
+	return syscall.FcntlFlock(fd, syscall.F_SETLK, flock)
+}
+
+func (s *LocalState) unlock() error {
+	flock := &syscall.Flock_t{
+		Type:   syscall.F_UNLCK,
+		Whence: int16(os.SEEK_SET),
+		Start:  0,
+		Len:    0,
+	}
+
+	fd := s.stateFileOut.Fd()
+	return syscall.FcntlFlock(fd, syscall.F_SETLK, flock)
+}

state/local_lock_windows.go

@@ -0,0 +1,140 @@
+// +build windows
+
+package state
+
+import (
+	"math"
+	"os"
+	"syscall"
+	"unsafe"
+)
+
+type stateLock struct {
+	handle syscall.Handle
+}
+
+var (
+	modkernel32      = syscall.NewLazyDLL("kernel32.dll")
+	procLockFileEx   = modkernel32.NewProc("LockFileEx")
+	procCreateEventW = modkernel32.NewProc("CreateEventW")
+
+	lockedFiles = map[*os.File]syscall.Handle{}
+)
+
+const (
+	_LOCKFILE_FAIL_IMMEDIATELY = 1
+	_LOCKFILE_EXCLUSIVE_LOCK   = 2
+)
+
+func (s *LocalState) lock() error {
+	name, err := syscall.UTF16PtrFromString(s.PathOut)
+	if err != nil {
+		return err
+	}
+
+	handle, err := syscall.CreateFile(
+		name,
+		syscall.GENERIC_READ|syscall.GENERIC_WRITE,
+		// since this file is already open in out process, we need shared
+		// access here for this call.
+		syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE,
+		nil,
+		syscall.OPEN_EXISTING,
+		syscall.FILE_ATTRIBUTE_NORMAL,
+		0,
+	)
+	if err != nil {
+		return err
+	}
+
+	lockedFiles[s.stateFileOut] = handle
+
+	// even though we're failing immediately, an overlapped event structure is
+	// required
+	ol, err := newOverlapped()
+	if err != nil {
+		return err
+	}
+	defer syscall.CloseHandle(ol.HEvent)
+
+	return lockFileEx(
+		handle,
+		_LOCKFILE_EXCLUSIVE_LOCK|_LOCKFILE_FAIL_IMMEDIATELY,
+		0,              // reserved
+		0,              // bytes low
+		math.MaxUint32, // bytes high
+		ol,
+	)
+}
+
+func (s *LocalState) unlock() error {
+	handle, ok := lockedFiles[s.stateFileOut]
+	if !ok {
+		// we allow multiple Unlock calls
+		return nil
+	}
+	delete(lockedFiles, s.stateFileOut)
+	return syscall.Close(handle)
+}
+
+func lockFileEx(h syscall.Handle, flags, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) {
+	r1, _, e1 := syscall.Syscall6(
+		procLockFileEx.Addr(),
+		6,
+		uintptr(h),
+		uintptr(flags),
+		uintptr(reserved),
+		uintptr(locklow),
+		uintptr(lockhigh),
+		uintptr(unsafe.Pointer(ol)),
+	)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = error(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+// newOverlapped creates a structure used to track asynchronous
+// I/O requests that have been issued.
+func newOverlapped() (*syscall.Overlapped, error) {
+	event, err := createEvent(nil, true, false, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &syscall.Overlapped{HEvent: event}, nil
+}
+
+func createEvent(sa *syscall.SecurityAttributes, manualReset bool, initialState bool, name *uint16) (handle syscall.Handle, err error) {
+	var _p0 uint32
+	if manualReset {
+		_p0 = 1
+	}
+	var _p1 uint32
+	if initialState {
+		_p1 = 1
+	}
+
+	r0, _, e1 := syscall.Syscall6(
+		procCreateEventW.Addr(),
+		4,
+		uintptr(unsafe.Pointer(sa)),
+		uintptr(_p0),
+		uintptr(_p1),
+		uintptr(unsafe.Pointer(name)),
+		0,
+		0,
+	)
+	handle = syscall.Handle(r0)
+	if handle == syscall.InvalidHandle {
+		if e1 != 0 {
+			err = error(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}

state/local_test.go

@@ -3,6 +3,7 @@ package state
 import (
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"testing"
 
 	"github.com/hashicorp/terraform/terraform"
@@ -14,6 +15,61 @@ func TestLocalState(t *testing.T) {
 	TestState(t, ls)
 }
 
+func TestLocalStateLocks(t *testing.T) {
+	s := testLocalState(t)
+	defer os.Remove(s.Path)
+
+	// lock first
+	if err := s.Lock("test"); err != nil {
+		t.Fatal(err)
+	}
+
+	out, err := exec.Command("go", "run", "testdata/lockstate.go", s.Path).CombinedOutput()
+
+	if err != nil {
+		t.Fatal("unexpected lock failure", err)
+	}
+
+	if string(out) != "lock failed" {
+		t.Fatal("expected 'locked failed', got", string(out))
+	}
+
+	// check our lock info
+	lockInfo, err := s.lockInfo()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if lockInfo.Reason != "test" {
+		t.Fatalf("invalid lock info %#v\n", lockInfo)
+	}
+
+	// a noop, since we unlock on exit
+	if err := s.Unlock(); err != nil {
+		t.Fatal(err)
+	}
+
+	// local locks can re-lock
+	if err := s.Lock("test"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Unlock should be repeatable
+	if err := s.Unlock(); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.Unlock(); err != nil {
+		t.Fatal(err)
+	}
+
+	// make sure lock info is gone
+	lockInfoPath := s.lockInfoPath()
+	if _, err := os.Stat(lockInfoPath); !os.IsNotExist(err) {
+		t.Fatal("lock info not removed")
+	}
+
+}
+
 func TestLocalState_pathOut(t *testing.T) {
 	f, err := ioutil.TempFile("", "tf")
 	if err != nil {

state/testdata/lockstate.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"io"
+	"log"
+	"os"
+
+	"github.com/hashicorp/terraform/state"
+)
+
+// Attempt to open and lock a terraform state file.
+// Lock failure exits with 0 and writes "lock failed" to stderr.
+func main() {
+	if len(os.Args) != 2 {
+		log.Fatal(os.Args[0], "statefile")
+	}
+
+	s := &state.LocalState{
+		Path: os.Args[1],
+	}
+
+	err := s.Lock("test")
+	if err != nil {
+		io.WriteString(os.Stderr, "lock failed")
+
+	}
+	return
+}